diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..c7b7e36fac --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +** +!dist/ +!dist/*.whl +!deploy/ +!deploy/Dockerfile +!deploy/project/ +!deploy/project/sqlbuild_project.toml +!deploy/project/models/ +!deploy/project/models/*.sql +!deploy/project/orchestration/ +!deploy/project/orchestration/*.py diff --git a/.github/diagnostics-recipient.pem b/.github/diagnostics-recipient.pem new file mode 100644 index 0000000000..93ede90365 --- /dev/null +++ b/.github/diagnostics-recipient.pem @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIEJTCCAo2gAwIBAgIUUgxZ6C8ykh+91sFO8z78yEl0flIwDQYJKoZIhvcNAQEL +BQAwIjEgMB4GA1UEAwwXc3FsYnVpbGQtY2ktZGlhZ25vc3RpY3MwHhcNMjYxMTAz +MDQ0ODU4WhcNMzYxMDMxMDQ0ODU4WjAiMSAwHgYDVQQDDBdzcWxidWlsZC1jaS1k +aWFnbm9zdGljczCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBANhDATXs +XtI0hyxJIba3/APxf8cVEZ7dTTHhyqyEpf2Pi5VKWvfLNkTJE+FIlXS37G5o4Hkb +Tnm44fWdQg/UAnQCY5J9QjEGJQAIv+4jx9jcALtbh0VG0hXHuEXBMFi3pakazhVf +rLHUtI4pDuU4Z0Zl7DsivZTgDiOAyCE+uIyCcKg5sn3vq3ITej7vIXaPHbrLV794 +M6hyPbQu8kiREW+0g6iktAiHiw4prYBlbGIVfWoCzkNUEtO6djMKQlg1es8bhCYo +cVOwChPJrgsTX4tpN7I1y6kAYQ3EAd9AcoG+YmZ/Lf0wsawH4BrM0XZWV7Tr93CO +nrLqegrCx8eYn9cweoCsWq4Ad5xDecYnbaxX3c9bFfuDmaU8tUKjm4KXTQgJFznW +0cc1dRSbO7W8ogZyBj9KvRoJPboHiXaZQSHwcx/rwKmjZJ6yDcGTlTD7qY9WciuR +KK5mWLJvsQ33fjMTGnaW5cd73UR3Y6phvHv1s4HpVeAWOjWJAVuEzDjU1QIDAQAB +o1MwUTAdBgNVHQ4EFgQUg0RhY1QY4ID4KbGS710v48wx+jQwHwYDVR0jBBgwFoAU +g0RhY1QY4ID4KbGS710v48wx+jQwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B +AQsFAAOCAYEAlMrRv+zYmWbdSTM5vZM1qJMlDifRbRqTqcbMzv8W0tOrm+Ch2b94 +VaPW0lCs+hOEUfAP2osegfLS6eAlv3MW774iVZzEcHjoQpvQQ2biaPgpJTe4jV+U +iwWlYrBbOjFPOQsnvxHSKjsrkt5bHcEXxShSkCYLjS+HvUT2huZ5LGxBUR9yfia2 +gkOIa1zlPlbAMyUt6Fj66NL2+nbsCYLuGfQafqF4UUd3rG2sF1pkxSEuof1VpKTQ +rKU9u0+csNFHF20CGZ7h9jUU1ATLLwBlPunFCCO70gIGcoNspqPRIOwsEXXGr81N +aXJAeA5XMesS/v0k0x2GBLXujTT5+kC3iQA0+OfxH8t82D4xTXd2wYIKYwnzK+nv +XkF3Of+Y/puUv7R+ADfG0K7aMdLkBDjllJJL7gUml+X281m+2S5UlJZIHxe2kYGB +/i2XCahytLRBY9EGVSLEWjkhfnOydpjwuHfjkNaJDqNRDexMbzifxi0/liv8jbvV +ZCqMfibxPUoT +-----END CERTIFICATE----- diff --git a/.github/workflows/build-distributions.yml b/.github/workflows/build-distributions.yml index ce6ed51bfb..e2c5b8e127 100644 --- a/.github/workflows/build-distributions.yml +++ b/.github/workflows/build-distributions.yml @@ -189,6 +189,11 @@ jobs: "${run_as[@]}" "${smoke_scripts}/sqb" --version "${run_as[@]}" "${smoke_scripts}/sqb" --help "${run_as[@]}" "${smoke_scripts}/sqlbuild" --help + if [[ "${{ inputs.prebuilt-run-id == '' }}" == "true" ]]; then + cd "${GITHUB_WORKSPACE}" + "${run_as[@]}" "${smoke_python}" -m tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.helpers \ + "${smoke_scripts}/sqb${{ runner.os == 'Windows' && '.exe' || '' }}" + fi - name: Check wheel contents and run a native command if: matrix.platform.name == 'linux-x86_64' run: | diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 3f51d3fbb9..cd908d1d90 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -142,7 +142,7 @@ jobs: *) docs_only=false ;; esac case "${path}" in - crates/*|Cargo.toml|Cargo.lock|pyproject.toml|.github/workflows/verify.yml|.github/workflows/build-distributions.yml) + crates/*|Cargo.toml|Cargo.lock|pyproject.toml|.github/workflows/verify.yml|.github/workflows/build-distributions.yml|deploy/Dockerfile|deploy/project/*|deploy/compose*|compose.yml|.dockerignore|Makefile|tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py|tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/*|benchmarks/duckshop/edge_cases/orchestration/container_deployment.json) wheels=true ;; esac @@ -249,6 +249,8 @@ jobs: run: uv python install 3.12 - name: Verify lockfile run: uv lock --check --python 3.12 + - name: Verify native workspace lockfile + run: cargo metadata --locked --format-version 1 > /dev/null checks: name: Static checks and unit + integration tests @@ -279,13 +281,59 @@ jobs: run: uv python install 3.12 - name: Install dependencies run: uv sync --locked --all-groups --all-extras + - uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: clients/typescript/package-lock.json + - name: Check generated control client against the built native API + run: | + npm --prefix clients/typescript ci + uv run --no-sync npm --prefix clients/typescript run check + uv run --no-sync npm --prefix clients/typescript test - name: Run static checks run: make check-ci - - name: Run native checks - run: make rust-check + - name: Prepare Docker worker tests + run: | + docker info > /dev/null + docker pull alpine:3.23 - name: Run unit and integration tests + env: + SQLBUILD_REQUIRE_DOCKER_TESTS: "1" run: make test + native_checks: + name: Native workspace checks + needs: [changes, lockfile] + if: needs.changes.outputs.checks == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + version: "0.12.3" + enable-cache: true + cache-dependency-glob: | + uv.lock + Cargo.lock + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - uses: Swatinem/rust-cache@v2 + with: + shared-key: verify-checks + save-if: false + - name: Install Python + run: uv python install 3.12 + - name: Install native test dependencies without rebuilding the packages + run: uv sync --locked --all-groups --all-extras --no-install-package sqlbuild --no-install-package sqlbuild-cli + - name: Run native checks + env: + UV_NO_SYNC: "1" + PYO3_PYTHON: ${{ github.workspace }}/.venv/bin/python + run: make rust-check + e2e_shards: name: DuckDB E2E (${{ matrix.name }}) needs: [changes, lockfile] @@ -345,8 +393,23 @@ jobs: f"{os.environ.get('PYTEST_XDIST_AUTO_NUM_WORKERS', '')}" ) PY + - name: Validate diagnostic encryption recipient + if: matrix.target == 'test-e2e-duckdb-cli' + run: openssl x509 -in .github/diagnostics-recipient.pem -noout - name: Run DuckDB E2E shard + env: + SQLBUILD_COMPARISON_ENCRYPTED_DIR: ${{ matrix.target == 'test-e2e-duckdb-cli' && format('{0}/comparison-encrypted', runner.temp) || '' }} + SQLBUILD_COMPARISON_RECIPIENT: ${{ github.workspace }}/.github/diagnostics-recipient.pem run: make ${{ matrix.target }} + - name: Retain encrypted comparison failures + if: failure() && matrix.target == 'test-e2e-duckdb-cli' + uses: actions/upload-artifact@v4 + with: + name: encrypted-comparison-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/comparison-encrypted/failure-*/failure.cms + if-no-files-found: warn + compression-level: 0 + retention-days: 7 compiler_differential: name: Compiler golden outputs @@ -966,6 +1029,33 @@ jobs: uses: ./.github/workflows/build-distributions.yml with: sdist: false + upload: true + retention-days: 3 + + compose: + name: Packaged Compose lifecycle + needs: [changes, lockfile, wheels] + if: needs.changes.outputs.wheels == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + version: "0.12.3" + enable-cache: true + - name: Install Python test dependencies without rebuilding the packages + run: | + uv python install 3.12 + uv sync --locked --all-groups --all-extras --no-install-package sqlbuild --no-install-package sqlbuild-cli + - uses: actions/download-artifact@v4 + with: + name: dist-wheel-linux-x86_64 + path: dist + - name: Install the same wheels used by the images + run: uv pip install --no-deps dist/*.whl + - name: Verify packaged Compose deployment and operational recovery + run: make test-compose-packaged verify: # The required Verify of a Release Please pull request is the commit status that the trusted @@ -985,6 +1075,7 @@ jobs: changes, lockfile, checks, + native_checks, e2e_shards, compiler_differential, compiler_differential_python_versions, @@ -999,6 +1090,7 @@ jobs: postgres, clickhouse, wheels, + compose, ] runs-on: ubuntu-24.04 steps: @@ -1008,6 +1100,7 @@ jobs: CHECKS_SELECTED: ${{ needs.changes.outputs.checks }} LOCKFILE_RESULT: ${{ needs.lockfile.result }} CHECKS_RESULT: ${{ needs.checks.result }} + NATIVE_CHECKS_RESULT: ${{ needs.native_checks.result }} E2E_SHARDS_RESULT: ${{ needs.e2e_shards.result }} COMPILER_DIFFERENTIAL_RESULT: ${{ needs.compiler_differential.result }} COMPILER_DIFFERENTIAL_PYTHON_VERSIONS_RESULT: ${{ needs.compiler_differential_python_versions.result }} @@ -1022,14 +1115,21 @@ jobs: POSTGRES_RESULT: ${{ needs.postgres.result }} CLICKHOUSE_RESULT: ${{ needs.clickhouse.result }} WHEELS_RESULT: ${{ needs.wheels.result }} + COMPOSE_RESULT: ${{ needs.compose.result }} run: | if [[ "${CHANGES_RESULT}" != "success" || "${LOCKFILE_RESULT}" != "success" ]]; then echo "Required classification or lockfile checks did not pass" >&2 exit 1 fi - if [[ "${CHECKS_RESULT}" != "success" ]] && \ - ! [[ "${CHECKS_SELECTED}" == "false" && "${CHECKS_RESULT}" == "skipped" ]]; then - echo "Required static checks did not pass: ${CHECKS_RESULT}" >&2 + for result in "${CHECKS_RESULT}" "${NATIVE_CHECKS_RESULT}"; do + if [[ "${result}" != "success" ]] && \ + ! [[ "${CHECKS_SELECTED}" == "false" && "${result}" == "skipped" ]]; then + echo "Required static or native checks did not pass: ${result}" >&2 + exit 1 + fi + done + if [[ "${WHEELS_RESULT}" == "success" && "${COMPOSE_RESULT}" != "success" ]]; then + echo "Required packaged Compose verification did not pass: ${COMPOSE_RESULT}" >&2 exit 1 fi for result in \ @@ -1046,7 +1146,8 @@ jobs: "${RULES_PERFORMANCE_RESULT}" \ "${POSTGRES_RESULT}" \ "${CLICKHOUSE_RESULT}" \ - "${WHEELS_RESULT}"; do + "${WHEELS_RESULT}" \ + "${COMPOSE_RESULT}"; do if [[ "${result}" != "success" && "${result}" != "skipped" ]]; then echo "A selected verification suite did not pass: ${result}" >&2 exit 1 diff --git a/.gitignore b/.gitignore index 3bf9e0897f..249dfe1fa1 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,7 @@ target/ *.dylib .dlt/ .amp/portals/* +/.env # Local SQLBuild overrides sqlbuild_local.yml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f0ab39173e..ba45c64ca6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,6 +42,19 @@ For a change confined to one Rust crate, run its focused tests before rebuilding cargo test -p sqlbuild-sqltext given_model_files_when_matching_headers_then_offsets_follow_the_python_pattern ``` +Control-plane integration tests require Docker and run both SQLite and PostgreSQL: + +```bash +make test-native NATIVE_TEST_ARGS='-p sqlbuild-orchestration --lib' +``` + +The same Testcontainers harness runs under `make rust-check` in CI. Each invocation starts one +digest-pinned PostgreSQL container, waits for database readiness, shares it across the suite with +an isolated schema per fixture, and removes it after success or failure. No host PostgreSQL setup +is needed. Missing Docker, startup failures or missing database configuration fail the gate; +PostgreSQL cases never silently skip. `NATIVE_TEST_ARGS` accepts Cargo test arguments; its default +runs the workspace with all features. + Keep the toolchain, `CARGO_TARGET_DIR`, profile and `RUSTFLAGS` consistent between iterations. Changing them can invalidate compiled artifacts. Coordinate builds that share a target directory; do not delete another task's artifacts or change global compiler-cache settings. The first build diff --git a/Cargo.lock b/Cargo.lock index 07324a98e1..6f4ef38c92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,23 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "aho-corasick" version = "1.1.5" @@ -11,6 +28,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + [[package]] name = "ar_archive_writer" version = "0.5.3" @@ -20,6 +52,27 @@ dependencies = [ "object", ] +[[package]] +name = "arc-swap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380962b3d93b99642e50b9febf72c071e97e8b32d86eae9cf4ef582bb960ae6e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "argon2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" +dependencies = [ + "base64ct", + "blake2", + "cpufeatures 0.2.17", + "password-hash", +] + [[package]] name = "arraydeque" version = "0.5.1" @@ -33,1829 +86,5853 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] -name = "attribute-derive" -version = "0.10.5" +name = "asn1-rs" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05832cdddc8f2650cc2cc187cc2e952b8c133a48eb055f35211f61ee81502d77" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.21", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ - "attribute-derive-macro", - "derive-where", - "manyhow", "proc-macro2", "quote", "syn 2.0.119", + "synstructure 0.13.2", ] [[package]] -name = "attribute-derive-macro" -version = "0.10.5" +name = "asn1-rs-impl" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a7cdbbd4bd005c5d3e2e9c885e6fa575db4f4a3572335b974d8db853b6beb61" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ - "collection_literals", - "interpolator", - "manyhow", - "proc-macro-utils", "proc-macro2", "quote", - "quote-use", "syn 2.0.119", ] [[package]] -name = "autocfg" -version = "1.5.1" +name = "async-broadcast" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +checksum = "435a87a52755b8f27fcf321ac4f04b2802e337c8c4872923137471ec39c37532" +dependencies = [ + "event-listener", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] [[package]] -name = "bitflags" -version = "2.13.1" +name = "async-channel" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] [[package]] -name = "blake3" -version = "1.8.7" +name = "async-io" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" dependencies = [ - "arrayvec", - "cc", + "autocfg", "cfg-if", - "constant_time_eq", - "cpufeatures 0.3.1", - "rayon-core", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "windows-sys 0.61.2", ] [[package]] -name = "block-buffer" -version = "0.10.4" +name = "async-lock" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "0876700b0d007ad25d5508865a4259c3af9af6d296bf663a0d586d0bc3369f6f" dependencies = [ - "generic-array", + "event-listener", + "event-listener-strategy", + "pin-project-lite", ] [[package]] -name = "block2" -version = "0.6.2" +name = "async-process" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +checksum = "fc50921ec0055cdd8a16de48773bfeec5c972598674347252c0399676be7da75" dependencies = [ - "objc2", + "async-channel", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener", + "futures-lite", + "rustix", ] [[package]] -name = "bstr" -version = "1.13.1" +name = "async-recursion" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +checksum = "4f8abc12baad266b1c8cec146854c195b5864b4221d4b2ca7296a7ae82d9e451" dependencies = [ - "memchr", - "regex-automata", - "serde_core", + "proc-macro2", + "quote", + "syn 3.0.3", ] [[package]] -name = "castaway" -version = "0.2.4" +name = "async-signal" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485" dependencies = [ - "rustversion", + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix", + "signal-hook-registry", + "slab", + "windows-sys 0.61.2", ] [[package]] -name = "cc" -version = "1.4.4" +name = "async-task" +version = "4.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" -dependencies = [ - "find-msvc-tools", - "shlex", -] +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] -name = "cfg-if" -version = "1.0.4" +name = "async-trait" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] [[package]] -name = "cfg_aliases" -version = "0.2.2" +name = "atoi" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] [[package]] -name = "collection_literals" -version = "1.0.3" +name = "atomic-waker" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] -name = "command-group" -version = "5.0.1" +name = "attribute-derive" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a68fa787550392a9d58f44c21a3022cfb3ea3e2458b7f85d3b399d0ceeccf409" +checksum = "05832cdddc8f2650cc2cc187cc2e952b8c133a48eb055f35211f61ee81502d77" dependencies = [ - "nix 0.27.1", - "winapi", + "attribute-derive-macro", + "derive-where", + "manyhow", + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] -name = "compact_str" -version = "0.9.1" +name = "attribute-derive-macro" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +checksum = "0a7cdbbd4bd005c5d3e2e9c885e6fa575db4f4a3572335b974d8db853b6beb61" dependencies = [ - "castaway", - "cfg-if", - "itoa", - "rustversion", - "ryu", - "static_assertions", + "collection_literals", + "interpolator", + "manyhow", + "proc-macro-utils", + "proc-macro2", + "quote", + "quote-use", + "syn 2.0.119", ] [[package]] -name = "constant_time_eq" -version = "0.4.2" +name = "autocfg" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] -name = "cpufeatures" -version = "0.2.17" +name = "axum" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ - "libc", + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", ] [[package]] -name = "cpufeatures" -version = "0.3.1" +name = "axum-core" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ - "libc", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", ] [[package]] -name = "crossbeam-deque" -version = "0.8.8" +name = "axum-server" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +checksum = "b1df331683d982a0b9492b38127151e6453639cd34926eb9c07d4cd8c6d22bfc" dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", + "arc-swap", + "bytes", + "either", + "fs-err", + "http", + "http-body", + "hyper", + "hyper-util", + "pin-project-lite", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", ] [[package]] -name = "crossbeam-epoch" -version = "0.9.21" +name = "base16ct" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" -dependencies = [ - "crossbeam-utils", -] +checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" [[package]] -name = "crossbeam-utils" -version = "0.8.23" +name = "base64" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] -name = "crypto-common" -version = "0.1.7" +name = "base64" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" [[package]] -name = "ctrlc" -version = "3.5.2" +name = "base64ct" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" -dependencies = [ - "dispatch2", - "nix 0.31.3", - "windows-sys", -] +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] -name = "derive-where" -version = "1.7.0" +name = "bit-set" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e2b94854e8576378ccda7c8de8a66ed8b4e8acbd2c50ec3418ea6c8aaf4b567" +checksum = "56d87354e4229f54a44f7bf2435906a4656dba36026ab6eaca629a2c436a691c" dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", + "bit-vec 0.10.1", ] [[package]] -name = "digest" -version = "0.10.7" +name = "bit-vec" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" dependencies = [ - "block-buffer", - "crypto-common", + "serde", ] [[package]] -name = "dispatch2" -version = "0.3.1" +name = "bit-vec" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +checksum = "5727b15fa97d4f4fee0a3b7c3d550ed0269f54329207b86388de918604e31269" dependencies = [ - "bitflags", - "block2", - "libc", - "objc2", + "borsh", + "serde", ] [[package]] -name = "either" -version = "1.18.0" +name = "bitflags" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] -name = "equivalent" -version = "1.0.2" +name = "bitflags" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] -name = "errno" -version = "0.3.14" +name = "blake2" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "libc", - "windows-sys", + "digest 0.10.7", ] [[package]] -name = "fallible-iterator" -version = "0.3.0" +name = "blake3" +version = "1.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.1", + "rayon-core", +] [[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" +name = "block-buffer" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] [[package]] -name = "fastrand" -version = "2.5.0" +name = "block-buffer" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] [[package]] -name = "fensu-policy" -version = "0.22.0" +name = "block-padding" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c2266bd27bdfab3a847ca80bcac56aa6605d1f19ffe910910a409edbe5b9a4a" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" dependencies = [ - "command-group", - "globset", - "serde", - "serde_json", - "sha2", + "generic-array", ] [[package]] -name = "find-msvc-tools" -version = "0.1.11" +name = "block2" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] [[package]] -name = "foldhash" -version = "0.1.5" +name = "blocking" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] [[package]] -name = "generic-array" -version = "0.14.7" +name = "borsh" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" dependencies = [ - "typenum", - "version_check", + "borsh-derive", + "bytes", + "cfg_aliases", ] [[package]] -name = "get-size-derive2" -version = "0.10.3" +name = "borsh-derive" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c736d226c32e496b8377813b52269e11ad3a48d8373b68862d0364f04fd1229d" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" dependencies = [ - "attribute-derive", + "once_cell", + "proc-macro-crate", + "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] -name = "get-size2" -version = "0.10.1" +name = "bs58" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "823645bc6404ae2915707777061a47d3a031a9ee0bff51b34ec973df3d8d2990" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" dependencies = [ - "compact_str", - "get-size-derive2", - "hashbrown 0.17.1", - "ordermap", - "smallvec", - "thin-vec", + "tinyvec", ] [[package]] -name = "getopts" -version = "0.2.24" +name = "bstr" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" dependencies = [ - "unicode-width", + "memchr", + "regex-automata", + "serde_core", ] [[package]] -name = "getrandom" -version = "0.2.17" +name = "bumpalo" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] -name = "getrandom" -version = "0.4.3" +name = "byteorder" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" -dependencies = [ - "cfg-if", - "libc", - "r-efi", -] +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] -name = "globset" -version = "0.4.19" +name = "bytes" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd" -dependencies = [ - "aho-corasick", - "bstr", - "log", - "regex-automata", - "regex-syntax", -] +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] -name = "hashbrown" -version = "0.15.5" +name = "castaway" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" dependencies = [ - "foldhash", + "rustversion", ] [[package]] -name = "hashbrown" -version = "0.17.1" +name = "cbc" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] [[package]] -name = "hashlink" -version = "0.10.0" +name = "cc" +version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" dependencies = [ - "hashbrown 0.15.5", + "find-msvc-tools", + "shlex", ] [[package]] -name = "heck" -version = "0.5.0" +name = "cfg-if" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] -name = "indexmap" -version = "2.14.2" +name = "cfg_aliases" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" -dependencies = [ - "equivalent", - "hashbrown 0.17.1", -] +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] -name = "indoc" -version = "2.0.7" +name = "chacha20" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ - "rustversion", + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", ] [[package]] -name = "interpolator" -version = "0.5.0" +name = "chrono" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] [[package]] -name = "is-macro" -version = "0.3.8" +name = "chrono-tz" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8267aa6001e25494f3015f9663bbd88a18240c74483afa5f0934a1b3e4c388e9" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.119", + "chrono", + "phf 0.12.1", ] [[package]] -name = "itertools" -version = "0.15.0" +name = "cipher" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "either", + "crypto-common 0.1.7", + "inout", ] [[package]] -name = "itoa" -version = "1.0.18" +name = "cmov" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" [[package]] -name = "junction" -version = "2.1.0" +name = "collection_literals" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde48f36530cbd91b65f050101e6c056d6eb743cea2a8f0f19beef69c6742ff8" +checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" + +[[package]] +name = "command-fds" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b60b5124979fccd9addd89d8b97a1d6eebb4950694520c75ddd722535ea443f" dependencies = [ - "scopeguard", - "windows-sys", + "nix 0.31.3", + "thiserror 2.0.21", ] [[package]] -name = "libc" -version = "0.2.189" +name = "command-group" +version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "a68fa787550392a9d58f44c21a3022cfb3ea3e2458b7f85d3b399d0ceeccf409" +dependencies = [ + "nix 0.27.1", + "winapi", +] [[package]] -name = "libmimalloc-sys" -version = "0.1.49" +name = "compact_str" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" dependencies = [ - "cc", + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", ] [[package]] -name = "libsqlite3-sys" -version = "0.35.0" +name = "concurrent-queue" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" dependencies = [ - "cc", - "pkg-config", - "vcpkg", + "crossbeam-utils", ] [[package]] -name = "linux-raw-sys" -version = "0.12.1" +name = "const-oid" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] -name = "log" -version = "0.4.33" +name = "const-oid" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" [[package]] -name = "manyhow" -version = "0.11.4" +name = "constant_time_eq" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b33efb3ca6d3b07393750d4030418d594ab1139cee518f0dc88db70fec873587" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ - "manyhow-macros", - "proc-macro2", - "quote", - "syn 2.0.119", + "core-foundation-sys", + "libc", ] [[package]] -name = "manyhow-macros" -version = "0.11.4" +name = "core-foundation" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fce34d199b78b6e6073abf984c9cf5fd3e9330145a93ee0738a7443e371495" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" dependencies = [ - "proc-macro-utils", - "proc-macro2", - "quote", + "core-foundation-sys", + "libc", ] [[package]] -name = "memchr" -version = "2.8.3" +name = "core-foundation-sys" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] -name = "memoffset" -version = "0.9.1" +name = "core_detect" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" -dependencies = [ - "autocfg", -] +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" [[package]] -name = "mimalloc" -version = "0.1.52" +name = "cpubits" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" -dependencies = [ - "libmimalloc-sys", -] +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" [[package]] -name = "nix" -version = "0.27.1" +name = "cpufeatures" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ - "bitflags", - "cfg-if", "libc", ] [[package]] -name = "nix" -version = "0.31.3" +name = "cpufeatures" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ - "bitflags", - "cfg-if", - "cfg_aliases", "libc", ] [[package]] -name = "objc2" -version = "0.6.5" +name = "crc" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08849bbd4767dfae9457696856ae1c84fe4e0281bbe4a7abff2d0e06fb7981f8" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" dependencies = [ - "objc2-encode", + "crc-catalog", ] [[package]] -name = "objc2-encode" -version = "4.1.0" +name = "crc-catalog" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] -name = "object" -version = "0.39.1" +name = "crc32fast" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" dependencies = [ - "memchr", + "cfg-if", ] [[package]] -name = "once_cell" -version = "1.21.4" +name = "croner" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +checksum = "4aa42bcd3d846ebf66e15bd528d1087f75d1c6c1c66ebff626178a106353c576" +dependencies = [ + "chrono", + "derive_builder", + "strum", +] [[package]] -name = "ordermap" -version = "1.2.2" +name = "crossbeam-deque" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a756ddcdeb6c0efc8927bdf0ed112885815a35ba350472a467be5104f01ca1" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" dependencies = [ - "indexmap", + "crossbeam-epoch", + "crossbeam-utils", ] [[package]] -name = "phf" -version = "0.11.3" +name = "crossbeam-epoch" +version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" dependencies = [ - "phf_shared", + "crossbeam-utils", ] [[package]] -name = "phf_codegen" -version = "0.11.3" +name = "crossbeam-queue" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" dependencies = [ - "phf_generator", - "phf_shared", + "crossbeam-utils", ] [[package]] -name = "phf_generator" -version = "0.11.3" +name = "crossbeam-utils" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" -dependencies = [ - "phf_shared", - "rand", -] +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] -name = "phf_shared" -version = "0.11.3" +name = "crypto-bigint" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +checksum = "9c035cb64cba03dad2fb921f3e218785e8cf9a41790da5eb3a1400f62fa2be11" dependencies = [ - "siphasher", + "cpubits", + "ctutils", + "getrandom 0.4.3", + "hybrid-array", + "num-traits", + "rand_core 0.10.1", + "subtle", + "zeroize", ] [[package]] -name = "pkg-config" -version = "0.3.34" +name = "crypto-common" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" - -[[package]] -name = "polyglot-sql" -version = "0.13.9" -source = "git+https://github.com/sqlstack-io/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ - "polyglot-sql-ast-derive", - "polyglot-sql-function-catalogs", - "regex", - "serde", - "serde_json", - "stacker", - "thiserror 1.0.69", - "unicode-segmentation", + "generic-array", + "typenum", ] [[package]] -name = "polyglot-sql-ast-derive" -version = "0.13.9" -source = "git+https://github.com/sqlstack-io/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", + "getrandom 0.4.3", + "hybrid-array", + "rand_core 0.10.1", ] [[package]] -name = "polyglot-sql-function-catalogs" -version = "0.13.9" -source = "git+https://github.com/sqlstack-io/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" +name = "ctrlc" +version = "3.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" +dependencies = [ + "dispatch2", + "nix 0.31.3", + "windows-sys 0.61.2", +] [[package]] -name = "portable-atomic" -version = "1.15.0" +name = "ctutils" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" +checksum = "a2376ffd6f96d8a9db59875b4b6b0237d6900b5baf64279714e05493ce64828a" +dependencies = [ + "cmov", + "subtle", +] [[package]] -name = "ppv-lite86" -version = "0.2.21" +name = "curve25519-dalek" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" dependencies = [ - "zerocopy", + "cfg-if", + "cpufeatures 0.3.1", + "curve25519-dalek-derive", + "digest 0.11.3", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", ] [[package]] -name = "proc-macro-utils" -version = "0.10.0" +name = "curve25519-dalek-derive" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeaf08a13de400bc215877b5bdc088f241b12eb42f0a548d3390dc1c56bb7071" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "smallvec", + "syn 2.0.119", ] [[package]] -name = "proc-macro2" -version = "1.0.107" +name = "darling" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "unicode-ident", + "darling_core 0.20.11", + "darling_macro 0.20.11", ] [[package]] -name = "psm" -version = "0.1.32" +name = "darling" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" dependencies = [ - "ar_archive_writer", - "cc", + "darling_core 0.24.1", + "darling_macro 0.24.1", ] [[package]] -name = "pyo3" -version = "0.26.0" +name = "darling_core" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba0117f4212101ee6544044dae45abe1083d30ce7b29c4b5cbdfa2354e07383" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" dependencies = [ - "indoc", - "libc", - "memoffset", - "once_cell", - "portable-atomic", - "pyo3-build-config", - "pyo3-ffi", - "pyo3-macros", - "unindent", + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", ] [[package]] -name = "pyo3-build-config" -version = "0.26.0" +name = "darling_core" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fc6ddaf24947d12a9aa31ac65431fb1b851b8f4365426e182901eabfb87df5f" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" dependencies = [ - "target-lexicon", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.3", ] [[package]] -name = "pyo3-ffi" -version = "0.26.0" +name = "darling_macro" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "025474d3928738efb38ac36d4744a74a400c901c7596199e20e45d98eb194105" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "libc", - "pyo3-build-config", + "darling_core 0.20.11", + "quote", + "syn 2.0.119", ] [[package]] -name = "pyo3-macros" -version = "0.26.0" +name = "darling_macro" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e64eb489f22fe1c95911b77c44cc41e7c19f3082fc81cce90f657cdc42ffded" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ - "proc-macro2", - "pyo3-macros-backend", + "darling_core 0.24.1", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] -name = "pyo3-macros-backend" -version = "0.26.0" +name = "data-encoding" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "100246c0ecf400b475341b8455a9213344569af29a3c841d29270e53102e0fcf" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "dbus" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ab69f03cc8c4340c9c8e315114e1658e6775a9b16a04357973aa21cec22b32e" dependencies = [ - "heck", - "proc-macro2", - "pyo3-build-config", - "quote", - "syn 2.0.119", + "libc", + "libdbus-sys", + "windows-sys 0.61.2", ] [[package]] -name = "quote" -version = "1.0.47" +name = "dbus-secret-service" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +checksum = "708b509edf7889e53d7efb0ffadd994cc6c2345ccb62f55cfd6b0682165e4fa6" dependencies = [ - "proc-macro2", + "aes", + "block-padding", + "cbc", + "dbus", + "fastrand", + "hkdf", + "num", + "once_cell", + "openssl", + "sha2 0.10.9", + "zeroize", ] [[package]] -name = "quote-use" -version = "0.8.4" +name = "defmt" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9619db1197b497a36178cfc736dc96b271fe918875fbf1344c436a7e93d0321e" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" dependencies = [ - "quote", - "quote-use-macros", + "bitflags 1.3.2", + "defmt-macros", ] [[package]] -name = "quote-use-macros" -version = "0.8.4" +name = "defmt-macros" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82ebfb7faafadc06a7ab141a6f67bcfb24cb8beb158c6fe933f2f035afa99f35" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" dependencies = [ - "proc-macro-utils", + "defmt-parser", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] -name = "r-efi" -version = "6.0.0" +name = "defmt-parser" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror 2.0.21", +] [[package]] -name = "rand" -version = "0.8.8" +name = "der" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "libc", - "rand_chacha", - "rand_core", + "const-oid 0.9.6", + "pem-rfc7468 0.7.0", + "zeroize", ] [[package]] -name = "rand_chacha" -version = "0.3.1" +name = "der" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +checksum = "6a996a108feb8b427a6e12fcf6af4aa83409b79d72c15337d7741368131ee004" dependencies = [ - "ppv-lite86", - "rand_core", + "const-oid 0.10.2", + "pem-rfc7468 1.0.0", + "zeroize", ] [[package]] -name = "rand_core" -version = "0.6.4" +name = "der-parser" +version = "10.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" dependencies = [ - "getrandom 0.2.17", + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", ] [[package]] -name = "rayon" -version = "1.12.0" +name = "deranged" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" dependencies = [ - "either", - "rayon-core", + "serde_core", ] [[package]] -name = "rayon-core" -version = "1.13.0" +name = "derive-where" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +checksum = "2e2b94854e8576378ccda7c8de8a66ed8b4e8acbd2c50ec3418ea6c8aaf4b567" dependencies = [ - "crossbeam-deque", - "crossbeam-utils", + "proc-macro2", + "quote", + "syn 3.0.3", ] [[package]] -name = "regex" -version = "1.13.1" +name = "derive_builder" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", + "derive_builder_macro", ] [[package]] -name = "regex-automata" -version = "0.4.18" +name = "derive_builder_core" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] -name = "regex-syntax" -version = "0.8.11" +name = "derive_builder_macro" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] [[package]] -name = "ruff_python_ast" -version = "0.0.4" +name = "digest" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12d40253033e38020db1bdb91a2cf5d0bee1e755c1938a26d43fa3c73d2b426" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "aho-corasick", - "arrayvec", - "bitflags", - "compact_str", - "get-size2", - "is-macro", - "memchr", - "ruff_python_trivia", - "ruff_source_file", - "ruff_text_size", - "rustc-hash", - "thin-vec", - "thiserror 2.0.21", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", + "subtle", ] [[package]] -name = "ruff_python_parser" -version = "0.0.4" +name = "digest" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "154345bd168f768a36f620180eabc9a5bae844d8c853182b1a24bb1a984c9b00" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "bitflags", - "bstr", - "compact_str", - "get-size2", - "memchr", - "ruff_python_ast", - "ruff_python_trivia", - "ruff_text_size", - "rustc-hash", - "static_assertions", - "thin-vec", - "unicode-ident", - "unicode-normalization", - "unicode_names2", + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", ] [[package]] -name = "ruff_python_trivia" -version = "0.0.4" +name = "dispatch2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dc22c8e07f58267a6c517ab87df57f503ef92927cd0fc46c8445ff318a39e54" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "itertools", - "ruff_source_file", - "ruff_text_size", - "rustc-hash", - "unicode-ident", + "bitflags 2.13.1", + "block2", + "libc", + "objc2", ] [[package]] -name = "ruff_source_file" -version = "0.0.4" +name = "displaydoc" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a7f467940ad19aead60f1a0d7ddbb08aa69f3bbfa49ab271886fb0d98459952" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ - "memchr", - "ruff_text_size", + "proc-macro2", + "quote", + "syn 3.0.3", ] [[package]] -name = "ruff_text_size" -version = "0.0.4" +name = "dotenvy" +version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dca33c148ef1d49a0dd430cdc05125e9c63a3c93028a2cbe7f6ffb3e5bbc4c7" -dependencies = [ - "get-size2", -] +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" [[package]] -name = "rusqlite" -version = "0.37.0" +name = "dyn-clone" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "ecdsa" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" dependencies = [ - "bitflags", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec", + "der 0.8.3", + "digest 0.11.3", + "elliptic-curve", + "rfc6979", + "signature 3.0.0", + "spki 0.8.1", + "zeroize", ] [[package]] -name = "rustc-hash" -version = "2.1.3" +name = "ed25519" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" +checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" +dependencies = [ + "pkcs8 0.11.0", + "signature 3.0.0", +] [[package]] -name = "rustix" -version = "1.1.4" +name = "ed25519-dalek" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de" dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys", - "windows-sys", + "curve25519-dalek", + "ed25519", + "serde", + "sha2 0.11.1", + "signature 3.0.0", + "subtle", + "zeroize", ] [[package]] -name = "rustversion" -version = "1.0.23" +name = "either" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" [[package]] -name = "ryu" -version = "1.0.23" +name = "elliptic-curve" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +checksum = "900f37e7e18d74a2347389af6cd69e3a30d5cc9e697f40004865679aa75e9c7c" +dependencies = [ + "base16ct", + "crypto-bigint", + "crypto-common 0.2.2", + "digest 0.11.3", + "ff", + "group", + "hybrid-array", + "pem-rfc7468 1.0.0", + "pkcs8 0.11.0", + "rand_core 0.10.1", + "sec1", + "subtle", + "zeroize", +] [[package]] -name = "same-file" -version = "1.0.6" +name = "endi" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099" + +[[package]] +name = "enumflags2" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef" dependencies = [ - "winapi-util", + "enumflags2_derive", + "serde", ] [[package]] -name = "saphyr-parser" -version = "0.1.0" +name = "enumflags2_derive" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7429158804c36e705d9423b7848018e535b3e92d439a763e364ee12670ef473b" +checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" dependencies = [ - "arraydeque", - "thiserror 2.0.21", + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] -name = "scopeguard" -version = "1.2.0" +name = "equivalent" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] -name = "serde" -version = "1.0.229" +name = "errno" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ - "serde_core", - "serde_derive", + "libc", + "windows-sys 0.61.2", ] [[package]] -name = "serde_core" -version = "1.0.229" +name = "etcetera" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ - "serde_derive", + "cfg-if", + "home", + "windows-sys 0.48.0", ] [[package]] -name = "serde_derive" -version = "1.0.229" +name = "event-listener" +version = "5.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", + "parking", + "pin-project-lite", ] [[package]] -name = "serde_json" -version = "1.0.151" +name = "event-listener-strategy" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" dependencies = [ - "indexmap", - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", + "event-listener", + "pin-project-lite", ] [[package]] -name = "serde_spanned" -version = "1.1.1" +name = "fallible-iterator" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fensu-policy" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c2266bd27bdfab3a847ca80bcac56aa6605d1f19ffe910910a409edbe5b9a4a" dependencies = [ - "serde_core", + "command-group", + "globset", + "serde", + "serde_json", + "sha2 0.10.9", ] [[package]] -name = "sha2" -version = "0.10.9" +name = "ff" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +checksum = "a1f686ab92a9fb0eaf188f6c6c87b89490baa6fdb0db4544ba4dc47f7942489f" dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest", + "rand_core 0.10.1", + "subtle", ] [[package]] -name = "shlex" -version = "2.0.1" +name = "fiat-crypto" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" [[package]] -name = "signal-hook" -version = "0.3.18" +name = "find-msvc-tools" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" -dependencies = [ - "libc", - "signal-hook-registry", -] +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" [[package]] -name = "signal-hook-registry" -version = "1.4.8" +name = "flate2" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ - "errno", - "libc", + "crc32fast", + "miniz_oxide", + "zlib-rs", ] [[package]] -name = "siphasher" -version = "1.0.4" +name = "fnv" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] -name = "smallvec" -version = "1.16.3" +name = "foldhash" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" [[package]] -name = "sqlbuild-analysis" -version = "0.187.3" +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" dependencies = [ - "memchr", - "polyglot-sql", - "polyglot-sql-function-catalogs", - "rayon", - "regex", - "rustc-hash", - "serde", - "serde_json", - "sha2", - "sqlbuild-attachments", - "sqlbuild-cache", - "sqlbuild-core", - "sqlbuild-discovery", - "sqlbuild-model-config", - "sqlbuild-render", - "sqlbuild-scopes", - "sqlbuild-sqltext", - "sqlclean", + "foreign-types-shared", ] [[package]] -name = "sqlbuild-attachments" -version = "0.187.3" +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ - "sqlbuild-core", - "sqlbuild-sqltext", + "percent-encoding", ] [[package]] -name = "sqlbuild-build-features" -version = "0.187.3" +name = "fs-err" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c95b673b8f6f7235229ae11c5642d81b04c2e64c1e2fb417bc0cf73ca45f29" dependencies = [ - "arrayvec", - "bstr", - "either", - "errno", - "memchr", - "once_cell", - "regex-automata", - "syn 2.0.119", - "syn 3.0.3", - "thiserror 2.0.21", - "windows-sys", + "autocfg", + "tokio", ] [[package]] -name = "sqlbuild-cache" -version = "0.187.3" +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "get-size-derive2" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c736d226c32e496b8377813b52269e11ad3a48d8373b68862d0364f04fd1229d" +dependencies = [ + "attribute-derive", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "get-size2" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "823645bc6404ae2915707777061a47d3a031a9ee0bff51b34ec973df3d8d2990" +dependencies = [ + "compact_str", + "get-size-derive2", + "hashbrown 0.17.1", + "ordermap", + "smallvec", + "thin-vec", +] + +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "globset" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "group" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" +dependencies = [ + "ff", + "rand_core 0.10.1", + "subtle", +] + +[[package]] +name = "h2" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.14.2", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac 0.12.1", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "subtle", + "typenum", + "zeroize", +] + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots 1.0.11", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" + +[[package]] +name = "icu_properties" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" + +[[package]] +name = "icu_provider" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "interpolator" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + +[[package]] +name = "is-macro" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8267aa6001e25494f3015f9663bbd88a18240c74483afa5f0934a1b3e4c388e9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "js-sys" +version = "0.3.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "junction" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cde48f36530cbd91b65f050101e6c056d6eb743cea2a8f0f19beef69c6742ff8" +dependencies = [ + "scopeguard", + "windows-sys 0.61.2", +] + +[[package]] +name = "keyring" +version = "3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebcc3aff044e5944a8fbaf69eb277d11986064cba30c468730e8b9909fb551c" +dependencies = [ + "byteorder", + "dbus-secret-service", + "log", + "openssl", + "secret-service", + "security-framework 2.11.1", + "security-framework 3.7.0", + "windows-sys 0.60.2", + "zeroize", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" +dependencies = [ + "spin", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libdbus-sys" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "328c4789d42200f1eeec05bd86c9c13c7f091d2ba9a6ea35acdf51f31bc0f043" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libmimalloc-sys" +version = "0.1.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" +dependencies = [ + "cc", +] + +[[package]] +name = "libredox" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" +dependencies = [ + "bitflags 2.13.1", + "libc", + "plain", + "redox_syscall 0.9.5", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + +[[package]] +name = "manyhow" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b33efb3ca6d3b07393750d4030418d594ab1139cee518f0dc88db70fec873587" +dependencies = [ + "manyhow-macros", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "manyhow-macros" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46fce34d199b78b6e6073abf984c9cf5fd3e9330145a93ee0738a7443e371495" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mimalloc" +version = "0.1.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" +dependencies = [ + "libmimalloc-sys", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nix" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "libc", +] + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "cfg_aliases", + "libc", + "memoffset", +] + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.8", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "oauth2" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" +dependencies = [ + "base64 0.22.1", + "chrono", + "getrandom 0.2.17", + "http", + "rand 0.8.8", + "reqwest", + "serde", + "serde_json", + "serde_path_to_error", + "sha2 0.10.9", + "thiserror 1.0.69", + "url", +] + +[[package]] +name = "objc2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08849bbd4767dfae9457696856ae1c84fe4e0281bbe4a7abff2d0e06fb7981f8" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "memchr", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openidconnect" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0d098a59353679624d406825940139e2b2d554a52b64c9fb1b1bc42a11fe5c" +dependencies = [ + "base64 0.22.1", + "chrono", + "dyn-clone", + "ed25519-dalek", + "getrandom 0.4.3", + "hmac 0.12.1", + "http", + "itertools 0.14.0", + "log", + "oauth2", + "p256", + "p384", + "rand 0.8.8", + "rsa", + "serde", + "serde-value", + "serde_json", + "serde_path_to_error", + "serde_plain", + "serde_with", + "sha2 0.10.9", + "subtle", + "thiserror 1.0.69", + "url", +] + +[[package]] +name = "openssl" +version = "0.10.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4de8111e94a56adca7ebfe6c365b6d42f99deb69b74b6becacd4204e2f193aa2" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "openssl-src" +version = "300.6.1+3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19c413d6ec9ebaf5e1c829d355cb3e9f783778d700f135be0921d52fce3c60ae" +dependencies = [ + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-stream" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aa2b01e1d916879f73a53d01d1d6cee68adbb31d6d9177a8cfce093cced1d50" +dependencies = [ + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "ordermap" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a756ddcdeb6c0efc8927bdf0ed112885815a35ba350472a467be5104f01ca1" +dependencies = [ + "indexmap 2.14.2", +] + +[[package]] +name = "p256" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c9239b2dbc807adbbe147e8cf72ea7450c3a0aabe62cb8e75ff4ec22e1f72a" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primefield", + "primeorder", + "sha2 0.11.1", +] + +[[package]] +name = "p384" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d17b851e6b3e378ab4ecb07fa2ed23f4d15f075735f8fec9fa1e7bdce5f8301f" +dependencies = [ + "ecdsa", + "elliptic-curve", + "fiat-crypto", + "primefield", + "primeorder", + "sha2 0.11.1", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "password-hash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "pem" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" +dependencies = [ + "base64 0.23.1", + "serde_core", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_shared 0.11.3", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared 0.12.1", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared 0.11.3", + "rand 0.8.8", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der 0.7.10", + "pkcs8 0.10.2", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der 0.7.10", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" +dependencies = [ + "der 0.8.3", + "spki 0.8.1", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "polyglot-sql" +version = "0.13.9" +source = "git+https://github.com/sqlstack-io/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" +dependencies = [ + "polyglot-sql-ast-derive", + "polyglot-sql-function-catalogs", + "regex", + "serde", + "serde_json", + "stacker", + "thiserror 1.0.69", + "unicode-segmentation", +] + +[[package]] +name = "polyglot-sql-ast-derive" +version = "0.13.9" +source = "git+https://github.com/sqlstack-io/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "polyglot-sql-function-catalogs" +version = "0.13.9" +source = "git+https://github.com/sqlstack-io/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "primefield" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" +dependencies = [ + "crypto-bigint", + "crypto-common 0.2.2", + "ff", + "rand_core 0.10.1", + "subtle", + "zeroize", +] + +[[package]] +name = "primeorder" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06" +dependencies = [ + "elliptic-curve", + "once_cell", + "primefield", + "serdect", + "wnaf", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro-utils" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeaf08a13de400bc215877b5bdc088f241b12eb42f0a548d3390dc1c56bb7071" +dependencies = [ + "proc-macro2", + "quote", + "smallvec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proptest" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8530004ccb15eae51c7e40009fbe317f341f804db54dc033eec1c50be28cfa0" +dependencies = [ + "bit-set", + "bit-vec 0.10.1", + "bitflags 2.13.1", + "chacha20", + "core_detect", + "num-traits", + "rand 0.10.3", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "psm" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" +dependencies = [ + "ar_archive_writer", + "cc", +] + +[[package]] +name = "pyo3" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ba0117f4212101ee6544044dae45abe1083d30ce7b29c4b5cbdfa2354e07383" +dependencies = [ + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc6ddaf24947d12a9aa31ac65431fb1b851b8f4365426e182901eabfb87df5f" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "025474d3928738efb38ac36d4744a74a400c901c7596199e20e45d98eb194105" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e64eb489f22fe1c95911b77c44cc41e7c19f3082fc81cce90f657cdc42ffded" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "100246c0ecf400b475341b8455a9213344569af29a3c841d29270e53102e0fcf" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.21", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.3", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.21", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "quote-use" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9619db1197b497a36178cfc736dc96b271fe918875fbf1344c436a7e93d0321e" +dependencies = [ + "quote", + "quote-use-macros", +] + +[[package]] +name = "quote-use-macros" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82ebfb7faafadc06a7ab141a6f67bcfb24cb8beb158c6fe933f2f035afa99f35" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "rand_xorshift" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60aa6af80be32871323012e02e6e65f8a7cc7890931ae421d217ad8fe0df2ccf" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rcgen" +version = "0.14.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "redox_syscall" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "580c875845b41ecb1564ad1ae4703044cef774145e3d7a94dc2b3fa7c97868fd" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots 1.0.11", +] + +[[package]] +name = "rfc6979" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b" +dependencies = [ + "crypto-bigint", + "hmac 0.13.0", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rpassword" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da316a15f47e3d053de9cb2c439650bd8fa4aaeb9365f2e5f27f492ff73c196" +dependencies = [ + "libc", + "rtoolbox", + "windows-sys 0.61.2", +] + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8 0.10.2", + "rand_core 0.6.4", + "signature 2.2.0", + "spki 0.7.3", + "subtle", + "zeroize", +] + +[[package]] +name = "rtoolbox" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a1efe12a1469752d0e6ff5ebec0b6ef4924cc5c4c71046b0ec730040535819d" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "ruff_python_ast" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12d40253033e38020db1bdb91a2cf5d0bee1e755c1938a26d43fa3c73d2b426" +dependencies = [ + "aho-corasick", + "arrayvec", + "bitflags 2.13.1", + "compact_str", + "get-size2", + "is-macro", + "memchr", + "ruff_python_trivia", + "ruff_source_file", + "ruff_text_size", + "rustc-hash", + "thin-vec", + "thiserror 2.0.21", +] + +[[package]] +name = "ruff_python_parser" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "154345bd168f768a36f620180eabc9a5bae844d8c853182b1a24bb1a984c9b00" +dependencies = [ + "bitflags 2.13.1", + "bstr", + "compact_str", + "get-size2", + "memchr", + "ruff_python_ast", + "ruff_python_trivia", + "ruff_text_size", + "rustc-hash", + "static_assertions", + "thin-vec", + "unicode-ident", + "unicode-normalization", + "unicode_names2", +] + +[[package]] +name = "ruff_python_trivia" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dc22c8e07f58267a6c517ab87df57f503ef92927cd0fc46c8445ff318a39e54" +dependencies = [ + "itertools 0.15.0", + "ruff_source_file", + "ruff_text_size", + "rustc-hash", + "unicode-ident", +] + +[[package]] +name = "ruff_source_file" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a7f467940ad19aead60f1a0d7ddbb08aa69f3bbfa49ab271886fb0d98459952" +dependencies = [ + "memchr", + "ruff_text_size", +] + +[[package]] +name = "ruff_text_size" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dca33c148ef1d49a0dd430cdc05125e9c63a3c93028a2cbe7f6ffb3e5bbc4c7" +dependencies = [ + "get-size2", +] + +[[package]] +name = "rusqlite" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" +dependencies = [ + "bitflags 2.13.1", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.13.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "saphyr-parser" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7429158804c36e705d9423b7848018e535b3e92d439a763e364ee12670ef473b" +dependencies = [ + "arraydeque", + "thiserror 2.0.21", +] + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sec1" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" +dependencies = [ + "base16ct", + "ctutils", + "der 0.8.3", + "hybrid-array", + "subtle", + "zeroize", +] + +[[package]] +name = "secret-service" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4d35ad99a181be0a60ffcbe85d680d98f87bdc4d7644ade319b87076b9dbfd4" +dependencies = [ + "aes", + "cbc", + "futures-util", + "generic-array", + "hkdf", + "num", + "once_cell", + "rand 0.8.8", + "serde", + "sha2 0.10.9", + "zbus", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.13.1", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.1", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "indexmap 2.14.2", + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_plain" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9adc193c780ef8f159aee8b61e2d5801aaa555e6eb0947fe45530ec506296f" +dependencies = [ + "base64 0.23.1", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.2", + "jiff", + "schemars 0.9.0", + "schemars 1.2.3", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49" +dependencies = [ + "darling 0.24.1", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serdect" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" +dependencies = [ + "base16ct", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "signature" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" +dependencies = [ + "digest 0.11.3", + "rand_core 0.10.1", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" +dependencies = [ + "serde", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der 0.7.10", +] + +[[package]] +name = "spki" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef958a98b9d5da290cfc78946e9f3e61e1e62a18db0d92cac0b83cc161491a9" +dependencies = [ + "base64ct", + "der 0.8.3", +] + +[[package]] +name = "sqlbuild-analysis" +version = "0.187.3" +dependencies = [ + "memchr", + "polyglot-sql", + "polyglot-sql-function-catalogs", + "rayon", + "regex", + "rustc-hash", + "serde", + "serde_json", + "sha2 0.10.9", + "sqlbuild-attachments", + "sqlbuild-cache", + "sqlbuild-core", + "sqlbuild-discovery", + "sqlbuild-model-config", + "sqlbuild-render", + "sqlbuild-scopes", + "sqlbuild-sqltext", + "sqlclean", +] + +[[package]] +name = "sqlbuild-attachments" +version = "0.187.3" +dependencies = [ + "sqlbuild-core", + "sqlbuild-sqltext", +] + +[[package]] +name = "sqlbuild-build-features" +version = "0.187.3" +dependencies = [ + "arrayvec", + "bstr", + "either", + "errno", + "memchr", + "once_cell", + "regex-automata", + "syn 2.0.119", + "syn 3.0.3", + "thiserror 2.0.21", + "windows-sys 0.61.2", +] + +[[package]] +name = "sqlbuild-cache" +version = "0.187.3" +dependencies = [ + "blake3", + "rayon", + "serde_json", + "sha2 0.10.9", + "sqlbuild-core", + "tempfile", + "walkdir", +] + +[[package]] +name = "sqlbuild-cli" +version = "0.187.3" +dependencies = [ + "blake3", + "chrono", + "command-fds", + "ctrlc", + "getrandom 0.4.3", + "keyring", + "reqwest", + "rpassword", + "rusqlite", + "rustix", + "same-file", + "serde", + "serde_json", + "sqlbuild-build-features", + "sqlbuild-cache", + "sqlbuild-config", + "sqlbuild-core", + "sqlbuild-cursor", + "sqlbuild-model-config", + "sqlbuild-orchestration", + "sqlbuild-recall", + "sqlbuild-rules", + "tempfile", + "tokio", + "uuid", +] + +[[package]] +name = "sqlbuild-config" +version = "0.187.3" +dependencies = [ + "saphyr-parser", + "sqlbuild-core", + "tempfile", + "toml_edit", + "toml_parser", +] + +[[package]] +name = "sqlbuild-core" +version = "0.187.3" +dependencies = [ + "regex", + "serde_json", + "sha2 0.10.9", +] + +[[package]] +name = "sqlbuild-cursor" +version = "0.187.3" +dependencies = [ + "sqlbuild-core", +] + +[[package]] +name = "sqlbuild-discovery" +version = "0.187.3" +dependencies = [ + "junction", + "rayon", + "sqlbuild-cache", + "sqlbuild-config", + "sqlbuild-core", + "sqlbuild-model-config", + "sqlbuild-sqltext", + "tempfile", +] + +[[package]] +name = "sqlbuild-host" +version = "0.187.3" +dependencies = [ + "sqlbuild-core", +] + +[[package]] +name = "sqlbuild-model-config" +version = "0.187.3" +dependencies = [ + "regex", + "sqlbuild-core", +] + +[[package]] +name = "sqlbuild-orchestration" +version = "0.187.3" +dependencies = [ + "argon2", + "axum", + "axum-server", + "chrono", + "chrono-tz", + "croner", + "flate2", + "futures-util", + "getrandom 0.4.3", + "openidconnect", + "proptest", + "rcgen", + "reqwest", + "rusqlite", + "serde", + "serde_json", + "sqlbuild-core", + "sqlbuild-recall", + "sqlbuild-state", + "sqlx-core", + "sqlx-postgres", + "subtle", + "tempfile", + "thiserror 2.0.21", + "tokio", + "utoipa", + "uuid", +] + +[[package]] +name = "sqlbuild-project" +version = "0.187.3" +dependencies = [ + "sqlbuild-analysis", + "sqlbuild-core", + "sqlbuild-sqltext", +] + +[[package]] +name = "sqlbuild-pysource" +version = "0.187.3" +dependencies = [ + "ruff_python_ast", + "ruff_python_parser", + "ruff_text_size", + "sha2 0.10.9", + "sqlbuild-core", +] + +[[package]] +name = "sqlbuild-python" +version = "0.187.3" +dependencies = [ + "chrono", + "chrono-tz", + "fensu-policy", + "mimalloc", + "pyo3", + "rayon", + "rusqlite", + "rustc-hash", + "serde", + "serde_json", + "sha2 0.10.9", + "signal-hook", + "sqlbuild-analysis", + "sqlbuild-attachments", + "sqlbuild-cache", + "sqlbuild-cli", + "sqlbuild-config", + "sqlbuild-core", + "sqlbuild-cursor", + "sqlbuild-discovery", + "sqlbuild-host", + "sqlbuild-model-config", + "sqlbuild-orchestration", + "sqlbuild-project", + "sqlbuild-pysource", + "sqlbuild-recall", + "sqlbuild-refactor", + "sqlbuild-render", + "sqlbuild-rules", + "sqlbuild-scopes", + "sqlbuild-sqltext", + "sqlbuild-state", + "sqlclean", + "stacker", + "tokio", + "uuid", +] + +[[package]] +name = "sqlbuild-recall" +version = "0.187.3" +dependencies = [ + "blake3", + "chrono", + "rusqlite", + "rustix", + "serde", + "serde_json", + "sha2 0.10.9", + "sqlbuild-config", + "sqlbuild-core", + "tempfile", + "utoipa", +] + +[[package]] +name = "sqlbuild-refactor" +version = "0.187.3" +dependencies = [ + "serde", + "serde_json", + "sqlbuild-analysis", + "sqlbuild-config", + "sqlbuild-core", + "sqlbuild-model-config", + "sqlbuild-pysource", + "sqlbuild-sqltext", + "tempfile", +] + +[[package]] +name = "sqlbuild-render" +version = "0.187.3" +dependencies = [ + "serde_json", + "sqlbuild-cache", + "sqlbuild-core", + "sqlbuild-sqltext", + "tempfile", +] + +[[package]] +name = "sqlbuild-rules" +version = "0.187.3" dependencies = [ "blake3", + "fensu-policy", + "globset", + "polyglot-sql", + "rayon", + "regex", + "rustc-hash", + "serde", + "serde_json", + "sha2 0.10.9", + "sqlbuild-analysis", + "sqlbuild-attachments", + "sqlbuild-cache", + "sqlbuild-core", + "sqlbuild-scopes", + "sqlbuild-sqltext", + "sqlclean", + "stacker", + "tempfile", + "toml", +] + +[[package]] +name = "sqlbuild-scopes" +version = "0.187.3" +dependencies = [ + "serde_json", + "sha2 0.10.9", + "sqlbuild-config", + "sqlbuild-core", + "sqlbuild-discovery", + "sqlbuild-render", + "sqlbuild-sqltext", +] + +[[package]] +name = "sqlbuild-sqltext" +version = "0.187.3" +dependencies = [ "rayon", + "serde", + "sqlbuild-core", +] + +[[package]] +name = "sqlbuild-state" +version = "0.187.3" +dependencies = [ "serde_json", - "sha2", + "sha2 0.10.9", "sqlbuild-core", + "sqlbuild-cursor", +] + +[[package]] +name = "sqlclean" +version = "0.4.3" +source = "git+https://github.com/sqlstack-io/sqlclean.git?rev=cf79656d247277a30bc81ef39b6dee420e721221#cf79656d247277a30bc81ef39b6dee420e721221" +dependencies = [ + "polyglot-sql", + "polyglot-sql-function-catalogs", + "rayon", + "rustc-hash", + "serde", + "serde_json", + "sha2 0.10.9", +] + +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64 0.22.1", + "bytes", + "chrono", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.5", + "hashlink", + "indexmap 2.14.2", + "log", + "memchr", + "once_cell", + "percent-encoding", + "rustls", + "serde", + "serde_json", + "smallvec", + "thiserror 2.0.21", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", + "webpki-roots 0.26.11", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.13.1", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac 0.12.1", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand 0.8.8", + "serde", + "serde_json", + "sha2 0.10.9", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.21", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "stacker" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thin-vec" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a4b9ba8738cb4a4f399d37e266becfd475e75eb73425b87a05a2f2039ba63e" + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl 2.0.21", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap 2.14.2", + "serde_core", + "serde_spanned", + "toml_datetime 0.7.5+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 0.7.15", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.15+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +dependencies = [ + "indexmap 2.14.2", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "winnow 1.0.4", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow 1.0.4", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.13.1", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "uds_windows" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" +dependencies = [ + "memoffset", "tempfile", - "walkdir", + "windows-sys 0.61.2", ] [[package]] -name = "sqlbuild-cli" -version = "0.187.3" +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" dependencies = [ - "blake3", - "ctrlc", - "getrandom 0.4.3", - "rustix", + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode_names2" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" +dependencies = [ + "phf 0.11.3", + "unicode_names2_generator", +] + +[[package]] +name = "unicode_names2_generator" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e" +dependencies = [ + "getopts", + "log", + "phf_codegen", + "rand 0.8.8", +] + +[[package]] +name = "unindent" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", +] + +[[package]] +name = "utf8_iter" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af3bd9d3e571c26fcce9d033b87ea5ab061b8d2ffe9b7a5675d810d68f03edf0" +dependencies = [ + "utf_types", +] + +[[package]] +name = "utf_types" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3b425286980a367278453f3b6d28cdcb134b574e5a35824a6ff5eebad50bb6" + +[[package]] +name = "utoipa" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bde15df68e80b16c7d16b9616e80770ad158988daa56a27dccd1e55558b0160" +dependencies = [ + "indexmap 2.14.2", "serde", "serde_json", - "sqlbuild-build-features", - "sqlbuild-cache", - "sqlbuild-config", - "sqlbuild-core", - "sqlbuild-cursor", - "sqlbuild-model-config", - "sqlbuild-rules", - "tempfile", + "utoipa-gen", +] + +[[package]] +name = "utoipa-gen" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba0b99ee52df3028635d93840c797102da61f8a7bb3cf751032455895b52ef8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "uuid", +] + +[[package]] +name = "uuid" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "serde", + "sha1_smol", + "wasm-bindgen", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c" +dependencies = [ + "cfg-if", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e14e0327cb5368ad60ccb22d04a9827a693c95a926d8d125e9fa39e0baec501" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.3", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7d179596ff5ec0f6de4db64d54011bde6289cc49603d3a6e4ce3f329207ab88" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", ] [[package]] -name = "sqlbuild-config" -version = "0.187.3" +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "saphyr-parser", - "sqlbuild-core", - "tempfile", - "toml_edit", - "toml_parser", + "webpki-roots 1.0.11", ] [[package]] -name = "sqlbuild-core" -version = "0.187.3" +name = "webpki-roots" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171" dependencies = [ - "regex", - "sha2", + "rustls-pki-types", ] [[package]] -name = "sqlbuild-cursor" -version = "0.187.3" +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" dependencies = [ - "sqlbuild-core", + "libredox", + "wasite", ] [[package]] -name = "sqlbuild-discovery" -version = "0.187.3" +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" dependencies = [ - "junction", - "rayon", - "sqlbuild-cache", - "sqlbuild-config", - "sqlbuild-core", - "sqlbuild-model-config", - "sqlbuild-sqltext", - "tempfile", + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", ] [[package]] -name = "sqlbuild-host" -version = "0.187.3" +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "sqlbuild-core", + "windows-sys 0.61.2", ] [[package]] -name = "sqlbuild-model-config" -version = "0.187.3" +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ - "regex", - "sqlbuild-core", + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", ] [[package]] -name = "sqlbuild-project" -version = "0.187.3" +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ - "sqlbuild-analysis", - "sqlbuild-core", - "sqlbuild-sqltext", + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] -name = "sqlbuild-pysource" -version = "0.187.3" +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ - "ruff_python_ast", - "ruff_python_parser", - "ruff_text_size", - "sha2", - "sqlbuild-core", + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] -name = "sqlbuild-python" -version = "0.187.3" +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "fensu-policy", - "mimalloc", - "pyo3", - "rayon", - "rusqlite", - "rustc-hash", - "serde", - "serde_json", - "signal-hook", - "sqlbuild-analysis", - "sqlbuild-attachments", - "sqlbuild-cache", - "sqlbuild-cli", - "sqlbuild-config", - "sqlbuild-core", - "sqlbuild-cursor", - "sqlbuild-discovery", - "sqlbuild-host", - "sqlbuild-model-config", - "sqlbuild-project", - "sqlbuild-pysource", - "sqlbuild-recall", - "sqlbuild-refactor", - "sqlbuild-render", - "sqlbuild-rules", - "sqlbuild-scopes", - "sqlbuild-sqltext", - "sqlbuild-state", - "sqlclean", - "stacker", + "windows-link", ] [[package]] -name = "sqlbuild-recall" -version = "0.187.3" +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "blake3", - "rusqlite", - "rustix", - "serde_json", - "sha2", - "sqlbuild-config", - "sqlbuild-core", - "tempfile", + "windows-link", ] [[package]] -name = "sqlbuild-refactor" -version = "0.187.3" +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "serde", - "serde_json", - "sqlbuild-analysis", - "sqlbuild-config", - "sqlbuild-core", - "sqlbuild-model-config", - "sqlbuild-pysource", - "sqlbuild-sqltext", - "tempfile", + "windows-targets 0.48.5", ] [[package]] -name = "sqlbuild-render" -version = "0.187.3" +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "serde_json", - "sqlbuild-cache", - "sqlbuild-core", - "sqlbuild-sqltext", - "tempfile", + "windows-targets 0.52.6", ] [[package]] -name = "sqlbuild-rules" -version = "0.187.3" +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "blake3", - "fensu-policy", - "globset", - "polyglot-sql", - "rayon", - "regex", - "rustc-hash", - "serde", - "serde_json", - "sha2", - "sqlbuild-analysis", - "sqlbuild-attachments", - "sqlbuild-cache", - "sqlbuild-core", - "sqlbuild-scopes", - "sqlbuild-sqltext", - "sqlclean", - "stacker", - "tempfile", - "toml", + "windows-targets 0.52.6", ] [[package]] -name = "sqlbuild-scopes" -version = "0.187.3" +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "serde_json", - "sha2", - "sqlbuild-config", - "sqlbuild-core", - "sqlbuild-discovery", - "sqlbuild-render", - "sqlbuild-sqltext", + "windows-targets 0.53.5", ] [[package]] -name = "sqlbuild-sqltext" -version = "0.187.3" +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "rayon", - "serde", - "sqlbuild-core", + "windows-link", ] [[package]] -name = "sqlbuild-state" -version = "0.187.3" +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "serde_json", - "sha2", - "sqlbuild-core", - "sqlbuild-cursor", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] -name = "sqlclean" -version = "0.4.3" -source = "git+https://github.com/sqlstack-io/sqlclean.git?rev=cf79656d247277a30bc81ef39b6dee420e721221#cf79656d247277a30bc81ef39b6dee420e721221" +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "polyglot-sql", - "polyglot-sql-function-catalogs", - "rayon", - "rustc-hash", - "serde", - "serde_json", - "sha2", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] -name = "stacker" -version = "0.1.25" +name = "windows-targets" +version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ - "cc", - "cfg-if", - "libc", - "psm", - "windows-sys", + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] -name = "static_assertions" -version = "1.1.0" +name = "windows_aarch64_gnullvm" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] -name = "syn" -version = "2.0.119" +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] -name = "syn" -version = "3.0.3" +name = "windows_i686_msvc" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] -name = "target-lexicon" -version = "0.13.5" +name = "windows_i686_msvc" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" [[package]] -name = "tempfile" -version = "3.27.0" +name = "windows_x86_64_gnu" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" -dependencies = [ - "fastrand", - "getrandom 0.4.3", - "once_cell", - "rustix", - "windows-sys", -] +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] -name = "thin-vec" -version = "0.2.21" +name = "windows_x86_64_gnu" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6a4b9ba8738cb4a4f399d37e266becfd475e75eb73425b87a05a2f2039ba63e" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] -name = "thiserror" -version = "1.0.69" +name = "windows_x86_64_gnu" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" [[package]] -name = "thiserror" -version = "2.0.21" +name = "windows_x86_64_gnullvm" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" -dependencies = [ - "thiserror-impl 2.0.21", -] +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] -name = "thiserror-impl" -version = "1.0.69" +name = "windows_x86_64_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] -name = "thiserror-impl" -version = "2.0.21" +name = "windows_x86_64_gnullvm" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" [[package]] -name = "tinyvec" -version = "1.13.3" +name = "windows_x86_64_msvc" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] -name = "toml" -version = "0.9.12+spec-1.1.0" +name = "windows_x86_64_msvc" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" -dependencies = [ - "indexmap", - "serde_core", - "serde_spanned", - "toml_datetime 0.7.5+spec-1.1.0", - "toml_parser", - "toml_writer", - "winnow 0.7.15", -] +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "toml_datetime" -version = "0.7.5+spec-1.1.0" +name = "windows_x86_64_msvc" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" -dependencies = [ - "serde_core", -] +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] -name = "toml_datetime" -version = "1.1.1+spec-1.1.0" +name = "winnow" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" -dependencies = [ - "serde_core", -] +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" [[package]] -name = "toml_edit" -version = "0.25.15+spec-1.1.0" +name = "winnow" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" dependencies = [ - "indexmap", - "toml_datetime 1.1.1+spec-1.1.0", - "toml_parser", - "winnow 1.0.4", + "memchr", ] [[package]] -name = "toml_parser" -version = "1.1.3+spec-1.1.0" +name = "wit-bindgen" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" -dependencies = [ - "winnow 1.0.4", -] +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] -name = "toml_writer" -version = "1.1.2+spec-1.1.0" +name = "wnaf" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" +checksum = "795ca18b3fdb5e62bf982199278341ddcf7ebf7d32e25e212ad05d496e95f6fa" +dependencies = [ + "ff", + "group", + "hybrid-array", + "primefield", +] [[package]] -name = "typenum" -version = "1.20.1" +name = "writeable" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] -name = "unicode-ident" -version = "1.0.24" +name = "x509-parser" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "ring", + "rusticata-macros", + "thiserror 2.0.21", + "time", +] [[package]] -name = "unicode-normalization" -version = "0.1.25" +name = "xdg-home" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +checksum = "ec1cdab258fb55c0da61328dc52c8764709b249011b2cad0454c72f0bf10a1f6" dependencies = [ - "tinyvec", + "libc", + "windows-sys 0.59.0", ] [[package]] -name = "unicode-segmentation" -version = "1.13.3" +name = "yasna" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec 0.9.1", + "time", +] [[package]] -name = "unicode-width" -version = "0.2.2" +name = "yoke" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] [[package]] -name = "unicode_names2" -version = "1.3.0" +name = "yoke-derive" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ - "phf", - "unicode_names2_generator", + "proc-macro2", + "quote", + "syn 3.0.3", + "synstructure 0.14.0", ] [[package]] -name = "unicode_names2_generator" -version = "1.3.0" +name = "zbus" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e" +checksum = "bb97012beadd29e654708a0fdb4c84bc046f537aecfde2c3ee0a9e4b4d48c725" dependencies = [ - "getopts", - "log", - "phf_codegen", - "rand", + "async-broadcast", + "async-process", + "async-recursion", + "async-trait", + "enumflags2", + "event-listener", + "futures-core", + "futures-sink", + "futures-util", + "hex", + "nix 0.29.0", + "ordered-stream", + "rand 0.8.8", + "serde", + "serde_repr", + "sha1", + "static_assertions", + "tracing", + "uds_windows", + "windows-sys 0.52.0", + "xdg-home", + "zbus_macros", + "zbus_names", + "zvariant", ] [[package]] -name = "unindent" -version = "0.2.4" +name = "zbus_macros" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +checksum = "267db9407081e90bbfa46d841d3cbc60f59c0351838c4bc65199ecd79ab1983e" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", + "zvariant_utils", +] [[package]] -name = "vcpkg" -version = "0.2.15" +name = "zbus_names" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +checksum = "4b9b1fef7d021261cc16cba64c351d291b715febe0fa10dc3a443ac5a5022e6c" +dependencies = [ + "serde", + "static_assertions", + "zvariant", +] [[package]] -name = "version_check" -version = "0.9.5" +name = "zerocopy" +version = "0.8.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +checksum = "86502bf56ac7c77571a32e2647bb2a15894565e981fb2a48d7bde2d91c965a9d" +dependencies = [ + "zerocopy-derive", +] [[package]] -name = "walkdir" -version = "2.5.0" +name = "zerocopy-derive" +version = "0.8.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +checksum = "5457206954b06561e2608c7e19cf58b1926586d999c246eebe4502f7e2039d1a" dependencies = [ - "same-file", - "winapi-util", + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" +name = "zerofrom" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] [[package]] -name = "winapi" -version = "0.3.9" +name = "zerofrom-derive" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", + "proc-macro2", + "quote", + "syn 3.0.3", + "synstructure 0.14.0", ] [[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" +name = "zeroize" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" +dependencies = [ + "zeroize_derive", +] [[package]] -name = "winapi-util" -version = "0.1.11" +name = "zeroize_derive" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" dependencies = [ - "windows-sys", + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" +name = "zerotrie" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] [[package]] -name = "windows-link" -version = "0.2.1" +name = "zerovec" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] [[package]] -name = "windows-sys" -version = "0.61.2" +name = "zerovec-derive" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" dependencies = [ - "windows-link", + "proc-macro2", + "quote", + "syn 3.0.3", ] [[package]] -name = "winnow" -version = "0.7.15" +name = "zlib-rs" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" [[package]] -name = "winnow" -version = "1.0.4" +name = "zmij" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" -dependencies = [ - "memchr", -] +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" [[package]] -name = "zerocopy" -version = "0.8.62" +name = "zvariant" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86502bf56ac7c77571a32e2647bb2a15894565e981fb2a48d7bde2d91c965a9d" +checksum = "2084290ab9a1c471c38fc524945837734fbf124487e105daec2bb57fd48c81fe" dependencies = [ - "zerocopy-derive", + "endi", + "enumflags2", + "serde", + "static_assertions", + "zvariant_derive", ] [[package]] -name = "zerocopy-derive" -version = "0.8.62" +name = "zvariant_derive" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5457206954b06561e2608c7e19cf58b1926586d999c246eebe4502f7e2039d1a" +checksum = "73e2ba546bda683a90652bac4a279bc146adad1386f25379cf73200d2002c449" dependencies = [ + "proc-macro-crate", "proc-macro2", "quote", "syn 2.0.119", + "zvariant_utils", ] [[package]] -name = "zmij" -version = "1.0.23" +name = "zvariant_utils" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" +checksum = "c51bcff7cc3dbb5055396bcf774748c3dab426b4b8659046963523cee4808340" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] diff --git a/Cargo.toml b/Cargo.toml index 9bd219e053..dcf3591c45 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "crates/sqlbuild-config", "crates/sqlbuild-host", "crates/sqlbuild-recall", + "crates/sqlbuild-orchestration", "crates/sqlbuild-discovery", "crates/sqlbuild-scopes", "crates/sqlbuild-model-config", @@ -39,6 +40,7 @@ order = [ "sqlbuild-config", "sqlbuild-host", "sqlbuild-recall", + "sqlbuild-orchestration", "sqlbuild-model-config", "sqlbuild-discovery", "sqlbuild-render", @@ -62,16 +64,23 @@ publish = false rust-version = "1.88" [workspace.dependencies] +argon2 = "0.5.3" +axum = { version = "0.8", default-features = false, features = ["http1", "json", "query", "tokio"] } +axum-server = { version = "0.8", default-features = false, features = ["tls-rustls-no-provider"] } fensu-policy = "=0.22.0" +futures-util = { version = "0.3", default-features = false, features = ["std"] } # `sqb` names compute log captures with random ids, as `uuid.uuid4().hex`. getrandom = "0.4" globset = "0.4" # Windows only: tells NTFS junctions, which Python walks as directories, from symbolic links. junction = { version = "2.1.0", default-features = false } +keyring = { version = "3.6.3", default-features = false } # Python loads the extension with dlopen, so mimalloc must not use initial-exec TLS. mimalloc = { version = "0.1", default-features = false, features = ["local_dynamic_tls"] } proc-macro2 = { version = "1", features = ["span-locations"] } pyo3 = { version = "0.26", features = ["abi3-py312"] } +rpassword = "7.5.4" +rcgen = "0.14" polyglot-sql = { git = "https://github.com/sqlstack-io/polyglot.git", rev = "8767d688c5b504ad609645e27ab77a260d91c5a1", default-features = false, features = [ "ast-tools", "builder", @@ -90,12 +99,14 @@ bstr = "1" either = { version = "1", default-features = false, features = ["use_std"] } memchr = "2" once_cell = "1" +openidconnect = { version = "4.1", default-features = false, features = ["reqwest", "rustls-tls"] } regex-automata = { version = "0.4", default-features = false, features = ["dfa-search"] } syn-v2-macro-features = { package = "syn", version = "2", default-features = false, features = ["extra-traits", "fold", "visit-mut"] } syn-v3-macro-features = { package = "syn", version = "3", default-features = false, features = ["extra-traits", "full"] } thiserror = "2" # Windows only: `sqb` ignores Ctrl+C while the Python CLI it started handles it. ctrlc = "3" +command-fds = "0.3" quote = "1" rayon = "1" # Static reading of project Python source; the parser fensu uses. @@ -112,13 +123,27 @@ errno = "0.3" windows-sys = { version = "0.61", default-features = false, features = ["Win32_System_Diagnostics_Debug"] } # Recall's local store: SQLite compiled in, so every build has the same engine version. rusqlite = { version = "0.37", features = ["bundled", "column_decltype"] } -rustix = { version = "1.1.4", default-features = false, features = ["fs", "std", "termios"] } +rustix = { version = "1.1.4", default-features = false, features = ["fs", "process", "std", "termios"] } saphyr-parser = "0.1" regex = "1" +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } serde = { version = "1", features = ["derive"] } # Every crate relies on insertion-ordered JSON objects, which the extension has always built with. serde_json = { version = "1", features = ["preserve_order"] } sha2 = "0.10" +chrono = { version = "0.4", features = ["serde"] } +chrono-tz = "0.10" +croner = "=3.0.1" +flate2 = "1" +proptest = "1" +same-file = "1" +semver = "1" +# Pin SQLx's driver crates together: its umbrella also resolves an incompatible SQLite engine. +sqlx-core = { version = "=0.8.6", default-features = false, features = ["_rt-tokio", "_tls-rustls-ring-webpki", "chrono", "uuid", "json"] } +sqlx-postgres = { version = "=0.8.6", default-features = false, features = ["chrono", "uuid", "json"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync", "time", "signal", "process", "net"] } +uuid = { version = "=1.18.1", features = ["v5", "v7", "serde"] } +utoipa = { version = "5", features = ["chrono", "uuid"] } sqlbuild-analysis = { path = "crates/sqlbuild-analysis" } sqlbuild-attachments = { path = "crates/sqlbuild-attachments" } sqlbuild-build-features = { path = "crates/sqlbuild-build-features" } @@ -133,6 +158,7 @@ sqlbuild-render = { path = "crates/sqlbuild-render" } sqlbuild-refactor = { path = "crates/sqlbuild-refactor" } sqlbuild-rules = { path = "crates/sqlbuild-rules" } sqlbuild-model-config = { path = "crates/sqlbuild-model-config" } +sqlbuild-orchestration = { path = "crates/sqlbuild-orchestration" } sqlbuild-project = { path = "crates/sqlbuild-project" } sqlbuild-pysource = { path = "crates/sqlbuild-pysource" } sqlbuild-recall = { path = "crates/sqlbuild-recall" } @@ -143,6 +169,7 @@ sqlbuild-state = { path = "crates/sqlbuild-state" } sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", rev = "cf79656d247277a30bc81ef39b6dee420e721221" } stacker = "0.1" syn = { version = "2", features = ["full", "visit", "extra-traits"] } +subtle = "2" tempfile = "3" toml = "0.9" toml_edit = { version = "0.25", default-features = false, features = ["parse"] } diff --git a/Makefile b/Makefile index 1753e39f05..b795940c4c 100644 --- a/Makefile +++ b/Makefile @@ -37,9 +37,31 @@ dev-native: fast-native: uv run --no-sync maturin develop --profile fast --skip-install +COMPOSE_BUILD_PROFILE ?= release +COMPOSE_BUILD_NETWORK ?= default +COMPOSE_PYTHON_VERSION ?= 3.12 + +.PHONY: test-compose +test-compose: + uv run --no-sync maturin build --profile $(COMPOSE_BUILD_PROFILE) --compatibility linux --out dist + uv run --no-sync maturin build --manifest-path crates/sqlbuild-cli/Cargo.toml --profile $(COMPOSE_BUILD_PROFILE) --compatibility linux --out dist + $(MAKE) test-compose-packaged + +.PHONY: test-compose-packaged +test-compose-packaged: + docker build --network $(COMPOSE_BUILD_NETWORK) --file deploy/Dockerfile --target server --build-arg PYTHON_VERSION=$(COMPOSE_PYTHON_VERSION) --tag sqlbuild-server:test . + docker build --network $(COMPOSE_BUILD_NETWORK) --file deploy/Dockerfile --target location --build-arg PYTHON_VERSION=$(COMPOSE_PYTHON_VERSION) --build-arg SQLBUILD_GIT_REVISION=$$(git rev-parse HEAD) --tag sqlbuild-location:test . + SQLBUILD_REQUIRE_COMPOSE_TESTS=1 uv run --no-sync pytest tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py -n auto --dist loadfile -vv + rust-check: rust-lint uv run fensu check - cargo test --workspace --all-features + $(MAKE) test-native + +NATIVE_TEST_ARGS ?= --workspace --all-features + +.PHONY: test-native +test-native: + uv run --no-sync python -m scripts.run_native_tests $(NATIVE_TEST_ARGS) .PHONY: rust-lint rust-lint: diff --git a/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json b/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json new file mode 100644 index 0000000000..8116485af0 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json @@ -0,0 +1,32 @@ +{ + "first_person": "orders-admin", + "second_person": "customers-admin", + "password": "orders-bootstrap-password-2026", + "disabled_person_sequence": ["assign", "disable", "revoke", "enable", "deny_run"], + "run_control_sequence": ["request", "cancel", "confirm_stopped", "rerun_frozen", "retry", "revoke", "deny_retry"], + "connection_test_roles": [ + {"role": null, "allowed": false}, + {"role": "viewer", "allowed": false}, + {"role": "operator", "allowed": false}, + {"role": "admin", "allowed": true}, + {"role": "developer", "allowed": true} + ], + "session_roles": [ + {"role": null, "expected_metadata": 403, "expected_connection": 403, "expected_job": 403, "expected_sql": 403}, + {"role": "viewer", "expected_metadata": 200, "expected_connection": 403, "expected_job": 403, "expected_sql": 403}, + {"role": "operator", "expected_metadata": 200, "expected_connection": 403, "expected_job": 200, "expected_sql": 403}, + {"role": "developer", "expected_metadata": 200, "expected_connection": 200, "expected_job": 200, "expected_sql": 200}, + {"role": "admin", "expected_metadata": 200, "expected_connection": 200, "expected_job": 200, "expected_sql": 200} + ], + "job_settings": [ + {"full_refresh": false}, + {"full_refresh": true}, + {"start_cursor_int": "3", "end_cursor_int": "8"}, + {"start_cursor": "3"}, + {"end_cursor": "8"}, + {"start_cursor_ts": "2026-01-03T00:00:00Z"}, + {"end_cursor_ts": "2026-01-08T00:00:00Z"}, + {"start_cursor_int": "3"}, + {"end_cursor_int": "8"} + ] +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json b/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json new file mode 100644 index 0000000000..f111fc629a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json @@ -0,0 +1,8 @@ +{ + "passed": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Order totals are valid.')\ndef orders_valid(ctx):\n return ctx.pass_()\n", + "warned": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Order totals are valid.')\ndef orders_valid(ctx):\n return ctx.warn(message='Review order totals.')\n", + "failed": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Order totals are valid.')\ndef orders_valid(ctx):\n return ctx.fail(message='Order totals are invalid.')\n", + "cancelled": "import signal\nfrom sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Heartbeat cancellation interrupts the driver without SIGTERM.')\ndef orders_valid(ctx):\n signal.signal(signal.SIGTERM, signal.SIG_IGN)\n ctx.query('WITH RECURSIVE pending(i) AS ((SELECT i FROM range(10000000000) AS seed(i) LIMIT 1) UNION ALL SELECT i FROM pending) SELECT sum(i) FROM pending')\n return ctx.pass_()\n", + "on_failure": "from sqlbuild.orchestration import NodeEventKind, job, node_event, schedule, sensor\nannual_orders = job('annual_orders', select='orders', description='Build orders.', schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\nrefresh_inventory = job('refresh_inventory', select='inventory', description='Refresh inventory.')\nfailed_orders = sensor('failed_orders', description='React to an actual orders check failure.', when=node_event(select='check:orders_valid', kinds=[NodeEventKind.FAILED]), run=refresh_inventory, minimum_interval_seconds=1)\n", + "inventory": "MODEL (description 'Inventory after check failure.', materialized table);\nSELECT 59 AS amount;\n" +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/audit_result_history.json b/benchmarks/duckshop/edge_cases/orchestration/audit_result_history.json new file mode 100644 index 0000000000..5fef8086b6 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/audit_result_history.json @@ -0,0 +1,60 @@ +{ + "record": { + "state_database": null, + "state_schema": "main", + "result_id": "result-000", + "schema_version": 1, + "occurred_at": "2026-02-03T04:05:06Z", + "invocation_id": "invocation-1", + "run_id": "", + "audit_name": "valid_orders", + "audit_definition_name": "dq_column_rate", + "audit_description": "Valid order percentage", + "binding_key": "orders.valid_orders", + "definition_fingerprint": "definition-fingerprint", + "execution_fingerprint": "execution-fingerprint", + "evaluation_mode": "measurement", + "run_scope_phase": "final", + "attachment_kind": "factory", + "attached_target_kind": "model", + "attached_target_name": "orders", + "attached_column_name": "order_id", + "target_database": null, + "target_schema": "main", + "target_name": "orders_physical", + "severity": "warn", + "outcome": "pass", + "execution_error": null, + "violation_count": null, + "measured_value": "99.5", + "sample_count": 200, + "sample_unit": "rows", + "minimum_samples": 100, + "thresholds_json": "{\"warn\":{\"below\":99}}", + "evidence_json": "[{\"order_id\":\"o'rders 🧾\"}]", + "evidence_count": 1, + "evidence_truncated": "true", + "evidence_error": null, + "measurement_sql": "SELECT 99.5 AS valid_rate", + "evidence_sql": null, + "executed_sql": "SELECT 99.5 AS valid_rate", + "sql_digest": "digest", + "metadata_json": "{\"owner\":\"data\"}", + "reused": "false", + "audit_meta_json": "{\"owner\":\"orders\"}" + }, + "source_rows": 104, + "duplicate_rows": 105, + "later_time": "2026-02-03T04:05:06.000001Z", + "retained_result": "result-103", + "kind": "audit_results", + "source_key": "result_id", + "id_prefix": "result", + "page_lengths": [100, 100, 4], + "later_index": 100, + "later_record": {"occurred_at": "2026-02-03T04:05:06.000001Z"}, + "last_record": {"result_id": "result-103"}, + "record_overrides": {}, + "source_count": 210, + "model": "MODEL (description 'Orders.', materialized table); SELECT 1 AS order_id" +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/automatic_warehouse_history.json b/benchmarks/duckshop/edge_cases/orchestration/automatic_warehouse_history.json new file mode 100644 index 0000000000..65c17a09a6 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/automatic_warehouse_history.json @@ -0,0 +1,6 @@ +{ + "model": "MODEL (description 'Orders.', materialized table); SELECT 17 AS amount\n", + "wide_fingerprint_count": 104, + "wide_definition_bytes": 700000, + "lifecycle": ["initial import", "empty mirror resync", "warehouse retention reconciliation"] +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/backfill_integer.sql b/benchmarks/duckshop/edge_cases/orchestration/backfill_integer.sql new file mode 100644 index 0000000000..2d882a92ae --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/backfill_integer.sql @@ -0,0 +1,16 @@ +MODEL ( + description 'Backfill orders over an explicit integer window.', + materialized incremental, + incremental_strategy delete_insert, + incremental_mode microbatch, + microbatch_strategy watermark, + cursor batch_id, + cursor_type integer, + cursor_start 0, + cursor_watermark_mode all, + cursor_inputs (raw_orders (column batch_id, roles [filter, watermark])), + batch_size '2', + batch_concurrency 1, +); +SELECT amount, batch_id FROM __source("raw_orders") +WHERE batch_id >= __cursor_start() AND batch_id < __cursor_end() diff --git a/benchmarks/duckshop/edge_cases/orchestration/backfill_timestamp.sql b/benchmarks/duckshop/edge_cases/orchestration/backfill_timestamp.sql new file mode 100644 index 0000000000..fa8b2bcd25 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/backfill_timestamp.sql @@ -0,0 +1,17 @@ +MODEL ( + description 'Backfill orders over an explicit timestamp window.', + materialized incremental, + incremental_strategy delete_insert, + incremental_mode microbatch, + microbatch_strategy watermark, + cursor ordered_at, + cursor_type timestamp, + cursor_grain day, + cursor_start '2026-07-01', + cursor_watermark_mode all, + cursor_inputs (raw_orders (column ordered_at, roles [filter, watermark])), + batch_size 1d, + batch_concurrency 1, +); +SELECT amount, ordered_at FROM __source("raw_orders") +WHERE ordered_at >= __cursor_start() AND ordered_at < __cursor_end() diff --git a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py new file mode 100644 index 0000000000..644b92c5d6 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py @@ -0,0 +1,15 @@ +"""Provider-backed fulfillment times evaluated in the code location, never by the scheduler.""" + +from sqlbuild.orchestration import calendar_schedule, job + +orders = job("orders", description="Build order reporting.", select="orders") + + +@calendar_schedule( + job=orders, + description="Refresh orders at upcoming fulfillment times.", + horizon="8s", + timezone="America/New_York", +) +def fulfillment_calendar(window, fulfillment): + return fulfillment.upcoming(window) diff --git a/benchmarks/duckshop/edge_cases/orchestration/catalog_connection.toml b/benchmarks/duckshop/edge_cases/orchestration/catalog_connection.toml new file mode 100644 index 0000000000..c610cac914 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/catalog_connection.toml @@ -0,0 +1,18 @@ +name = "orders" +adapter = "sqlite" +default_target = "development" + +[connection] +path = "must-not-open.db" +password = "${ENV:SQLBUILD_MISSING_WRITER_PASSWORD}" + +[connections.writer] +path = "must-not-open-writer.db" + +[connections.catalog] +path = "${ENV:SQLBUILD_CATALOG_DATABASE}" + +[targets.development] +connection = "writer" +catalog_connection = "catalog" +schema = "main" diff --git a/benchmarks/duckshop/edge_cases/orchestration/check_transitions.json b/benchmarks/duckshop/edge_cases/orchestration/check_transitions.json new file mode 100644 index 0000000000..98838e4419 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/check_transitions.json @@ -0,0 +1,13 @@ +[ + {"nodes": ["success", "failed"], "started": [true, true], "outcome": "partial"}, + {"nodes": ["failed", "failed"], "started": [true, true], "outcome": "failed"}, + {"nodes": ["success", "cancelled"], "started": [true, true], "outcome": "partial"}, + {"nodes": ["success", "skipped"], "started": [true, false], "outcome": "partial"}, + {"nodes": ["success", "warn"], "started": [true, true], "outcome": "warn"}, + {"nodes": ["success", "failed"], "started": [true, true], "outcome": "partial"}, + {"nodes": ["success", "success"], "started": [true, false], "outcome": "success"}, + {"nodes": ["success", "failed"], "started": [true, true], "outcome": "partial"}, + {"nodes": ["success", "success"], "started": [true, true], "outcome": "success"}, + {"nodes": ["success", "success"], "started": [true, true], "outcome": "success"}, + {"nodes": ["success", "failed"], "started": [true, true], "outcome": "partial"} +] diff --git a/benchmarks/duckshop/edge_cases/orchestration/cli_login.json b/benchmarks/duckshop/edge_cases/orchestration/cli_login.json new file mode 100644 index 0000000000..76b1e580fc --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/cli_login.json @@ -0,0 +1,6 @@ +{ + "origin": "https://control.example.com", + "password": "orders-browser-password", + "replacement": "orders-replacement-password", + "client_secret": "synthetic-cli-exchange-capability-not-a-real-credential" +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py b/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py new file mode 100644 index 0000000000..ae5a4c9dec --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py @@ -0,0 +1,10 @@ +"""Negative fixture: mutually exclusive build flags must fail before publication.""" + +from sqlbuild.orchestration import job + +orders = job( + "orders", + description="Refresh orders.", + select="orders", + settings={"load_sources": False, "reload": True}, +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/container_deployment.json b/benchmarks/duckshop/edge_cases/orchestration/container_deployment.json new file mode 100644 index 0000000000..d1f1a8783e --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/container_deployment.json @@ -0,0 +1,5 @@ +{ + "first": "SELECT 37 AS order_id, sum(sin(value)) AS total FROM range(2000000000) t(value)", + "second": "SELECT 41 AS order_id, 0.0 AS total", + "broken": "SELECT FROM" +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/custom_orders.py b/benchmarks/duckshop/edge_cases/orchestration/custom_orders.py new file mode 100644 index 0000000000..f448f2b85f --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/custom_orders.py @@ -0,0 +1,15 @@ +"""Polling callbacks return durable run keys and keep their cursor in the native transaction.""" + +from sqlbuild.orchestration import job, sensor + +orders = job("annual_orders", description="Build orders after source inspection.", select="orders") + + +@sensor( + "poll_orders", description="Poll fulfillment readiness.", run=orders, minimum_interval_seconds=1 +) +def poll_orders(ctx, fulfillment): + def poll(): + return fulfillment.poll(ctx) + + return poll() diff --git a/benchmarks/duckshop/edge_cases/orchestration/custom_role_publication.json b/benchmarks/duckshop/edge_cases/orchestration/custom_role_publication.json new file mode 100644 index 0000000000..482290cc7d --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/custom_role_publication.json @@ -0,0 +1,8 @@ +{ + "description": "A support role is published, widened in a different action order, narrowed, then removed. Every publication takes effect at once.", + "publications": [ + {"actions": ["view"], "expected": {"view": "orders_support", "job.run": null}}, + {"actions": ["job.run", "view"], "expected": {"view": "orders_support", "job.run": "orders_support"}}, + {"actions": ["job.run"], "expected": {"view": null, "job.run": "orders_support"}} + ] +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/development_orders.py b/benchmarks/duckshop/edge_cases/orchestration/development_orders.py new file mode 100644 index 0000000000..7306a2147c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/development_orders.py @@ -0,0 +1,8 @@ +"""Development defaults pause code-enabled triggers without changing their definitions.""" + +from sqlbuild.orchestration import job, schedule + +annual = schedule(cron="0 0 1 1 *", timezone="UTC", starts_at="2099-01-01T00:00:00Z") +annual_orders = job( + "annual_orders", description="Build annual orders.", select="+orders", schedule=annual +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/failed_import.py b/benchmarks/duckshop/edge_cases/orchestration/failed_import.py new file mode 100644 index 0000000000..6483f0b2ab --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/failed_import.py @@ -0,0 +1,3 @@ +"""Negative fixture: a failed declaration import must fail every publication retry.""" + +raise RuntimeError("broken declaration") diff --git a/benchmarks/duckshop/edge_cases/orchestration/fingerprint_definition_variants.json b/benchmarks/duckshop/edge_cases/orchestration/fingerprint_definition_variants.json new file mode 100644 index 0000000000..afcc0383c4 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/fingerprint_definition_variants.json @@ -0,0 +1,20 @@ +{ + "semantic_hash": "fd5b7a46925b590b562e8b32bc94ee204b05867e9c9c266cf5b734f1586ae6e0", + "definitions": [ + { + "sql": "select order_id from orders", + "text_hash": "76d5ce528df9a090f025926fca724f92ac23e1021941e5061d211fcb5ded1e77" + }, + { + "sql": "SELECT order_id FROM orders -- retained source comment\n", + "text_hash": "1daa5068b226bbc6b50d082920762afc31df73b0c02e9d7e9f05fe6b7297d205" + } + ], + "review_marker": { + "node_type": "review_schema", + "definition_hash": "", + "version_hash": "", + "schema_fingerprint": "", + "definition": "" + } +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/fingerprint_lookup.json b/benchmarks/duckshop/edge_cases/orchestration/fingerprint_lookup.json new file mode 100644 index 0000000000..4849efbf6c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/fingerprint_lookup.json @@ -0,0 +1,21 @@ +{ + "record": { + "event_id": "orders-build-latest", + "state_database": null, + "state_schema": "build_history", + "node_type": "model", + "node_name": "orders", + "target_database": null, + "target_schema": "main", + "target_name": "orders", + "run_id": "orders-run-latest", + "definition_hash": "orders-definition-latest", + "version_hash": "orders-version-latest", + "schema_fingerprint": "orders-schema-latest", + "definition": "SELECT order_id FROM orders", + "metadata_json": "{\"private_note\":\"not part of the person response\"}", + "ts": "2026-01-03T10:00:00Z" + }, + "older_ts": "2026-01-01T10:00:00Z", + "distractor_ts": "2026-01-04T10:00:00Z" +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/first_tick_precision.json b/benchmarks/duckshop/edge_cases/orchestration/first_tick_precision.json new file mode 100644 index 0000000000..ce99f37798 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/first_tick_precision.json @@ -0,0 +1,7 @@ +{ + "description": "A minutely cron trigger whose recorded start has a sub-second fraction first falls due on the next whole minute, never at its start's own whole minute. A cursor left just past the latest occurrence by an earlier build resumes without ticking that occurrence again.", + "expression": "* * * * *", + "start_minutes_ahead": 1, + "start_fraction_microseconds": 123456, + "legacy_cursor_fraction_microseconds": 999999 +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/group_shared_limit.json b/benchmarks/duckshop/edge_cases/orchestration/group_shared_limit.json new file mode 100644 index 0000000000..ce32db3db1 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/group_shared_limit.json @@ -0,0 +1,10 @@ +{ + "description": "A daily group whose two load jobs share a warehouse limit of one. The group holds the warehouse slot and its children use it one at a time.", + "jobs": { + "load_customers": {"description": "Load customers", "selection": ["customers"], "concurrency_groups": ["warehouse"]}, + "load_inventory": {"description": "Load inventory", "selection": ["inventory"], "concurrency_groups": ["warehouse"]} + }, + "groups": { + "daily": {"description": "Daily loads", "selection": ["customers", "inventory"], "jobs": {"load_customers": [], "load_inventory": []}} + } +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json b/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json new file mode 100644 index 0000000000..5be86e4798 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json @@ -0,0 +1,22 @@ +{ + "initial": [{"run_id": "watermark-run", "observed_at": "2026-02-01 12:00:00"}], + "stages": [ + {"arrivals": [], "overlap_count": 0, "completed_cycles": 1, "mirror_sql": "SELECT 1", "health_records_per_table": 2, "expected_runs": ["legacy-run", "watermark-run"]}, + {"arrivals": [ + {"run_id": "a-future", "observed_at": "2026-02-01 12:01:00"}, + {"run_id": "within-overlap", "observed_at": "2026-02-01 11:58:00"}, + {"run_id": "zz-boundary", "observed_at": "2026-02-01 11:55:00"}, + {"run_id": "before-overlap", "observed_at": "2026-02-01 11:54:00"} + ], "overlap_count": 104, "completed_cycles": 2, "mirror_sql": "SELECT 1", "health_records_per_table": 108, "expected_runs": ["legacy-run", "watermark-run", "within-overlap", "zz-boundary", "a-future"]}, + {"arrivals": [], "overlap_count": 0, "completed_cycles": 3, "mirror_sql": "DELETE FROM {table}", "health_records_per_table": 110, "expected_overlap_count": 104, "expected_runs": ["legacy-run", "watermark-run", "within-overlap", "zz-boundary", "a-future", "before-overlap"]} + ], + "health": {"failure": null, "tables": ["fingerprints", "source_freshness"]}, + "availability": [ + {"available": false, "source": "warehouse.db", "destination": "../retained-warehouse.db", "failure": "read", "tables": ["fingerprints"]}, + {"available": true, "source": "../retained-warehouse.db", "destination": "warehouse.db", "failure": null, "tables": ["fingerprints", "source_freshness"]} + ], + "insert_sql": [ + "INSERT INTO _sqlbuild_fingerprints SELECT node_type,node_name,target_database,target_schema,target_name,'{run_id}',definition_hash,version_hash,schema_fingerprint,definition_b64,metadata_json_b64,'{observed_at}' FROM _sqlbuild_fingerprints WHERE run_id='legacy-run'", + "INSERT INTO _sqlbuild_source_freshness SELECT source_name,target_database,target_schema,target_name,'{run_id}',strategy,value_kind,data_version,data_version_hash,'{observed_at}' FROM _sqlbuild_source_freshness WHERE run_id='legacy-run'" + ] +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py b/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py new file mode 100644 index 0000000000..248b1b4ce1 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py @@ -0,0 +1,10 @@ +"""Negative fixture: job settings must not override the frozen selection.""" + +from sqlbuild.orchestration import job + +orders = job( + "orders", + description="Refresh orders.", + select="orders", + settings={"select": "customers"}, +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/janitor_event_history.json b/benchmarks/duckshop/edge_cases/orchestration/janitor_event_history.json new file mode 100644 index 0000000000..b3fed6df80 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/janitor_event_history.json @@ -0,0 +1,33 @@ +{ + "record": { + "state_database": null, + "state_schema": "main", + "event_id": "event-000", + "schema_version": 1, + "event_type": "archive", + "occurred_at": "2026-02-03T04:05:06Z", + "run_id": "cleanup-run", + "relation_database": null, + "relation_schema": "main", + "relation_type": "BASE TABLE", + "original_name": "orders", + "original_qualified_name": "main.orders", + "archive_name": "_sqb_archive_orders_🧾", + "archive_qualified_name": "main.\"_sqb_archive_orders_🧾\"", + "archived_at": "2026-01-02T03:04:05Z" + }, + "source_rows": 104, + "duplicate_rows": 105, + "later_time": "2026-02-03T04:05:06.000001Z", + "retained_result": "event-103", + "kind": "janitor_events", + "source_key": "event_id", + "id_prefix": "event", + "page_lengths": [100, 4], + "later_index": 103, + "later_record": {"event_id": "event-103", "event_type": "delete", "occurred_at": "2026-02-03T04:05:06.000001Z"}, + "last_record": {"event_id": "event-103", "event_type": "delete", "occurred_at": "2026-02-03T04:05:06.000001Z"}, + "record_overrides": {"event-103": {"event_type": "delete", "occurred_at": "2026-02-03T04:05:06.000001Z"}}, + "source_count": 104, + "model": "MODEL (description 'Orders.', materialized table); SELECT 1 AS order_id" +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_fingerprint_offsets.sql b/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_fingerprint_offsets.sql new file mode 100644 index 0000000000..fe7ba68f98 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_fingerprint_offsets.sql @@ -0,0 +1,10 @@ +-- SQLite predecessor text can carry the writer's offset, across the UTC date boundary. +-- Load after legacy_warehouse_state.sql, before any writer adds event_id. +INSERT INTO main._sqlbuild_fingerprints ( + node_type, node_name, target_database, target_schema, target_name, run_id, + definition_hash, version_hash, schema_fingerprint, definition_b64, metadata_json_b64, ts +) VALUES ( + 'model', 'offset_orders', NULL, 'main', 'offset_orders', 'offset-run', + 'legacy-definition', 'legacy-version', 'legacy-schema', + 'U0VMRUNUIDEgQVMgb3JkZXJfaWQ=', 'e30=', '2025-12-31T23:00:00-11:00' +); diff --git a/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql b/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql new file mode 100644 index 0000000000..b135ecff51 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql @@ -0,0 +1,22 @@ +-- Populated predecessor schemas for additive event-ID and read-only history-sync checks. +CREATE TABLE main._sqlbuild_fingerprints ( + node_type VARCHAR NOT NULL, node_name VARCHAR NOT NULL, + target_database VARCHAR, target_schema VARCHAR, target_name VARCHAR, + run_id VARCHAR NOT NULL, definition_hash VARCHAR NOT NULL, version_hash VARCHAR NOT NULL, + schema_fingerprint VARCHAR NOT NULL, definition_b64 VARCHAR NOT NULL, + metadata_json_b64 VARCHAR NOT NULL, ts TIMESTAMP NOT NULL +); +INSERT INTO main._sqlbuild_fingerprints VALUES ( + 'model', 'orders', NULL, 'main', 'orders', 'legacy-run', 'legacy-definition', + 'legacy-version', 'legacy-schema', 'U0VMRUNUIDEgQVMgb3JkZXJfaWQ=', 'e30=', '2026-01-01 10:00:00' +); +CREATE TABLE main._sqlbuild_source_freshness ( + source_name VARCHAR NOT NULL, target_database VARCHAR, target_schema VARCHAR, + target_name VARCHAR, run_id VARCHAR NOT NULL, strategy VARCHAR NOT NULL, + value_kind VARCHAR NOT NULL, data_version VARCHAR, data_version_hash VARCHAR NOT NULL, + observed_at TIMESTAMP NOT NULL +); +INSERT INTO main._sqlbuild_source_freshness VALUES ( + 'raw.orders', NULL, 'main', 'orders', 'legacy-run', 'max_column', 'integer', + '7', 'legacy-value', '2026-01-01 10:00:00' +); diff --git a/benchmarks/duckshop/edge_cases/orchestration/local_catalog_contention.json b/benchmarks/duckshop/edge_cases/orchestration/local_catalog_contention.json new file mode 100644 index 0000000000..d14975603b --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/local_catalog_contention.json @@ -0,0 +1,6 @@ +{ + "setup_sql": "CREATE TABLE orders AS SELECT 23 AS amount", + "query": "SELECT amount FROM orders", + "release_after_seconds": 0.5, + "persistent_release_after_seconds": 6.0 +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py b/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py new file mode 100644 index 0000000000..6724f992aa --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py @@ -0,0 +1,11 @@ +"""Keep a populated concurrency slot until recovered terminal publication commits.""" + +from sqlbuild.orchestration import job, schedule + +annual_orders = job( + "annual_orders", + description="Build orders under a shared concurrency limit.", + select="orders", + concurrency_groups=("orders_limit",), + schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/local_recovery_publication_fault.sql b/benchmarks/duckshop/edge_cases/orchestration/local_recovery_publication_fault.sql new file mode 100644 index 0000000000..a7ddaa159f --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/local_recovery_publication_fault.sql @@ -0,0 +1,5 @@ +-- Crash successive supervisors while a stopped child's terminal publication is rejected. +-- Claims and the inherited ownership fence must survive until the terminal fact commits. +CREATE TRIGGER reject_terminal BEFORE UPDATE OF status ON sqb_runs +WHEN NEW.status = 'failed' +BEGIN SELECT RAISE(ABORT, 'synthetic terminal publication failure'); END; diff --git a/benchmarks/duckshop/edge_cases/orchestration/local_worker_recovery.sql b/benchmarks/duckshop/edge_cases/orchestration/local_worker_recovery.sql new file mode 100644 index 0000000000..fe752e60d2 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/local_worker_recovery.sql @@ -0,0 +1,3 @@ +-- A long-running statement for scheduler crash, adoption and cancellation checks. +-- The supervising test cancels this query; it is not part of benchmark measurements. +SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value) diff --git a/benchmarks/duckshop/edge_cases/orchestration/managed_orders.yml b/benchmarks/duckshop/edge_cases/orchestration/managed_orders.yml new file mode 100644 index 0000000000..4ea1d73bb7 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/managed_orders.yml @@ -0,0 +1,9 @@ +sources: + - name: raw_orders + description: Orders inheriting the managed loader namespace. + managed: true + write_strategy: table + table: raw_orders + columns: + - name: order_id + type: BIGINT diff --git a/benchmarks/duckshop/edge_cases/orchestration/microbatch_history.json b/benchmarks/duckshop/edge_cases/orchestration/microbatch_history.json new file mode 100644 index 0000000000..9f3464975e --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/microbatch_history.json @@ -0,0 +1,109 @@ +{ + "record": { + "state_database": null, + "state_schema": "main", + "event_id": "batch-000", + "record_type": "partition_completion", + "scope_kind": "direct_logical", + "scope_key": "orders:main.orders_🧾", + "model_name": "orders", + "target_database": null, + "target_schema": "main", + "target_name": "orders_🧾", + "physical_generation_id": "orders-generation-1", + "origin_run_id": "external-origin", + "origin_run_started_at": "2026-02-02T03:04:05Z", + "execution_run_id": "", + "execution_run_started_at": null, + "run_type": "normal", + "completion_type": "initial", + "run_start": "18446744073709551616", + "run_end": "18446744073709551627", + "partition_start": "18446744073709551616", + "partition_end": "18446744073709551619", + "batch_size": "3", + "cursor_column": "order_id", + "cursor_type": "integer", + "cursor_grain": null, + "model_version_hash": "orders-version", + "definition_hash": "orders-definition", + "fingerprint_status": "known", + "replay_requirement_id": null, + "required_model_version_hash": null, + "previous_model_version_hash": null, + "replay_policy": null, + "rows_affected": 7, + "completed_at": "2026-02-03T03:04:05Z", + "coverage_source": null, + "observed_row_count": null, + "observed_at": "2026-02-03T05:06:07Z", + "synthetic_reason": null, + "unaccounted_policy": null, + "created_at": "2026-02-03T04:05:06Z" + }, + "source_rows": 104, + "duplicate_rows": 3, + "later_time": "2026-02-03T04:05:06.000001Z", + "retained_result": "batch-103", + "kind": "microbatches", + "source_key": "event_id", + "timestamp_key": "created_at", + "run_key": "execution_run_id", + "id_prefix": "batch", + "page_lengths": [100, 4], + "later_index": 103, + "later_record": { + "event_id": "batch-103", + "record_type": "replay_requirement", + "created_at": "2026-02-03T04:05:06.000001Z", + "completion_type": null, + "partition_start": null, + "partition_end": null, + "replay_requirement_id": "requirement-🧾", + "required_model_version_hash": "orders-next-version", + "previous_model_version_hash": "orders-version", + "replay_policy": "all", + "completed_at": null, + "rows_affected": null + }, + "last_record": { + "event_id": "batch-103", + "record_type": "replay_requirement", + "created_at": "2026-02-03T04:05:06.000001Z", + "completion_type": null, + "partition_start": null, + "partition_end": null, + "replay_requirement_id": "requirement-🧾", + "required_model_version_hash": "orders-next-version", + "previous_model_version_hash": "orders-version", + "replay_policy": "all", + "completed_at": null, + "rows_affected": null + }, + "record_overrides": { + "batch-102": { + "record_type": "synthetic_completion", + "fingerprint_status": "unknown", + "model_version_hash": null, + "definition_hash": null, + "observed_row_count": 19, + "synthetic_reason": "Retained 'orders' 🧾", + "unaccounted_policy": "synthesize" + }, + "batch-103": { + "record_type": "replay_requirement", + "created_at": "2026-02-03T04:05:06.000001Z", + "completion_type": null, + "partition_start": null, + "partition_end": null, + "replay_requirement_id": "requirement-🧾", + "required_model_version_hash": "orders-next-version", + "previous_model_version_hash": "orders-version", + "replay_policy": "all", + "completed_at": null, + "rows_affected": null + } + }, + "source_count": 104, + "model": "MODEL (description 'Orders.', materialized table); SELECT 1 AS order_id" +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/migration_history.json b/benchmarks/duckshop/edge_cases/orchestration/migration_history.json new file mode 100644 index 0000000000..e51a92592c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/migration_history.json @@ -0,0 +1,37 @@ +{ + "record": { + "state_database": null, + "state_schema": "main", + "event_id": "move-000", + "target_name": null, + "origin_model": null, + "origin_database": null, + "origin_schema": "legacy", + "origin_name": "orders_🧾", + "destination_model": "orders", + "destination_database": null, + "destination_schema": "main", + "destination_name": "orders_current", + "origin_version_hash": "orders-version", + "discovery": "manual", + "decision": "migrate", + "run_id": "external-move", + "created_at": "2026-02-03T04:05:06Z" + }, + "source_rows": 104, + "duplicate_rows": 3, + "later_time": "2026-02-03T04:05:06.000001Z", + "retained_result": "move-103", + "kind": "migrations", + "source_key": "event_id", + "timestamp_key": "created_at", + "legacy_columns": ["target_name", "origin_model"], + "id_prefix": "move", + "page_lengths": [100, 4], + "later_index": 103, + "later_record": {"event_id": "move-103", "created_at": "2026-02-03T04:05:06.000001Z"}, + "last_record": {"event_id": "move-103", "created_at": "2026-02-03T04:05:06.000001Z"}, + "record_overrides": {"move-103": {"created_at": "2026-02-03T04:05:06.000001Z"}}, + "source_count": 104, + "model": "MODEL (description 'Orders.', materialized table); SELECT 1 AS order_id" +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/nested_groups.py b/benchmarks/duckshop/edge_cases/orchestration/nested_groups.py new file mode 100644 index 0000000000..35727b2092 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/nested_groups.py @@ -0,0 +1,7 @@ +"""A group can contain jobs, but cannot contain another group.""" + +from sqlbuild.orchestration import job, job_group + +orders = job("orders_job", description="Build orders.", select="orders") +daily = job_group("daily_orders", description="Build daily orders.", jobs=[orders]) +weekly = job_group("weekly_orders", description="Unsupported nested membership.", jobs=[daily]) diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_orders.py b/benchmarks/duckshop/edge_cases/orchestration/node_orders.py new file mode 100644 index 0000000000..ac8f6cb8fc --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/node_orders.py @@ -0,0 +1,12 @@ +"""Trigger on each real orders rebuild, not on freshness-only success.""" + +from sqlbuild.orchestration import NodeEventKind, job, node_event, sensor + +source_orders = job("source_orders", description="Source orders.", select="orders") +inventory = job("inventory", description="Destination inventory.", select="stg_orders") +on_orders = sensor( + "on_orders", + description="Build inventory for every genuine orders rebuild.", + when=node_event(select="orders orders*", kinds=[NodeEventKind.SUCCESS]), + run=inventory, +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_sql.json b/benchmarks/duckshop/edge_cases/orchestration/node_sql.json new file mode 100644 index 0000000000..1e8b77124a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/node_sql.json @@ -0,0 +1,7 @@ +{ + "raw_sql": "SELECT {{ order_amount() }} AS amount -- Orders 🧾\n", + "compiled_sql": "SELECT 23 AS amount -- Orders 🧾\n", + "python_source": "def order_amount():\n return 23\n", + "function_sql": "43", + "updated_sql": "SELECT 37 AS amount -- Updated orders 🧾\n" +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_status.json b/benchmarks/duckshop/edge_cases/orchestration/node_status.json new file mode 100644 index 0000000000..420e0486be --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/node_status.json @@ -0,0 +1,39 @@ +{ + "model_metadata": { + "model_name": "orders", + "config": {"materialized": "table", "lookback": "2d"}, + "execution_signature": {"custom_config": {"label": "orders 🧾", "threshold": 1.25}}, + "local_function_hashes": {"order_total": "function-v1"} + }, + "model_local_hash": "34dd67f2ccccda42100a392d5bb611da80d22205a358cd5a45e8dc1c3f6d7e4b", + "query_hash": "orders-definition-latest", + "same_named_function": { + "name": "orders", + "destination": {"database": null, "schema": "main", "name": "orders"}, + "arguments": [], + "returns": "INTEGER", + "deps": [] + }, + "physical": { + "state_database": null, + "state_schema": "main", + "name": "orders", + "relation_type": "BASE TABLE", + "columns": [{"name": "order_id", "data_type": "INTEGER"}] + }, + "freshness": { + "event_id": "orders-source-newer", + "state_database": null, + "state_schema": "main", + "source_name": "raw_orders", + "target_database": null, + "target_schema": "main", + "target_name": "raw_orders", + "run_id": "external-orders", + "strategy": "max_column", + "value_kind": "integer", + "data_version": "18446744073709551619", + "data_version_hash": "orders-source-newer-version", + "observed_at": "2026-02-03T04:05:06Z" + } +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/oidc_identities.json b/benchmarks/duckshop/edge_cases/orchestration/oidc_identities.json new file mode 100644 index 0000000000..2b6150bdbc --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/oidc_identities.json @@ -0,0 +1,15 @@ +{ + "origin": "https://control.example.com", + "subject": "orders-person", + "invalid_tokens": [ + {"description": "issuer", "claims": {"iss": "https://other.example.com"}}, + {"description": "audience", "claims": {"aud": ["customers-control"]}}, + {"description": "nonce", "claims": {"nonce": "other-nonce"}}, + {"description": "expired", "claims": {"exp": 1}}, + {"description": "signature", "signing_key": "customers"}, + {"description": "access hash", "response": {"access_token": "customers-access"}}, + {"description": "missing token", "remove_fields": ["id_token"]} + ], + "ineligible_subjects": ["unknown-person", "absent-person"], + "lifecycle": ["login", "browser-mismatch", "replica-callback", "replay", "disabled-person", "expired-flow"] +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/old_name_history.json b/benchmarks/duckshop/edge_cases/orchestration/old_name_history.json new file mode 100644 index 0000000000..13d8a8496b --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/old_name_history.json @@ -0,0 +1,42 @@ +{ + "record": { + "state_database": null, + "state_schema": "main", + "event_id": "view-000", + "target_name": null, + "event_type": "required", + "migration_event_id": "move-000", + "destination_model": "orders", + "old_database": null, + "old_schema": "legacy", + "old_name": "orders_🧾", + "new_database": null, + "new_schema": "main", + "new_name": "orders_current", + "view_retention": "7d", + "archive_name": null, + "column_aliases": null, + "grants_copied": null, + "view_sql": null, + "expires_at": null, + "drop_reason": null, + "run_id": "external-move", + "created_at": "2026-02-03T04:05:06Z" + }, + "source_rows": 104, + "duplicate_rows": 3, + "later_time": "2026-02-03T04:05:06.000001Z", + "retained_result": "view-103", + "kind": "old_name_views", + "source_key": "event_id", + "timestamp_key": "created_at", + "legacy_columns": ["view_retention"], + "id_prefix": "view", + "page_lengths": [100, 4], + "later_index": 103, + "later_record": {"event_id": "view-103", "created_at": "2026-02-03T04:05:06.000001Z", "column_aliases": "{\"order_id\":\"legacy_order_id\"}", "expires_at": "2026-02-10T04:05:06Z", "view_sql": "SELECT 'o''rders 🧾' AS label"}, + "last_record": {"event_id": "view-103", "created_at": "2026-02-03T04:05:06.000001Z", "column_aliases": "{\"order_id\":\"legacy_order_id\"}", "expires_at": "2026-02-10T04:05:06Z", "view_sql": "SELECT 'o''rders 🧾' AS label"}, + "record_overrides": {"view-103": {"created_at": "2026-02-03T04:05:06.000001Z", "column_aliases": "{\"order_id\":\"legacy_order_id\"}", "expires_at": "2026-02-10T04:05:06Z", "view_sql": "SELECT 'o''rders 🧾' AS label"}}, + "source_count": 104, + "model": "MODEL (description 'Orders.', materialized table); SELECT 1 AS order_id" +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/orders_loader.py b/benchmarks/duckshop/edge_cases/orchestration/orders_loader.py new file mode 100644 index 0000000000..ad4c8b828d --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/orders_loader.py @@ -0,0 +1,7 @@ +from sqlbuild.loaders import loader + + +@loader +def raw_orders(ctx): + """Load synthetic orders in the target loader namespace.""" + return [{"order_id": 1}] diff --git a/benchmarks/duckshop/edge_cases/orchestration/password_credentials.json b/benchmarks/duckshop/edge_cases/orchestration/password_credentials.json new file mode 100644 index 0000000000..0f690df069 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/password_credentials.json @@ -0,0 +1,14 @@ +{ + "password": "synthetic-orders-password", + "replacement": "synthetic-customers-password", + "wrong": "synthetic-inventory-password", + "too_short": "订单客户库存", + "minimum_unicode": "订单客户库存发货支持产品", + "username": "orders-reader", + "origin": "https://control.example.com", + "maximum_password_bytes": 1024, + "http_password_character": "\u0000", + "http_replacement_character": "\u0001", + "invalid_origins": ["https://other.example.com", "null"], + "invalid_csrf": ["", "not-a-session-secret"] +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/platform_orders.py b/benchmarks/duckshop/edge_cases/orchestration/platform_orders.py new file mode 100644 index 0000000000..575d5ccaa6 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/platform_orders.py @@ -0,0 +1,30 @@ +"""Sensor-only roots retain referenced jobs while target filters and event enums stay native.""" + +from sqlbuild.orchestration import PlatformEventKind, job, platform_event, sensor + + +def definitions(): + refresh = job("health_orders", description="Refresh orders.", select="orders") + return [ + sensor( + "overdue_orders", + description="Catch up overdue order reporting.", + when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]), + run=refresh, + minimum_interval_seconds=7, + default_status="paused", + targets=("development",), + ), + sensor( + "other_target_orders", + description="Refresh another target.", + when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), + run=refresh, + targets=("production",), + ), + ] + + +triggers = definitions() +alias = triggers[0] +triggers.append(triggers) diff --git a/benchmarks/duckshop/edge_cases/orchestration/proxy_identities.json b/benchmarks/duckshop/edge_cases/orchestration/proxy_identities.json new file mode 100644 index 0000000000..bd2a25e5d1 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/proxy_identities.json @@ -0,0 +1,12 @@ +{ + "origin": "https://control.example.com", + "person": "orders-reader", + "subject": "external-orders-reader", + "absent_subject": "external-absent-reader", + "unknown_subject": "external-unbound-reader", + "untrusted_peer": "192.0.2.24", + "proxy_authorization": "Basic b3JkZXJzLXByb3h5", + "administrator_journey": { + "bound": ["orders-admin", "customers-admin"] + } +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_execution_inputs.json b/benchmarks/duckshop/edge_cases/orchestration/python_execution_inputs.json new file mode 100644 index 0000000000..87f831d781 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_execution_inputs.json @@ -0,0 +1,6 @@ +{ + "initial": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Orders are present.')\ndef orders_present(ctx):\n return ctx.pass_()\n", + "changed": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Orders are present.')\ndef orders_present(ctx):\n raise RuntimeError('changed check executed after publication')\n", + "task": "from sqlbuild.tasks import task\n@task(description='Notify support.')\ndef notify_support(ctx):\n return None\n", + "asset": "from sqlbuild.assets import asset\n@asset(description='Order export.')\ndef order_export(ctx):\n return []\n" +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql b/benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql new file mode 100644 index 0000000000..79baa81e01 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql @@ -0,0 +1,9 @@ +MODEL (description 'Retry a failed cast after warehouse input changes.', materialized table); +WITH source(value) AS (VALUES ('invalid')) +SELECT CASE + WHEN EXISTS ( + SELECT 1 FROM information_schema.tables WHERE table_name = 'retry_ready' + ) THEN 17 + ELSE CAST(value AS INTEGER) +END AS amount +FROM source diff --git a/benchmarks/duckshop/edge_cases/orchestration/run_orders.py b/benchmarks/duckshop/edge_cases/orchestration/run_orders.py new file mode 100644 index 0000000000..bf6d376187 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/run_orders.py @@ -0,0 +1,16 @@ +"""Freeze source-job selectors without executing user code in the scheduler.""" + +from sqlbuild.orchestration import RunEventKind, job, run_event, sensor + +source_orders = job("source_orders", description="Source orders.", select="orders") +inventory = job("inventory", description="Destination inventory.", select="stg_orders") +on_orders = sensor( + "on_orders", + description="Build inventory only after orders actually succeeds.", + when=run_event( + job=source_orders, + kinds=[RunEventKind.COMPLETED], + executed_successfully=("orders",), + ), + run=inventory, +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json b/benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json new file mode 100644 index 0000000000..12eb4f25d5 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json @@ -0,0 +1,12 @@ +{ + "description": "Three minutely cron jobs start ten hours back with the default latest catch-up. Two installed schedulers catch up while being SIGKILLed at seeded random moments; every lapse of a killed leader's lease is simulated. After a final uninterrupted scheduler, each trigger has ticked its latest due minute exactly once and nothing off schedule, each tick requests or is covered by exactly one run, and every requested run has one request event.", + "jobs": [ + "orders", + "customers", + "inventory" + ], + "backlog_minutes": 600, + "rounds": 16, + "seed": 912, + "max_kill_delay_ms": 600 +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/security_audit_delivery.json b/benchmarks/duckshop/edge_cases/orchestration/security_audit_delivery.json new file mode 100644 index 0000000000..cde7c2ee40 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/security_audit_delivery.json @@ -0,0 +1,7 @@ +{ + "events": 105, + "action": "role.assigned", + "scope": {"location": "orders", "target": "development"}, + "before": {"role": null}, + "after": {"role": "viewer", "note": "Order metadata — read only"} +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/security_audit_order.json b/benchmarks/duckshop/edge_cases/orchestration/security_audit_order.json new file mode 100644 index 0000000000..6e80dd8d79 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/security_audit_order.json @@ -0,0 +1,7 @@ +{ + "occurred_at": [ + "2026-02-02T12:00:00Z", + "2020-01-01T00:00:00Z", + "2026-02-01T12:00:00Z" + ] +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/security_audit_retention.json b/benchmarks/duckshop/edge_cases/orchestration/security_audit_retention.json new file mode 100644 index 0000000000..c34d0ef6cb --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/security_audit_retention.json @@ -0,0 +1,12 @@ +{ + "undelivered_past_horizon": { + "server_arguments": ["--audit-retention-days", "1"], + "rows": [ + {"action": "orders.delivered", "age_days": 2}, + {"action": "orders.undelivered", "age_days": 2}, + {"action": "orders.newest", "age_days": 0} + ], + "delivered_actions": ["orders.delivered"], + "expected_actions": ["orders.undelivered", "orders.newest"] + } +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/semantic_identity.json b/benchmarks/duckshop/edge_cases/orchestration/semantic_identity.json new file mode 100644 index 0000000000..5b1c5e0140 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/semantic_identity.json @@ -0,0 +1,57 @@ +{ + "files": { + "hooks/python/notifications.py": "from sqlbuild.hooks import hook\n\n@hook\ndef notify(ctx, channel):\n \"\"\"Notify support.\"\"\"\n return 'orders'\n", + "models/orders.sql": "MODEL (description 'Orders.', materialized 'table', pre_hooks [python('notify', channel: 'support')], columns (order_id (type INTEGER), amount (type BIGINT)));\nSELECT order_id, __udf(\"edge_next_quantity\")(amount) AS amount FROM __ref(\"stg_orders\")\n" + }, + "cases": [ + { + "description": "layout does not change semantic identity", + "path": "models/orders.sql", + "before": "SELECT order_id,", + "after": "-- Order presentation\nSELECT\n order_id,", + "expected_changed": [] + }, + { + "description": "config changes with identical query", + "path": "models/orders.sql", + "before": "materialized 'table'", + "after": "materialized 'view'", + "expected_changed": ["model:orders"] + }, + { + "description": "hook code changes with identical query and config", + "path": "hooks/python/notifications.py", + "before": "return 'orders'", + "after": "return 'products'", + "expected_changed": ["model:orders"] + }, + { + "description": "referenced function changes consumer local identity", + "path": "functions/sql/edge_cases/edge_next_quantity.sql", + "before": "quantity + 1", + "after": "quantity + 3", + "expected_changed": ["udf:edge_next_quantity", "model:orders"] + }, + { + "description": "upstream query changes only its local identity", + "path": "models/stg_orders.sql", + "before": "CAST(5 AS INTEGER)", + "after": "CAST(9 AS INTEGER)", + "expected_changed": ["model:stg_orders"] + }, + { + "description": "seed data changes its planner content identity", + "path": "seeds/edge_cases/edge_quantity_steps.csv", + "before": "\n7\n", + "after": "\n11\n", + "expected_changed": ["seed:edge_quantity_steps"] + }, + { + "description": "CSV layout is not seed semantic data", + "path": "seeds/edge_cases/edge_quantity_steps.csv", + "before": "\n7\n", + "after": "\n\"7\"\n", + "expected_changed": [] + } + ] +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/snapshot_retention.json b/benchmarks/duckshop/edge_cases/orchestration/snapshot_retention.json new file mode 100644 index 0000000000..e786fafac8 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/snapshot_retention.json @@ -0,0 +1,8 @@ +{ + "description": "Twelve publications with distinct SQL; a run pins the first. Collection keeps the newest ten plus the pinned one, then a one-snapshot horizon keeps only the newest and the pinned one.", + "publications": 12, + "keep": 10, + "expected_collected": 1, + "shorter_keep": 1, + "expected_after_shorter": [1, 12] +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/source_data_sensor.json b/benchmarks/duckshop/edge_cases/orchestration/source_data_sensor.json new file mode 100644 index 0000000000..6cbb034c46 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/source_data_sensor.json @@ -0,0 +1,26 @@ +{ + "record": { + "event_id": "orders-source-000", + "state_database": null, + "state_schema": "main", + "source_name": "raw_orders", + "target_database": null, + "target_schema": "main", + "target_name": "raw_orders", + "run_id": "external-orders-🧾", + "strategy": "max_column", + "value_kind": "integer", + "data_version": "18446744073709551617", + "data_version_hash": "orders-version-1", + "observed_at": "2026-02-03T04:05:06Z" + }, + "source_rows": 104, + "null_indices": [1, 102], + "unselected_index": 103, + "late_record": { + "event_id": "orders-source-late", + "data_version": "18446744073709551619", + "data_version_hash": "orders-version-2", + "observed_at": "2026-01-01T01:02:03Z" + } +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/source_orders.py b/benchmarks/duckshop/edge_cases/orchestration/source_orders.py new file mode 100644 index 0000000000..c700b7d236 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/source_orders.py @@ -0,0 +1,16 @@ +"""Consume changed warehouse source versions without treating a sync poll as new data.""" + +from sqlbuild.orchestration import job, sensor, source_new_data + +orders_refresh = job( + "orders_refresh", + description="Refresh orders for a changed source version.", + select="orders", +) +on_source_orders = sensor( + "on_source_orders", + description="Refresh orders for the first source version and each changed version.", + when=source_new_data(select="source:raw_orders"), + run=orders_refresh, + minimum_interval_seconds=1, +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/source_orders.yml b/benchmarks/duckshop/edge_cases/orchestration/source_orders.yml new file mode 100644 index 0000000000..17fc10a3dd --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/source_orders.yml @@ -0,0 +1,8 @@ +sources: + - name: raw_orders + description: Raw orders. + schema: main + table: raw_orders + columns: + - name: order_id + type: BIGINT diff --git a/benchmarks/duckshop/edge_cases/orchestration/source_stale_journey.json b/benchmarks/duckshop/edge_cases/orchestration/source_stale_journey.json new file mode 100644 index 0000000000..e993214129 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/source_stale_journey.json @@ -0,0 +1,24 @@ +{ + "record": { + "event_id": "orders-source-000", + "state_database": null, + "state_schema": "main", + "source_name": "raw_orders", + "target_database": null, + "target_schema": "main", + "target_name": "raw_orders", + "run_id": "external-orders", + "strategy": "max_column", + "value_kind": "timestamp", + "data_version": "2026-02-03T03:00:00+00:00", + "data_version_hash": "orders-timestamp-1", + "observed_at": "2026-02-03T04:05:06Z" + }, + "source_rows": 104, + "late_record": { + "event_id": "orders-source-later", + "data_version": "2026-02-03T03:01:00+00:00", + "data_version_hash": "orders-timestamp-2", + "observed_at": "2026-02-03T04:05:07Z" + } +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/source_stale_sensor.json b/benchmarks/duckshop/edge_cases/orchestration/source_stale_sensor.json new file mode 100644 index 0000000000..25ae61920d --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/source_stale_sensor.json @@ -0,0 +1,25 @@ +{ + "after_seconds": 60, + "source_count": 104, + "source_definitions": { + "source:raw_orders_103": { + "database": null, "schema": null, "table": "raw_orders", + "namespace": ["orders_data", "landing"] + } + }, + "cases": [ + {"index": 0, "age_seconds": 61, "offset_seconds": 10800}, + {"index": 1, "age_seconds": 60}, + {"index": 2, "age_seconds": 59}, + {"index": 3, "age_seconds": 600}, + {"index": 3, "age_seconds": 600, "null": true, "observed_offset_microseconds": 1}, + {"index": 4, "age_seconds": 600, "value_kind": "integer", "data_version": "18446744073709551617"}, + {"index": 5, "age_seconds": 600, "target_name": "archived_orders"}, + {"index": 103, "age_seconds": 61, "format": "%Y-%m-%dT%H:%M:%S%.f", "target_database": "orders_data", "target_schema": "landing"} + ], + "later_pages": [ + {"expected_cursor": "source:raw_orders_099", "expected_runs": 2}, + {"expected_cursor": null, "expected_runs": 4}, + {"expected_cursor": "source:raw_orders_099", "expected_runs": 4} + ] +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/source_version_returns.json b/benchmarks/duckshop/edge_cases/orchestration/source_version_returns.json new file mode 100644 index 0000000000..47d7c74963 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/source_version_returns.json @@ -0,0 +1,21 @@ +{ + "description": "A source's version changes 1 -> 2 -> 1. Before the last observation arrives, the warehouse janitor has pruned the earlier rows, so a complete scan delivers only the newest. Each change, including the return to version 1, queues exactly one run, and repeating the evaluation queues nothing more.", + "record": { + "state_database": null, + "state_schema": "main", + "source_name": "raw_orders", + "target_database": null, + "target_schema": "main", + "target_name": "raw_orders", + "run_id": "external-orders", + "strategy": "max_column", + "value_kind": "integer", + "observed_at": "2026-02-03T04:05:06Z" + }, + "observations": [ + {"event_id": "orders-version-a", "data_version": "1", "data_version_hash": "orders-version-1", "complete_scan": false}, + {"event_id": "orders-version-b", "data_version": "2", "data_version_hash": "orders-version-2", "complete_scan": false}, + {"event_id": "orders-version-c", "data_version": "1", "data_version_hash": "orders-version-1", "complete_scan": true} + ], + "expected_runs_after_each": [1, 2, 3] +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/stale_orders.py b/benchmarks/duckshop/edge_cases/orchestration/stale_orders.py new file mode 100644 index 0000000000..bf17c34280 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/stale_orders.py @@ -0,0 +1,13 @@ +from sqlbuild.orchestration import job, sensor, source_stale + +orders_refresh = job( + "orders_refresh", description="Refresh orders from stale timestamp data.", select="orders" +) + +on_stale_orders = sensor( + "on_stale_orders", + description="Refresh orders once for each timestamp version older than one minute.", + when=source_stale(select="source:raw_orders", after_seconds=60), + run=orders_refresh, + minimum_interval_seconds=1, +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/tick_retention.json b/benchmarks/duckshop/edge_cases/orchestration/tick_retention.json new file mode 100644 index 0000000000..b65084359b --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/tick_retention.json @@ -0,0 +1,13 @@ +{ + "description": "A backlog of covered ticks from coalesced runs ages past the tick horizon. One cleanup pass stops at its batch cap and the next pass finishes; the requested tick and the latest covered tick stay. The operator then resets the cursor, and catch-up replays only the latest occurrence, whose tick already exists, so nothing is recreated. A yearly schedule starting years back still gets exactly one catch-up tick, at its latest occurrence.", + "shift_days": 40, + "backlog_ticks": 1002, + "expected_first_pass": 1000, + "expected_pruned": 2, + "expected_survivors": [ + "covered", + "requested" + ], + "yearly_starts_at": "2015-01-01T00:00:00Z", + "yearly_latest_days_ago": 100 +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/token_rotation.sql b/benchmarks/duckshop/edge_cases/orchestration/token_rotation.sql new file mode 100644 index 0000000000..edc09eb1be --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/token_rotation.sql @@ -0,0 +1,13 @@ +-- Historical credentials must not regain authority through rotation. +INSERT INTO sqb_locations (location_id, name, created_by, created_at, archived_at) +VALUES ('00000000-0000-4000-8000-000000000031', 'archived-orders', 'synthetic-operator', '2020-01-01T00:00:00Z', '2020-01-02T00:00:00Z'); +INSERT INTO sqb_access_tokens + (token_id, secret_hash, location_id, scope_json, created_at, expires_at) +VALUES ( + '00000000-0000-4000-8000-000000000032', + '0000000000000000000000000000000000000000000000000000000000000000', + '00000000-0000-4000-8000-000000000031', + '{"kind":"location","location_id":"00000000-0000-4000-8000-000000000031"}', + '2020-01-01T00:00:00Z', + '2999-01-01T00:00:00Z' +); diff --git a/benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json b/benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json new file mode 100644 index 0000000000..b48ceea400 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json @@ -0,0 +1,34 @@ +{ + "record": { + "state_database": null, + "state_schema": "main", + "name": "orders_🧾", + "relation_type": "BASE TABLE", + "columns": [ + {"name": "order_id", "data_type": "INTEGER"}, + {"name": "note\"text", "data_type": "VARCHAR"} + ] + }, + "create_sql": "CREATE TABLE \"orders_🧾\" (order_id INTEGER, \"note\"\"text\" VARCHAR)", + "alter_sql": "ALTER TABLE \"orders_🧾\" ADD COLUMN shipped INTEGER", + "replace_sql": ["DROP TABLE \"orders_🧾\"", "CREATE VIEW \"orders_🧾\" AS SELECT 7 AS order_id"], + "drop_sql": "DROP VIEW \"orders_🧾\"", + "model": "MODEL (description 'Orders.', materialized table); SELECT 17 AS amount\n", + "extra_relations": 104, + "non_authoritative_database": null, + "postgres_sql": [ + "CREATE SCHEMA main", + "CREATE TABLE main.orders (order_id INTEGER, amount NUMERIC)", + "CREATE TABLE main.\"Orders\" (note TEXT, shipped BOOLEAN)" + ], + "postgres_records": [ + {"state_database": null, "state_schema": "main", "name": "Orders", "relation_type": "BASE TABLE", "columns": [{"name": "note", "data_type": "text"}, {"name": "shipped", "data_type": "boolean"}]}, + {"state_database": null, "state_schema": "main", "name": "orders", "relation_type": "BASE TABLE", "columns": [{"name": "order_id", "data_type": "integer"}, {"name": "amount", "data_type": "numeric"}]} + ], + "lifecycle": ["create", "unchanged scan", "column added", "table becomes view", "drop", "empty mirror rebuild"], + "read_namespaces": [ + {"database": null, "schema": "main"}, + {"database": "orders_data", "schema": "main"}, + {"database": null, "schema": "reporting"} + ] +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/warehouse_reconciliation.json b/benchmarks/duckshop/edge_cases/orchestration/warehouse_reconciliation.json new file mode 100644 index 0000000000..b354d4018f --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/warehouse_reconciliation.json @@ -0,0 +1,36 @@ +{ + "obsolete_count": 205, + "retained_count": 1000, + "source_freshness": { + "event_id": "reconciliation-keep", + "state_database": null, + "state_schema": "reconcile_orders", + "source_name": "raw_orders", + "target_database": null, + "target_schema": "main", + "target_name": "orders", + "run_id": "orders-build", + "strategy": "max_column", + "value_kind": "integer", + "data_version": "17", + "data_version_hash": "orders-version-17", + "observed_at": "2026-01-01T10:00:00Z" + }, + "fingerprints": { + "event_id": "reconciliation-keep", + "state_database": null, + "state_schema": "reconcile_orders", + "node_type": "model", + "node_name": "orders", + "target_database": null, + "target_schema": "main", + "target_name": "orders", + "run_id": "orders-build", + "definition_hash": "orders-definition", + "version_hash": "orders-version", + "schema_fingerprint": "orders-schema", + "definition": "select order_id from orders", + "metadata_json": "{}", + "ts": "2026-01-01T10:00:00Z" + } +} diff --git a/benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json b/benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json new file mode 100644 index 0000000000..3c99647743 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json @@ -0,0 +1,34 @@ +{ + "retained_hours": 205, + "expired_samples": 205, + "drop_during_sampling": { + "name": "orders_🧾", + "create_sql": "CREATE TABLE \"orders_🧾\" (order_id INTEGER)", + "drop_sql": "DROP TABLE \"orders_🧾\"", + "model": "MODEL (description 'Orders.', materialized table); SELECT 17 AS amount\n", + "failure": "relation disappeared before statistics sampling" + }, + "cleanup_interruption": { + "block_sql": "CREATE FUNCTION pause_statistics_cleanup() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN PERFORM pg_sleep(30); RETURN NULL; END $$; CREATE TRIGGER pause_statistics_cleanup BEFORE DELETE ON sqb_warehouse_statistics FOR EACH STATEMENT EXECUTE FUNCTION pause_statistics_cleanup();", + "wait_sql": "DO $$ DECLARE deadline timestamptz := clock_timestamp() + interval '5 seconds'; BEGIN WHILE NOT EXISTS (SELECT 1 FROM pg_locks l JOIN pg_stat_activity a ON a.pid=l.pid WHERE l.relation='sqb_warehouse_statistics'::regclass AND a.wait_event='PgSleep' AND a.query LIKE 'DELETE FROM sqb_warehouse_statistics%') LOOP IF clock_timestamp() >= deadline THEN RAISE EXCEPTION 'cleanup did not start'; END IF; PERFORM pg_stat_clear_snapshot(); PERFORM pg_sleep(0.02); END LOOP; END $$;", + "restore_sql": "SELECT pg_cancel_backend(a.pid) FROM pg_stat_activity a JOIN pg_locks l ON l.pid=a.pid WHERE l.relation='sqb_warehouse_statistics'::regclass AND a.wait_event='PgSleep' AND a.query LIKE 'DELETE FROM sqb_warehouse_statistics%'; DROP TRIGGER pause_statistics_cleanup ON sqb_warehouse_statistics; DROP FUNCTION pause_statistics_cleanup();" + }, + "record": { + "state_database": null, + "state_schema": "main", + "name": "orders_🧾_000", + "observed_at": "2026-01-01T00:00:00Z", + "row_count": 7, + "detail": "synthetic catalog estimate", + "byte_count": 4096, + "byte_count_basis": "synthetic_catalog_bytes" + }, + "sqlite": { + "setup_sql": [ + "CREATE TABLE orders (order_id INTEGER)", + "INSERT INTO orders VALUES (17), (31), (59)" + ], + "name": "orders", + "metadata_detail": "SQLite does not report table row counts from metadata" + } +} diff --git a/benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql b/benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql new file mode 100644 index 0000000000..d49cb02a03 --- /dev/null +++ b/benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql @@ -0,0 +1,8 @@ +FUNCTION ( + description 'Advance an order quantity in the inherited function namespace.', + tags [edge_case], + arguments (quantity INTEGER), + returns INTEGER, +); + +quantity + 1 diff --git a/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql new file mode 100644 index 0000000000..f86364144c --- /dev/null +++ b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql @@ -0,0 +1,16 @@ +MODEL ( + description 'A scheduled model whose implicit scalar-function writer must be claimed.', + tags [edge_case], + materialized table, + columns ( + quantity (type INTEGER, nullable false, audits [not_null]), + ), +); + +WITH final AS ( + SELECT __udf("edge_next_quantity")(q.quantity) AS quantity + FROM __seed("edge_quantity_steps") AS q +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py new file mode 100644 index 0000000000..8c5069c1d8 --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py @@ -0,0 +1,95 @@ +"""Exercise alias deduplication and cyclic factory containers during native discovery.""" + +from sqlbuild.orchestration import ( + PlatformEventKind, + RunEventKind, + job, + job_group, + platform_event, + run_event, + schedule, + sensor, +) + +edge_orders = job( + "edge_snapshot_orders", + description="Refresh only the synthetic edge-case models.", + select="tag:edge_case", + settings={"run_tests": False, "vars": {"edge_delivery_batch": "1"}, "reload": True}, + concurrency=2, + schedule=schedule( + cron="15 2 * * *", + timezone="UTC", + starts_at="2026-07-03T01:17:23.456789+05:30", + default_status="paused", + ), +) +edge_alias = edge_orders +edge_factory_results = [[edge_orders]] +edge_factory_results.append(edge_factory_results) +edge_group = job_group( + "edge_snapshot_group", + description="Exercise group discovery through shared, cyclic declaration containers.", + jobs=[edge_orders], +) +edge_factory_results.append(edge_group) + +edge_inventory = job( + "edge_snapshot_inventory", + description="Exercise a child limit independent of its parent.", + select="tag:edge_case", + concurrency_groups=("edge_products_limit",), +) +edge_products = job( + "edge_snapshot_products", + description="Exercise the opposing parent and child acquisition order.", + select="tag:edge_case", + concurrency_groups=("edge_inventory_limit",), +) +edge_inventory_group = job_group( + "edge_inventory_group", + description="Complete inventory without reserving its child's product limit.", + jobs=[edge_inventory], + concurrency_groups=("edge_inventory_limit",), +) +edge_products_group = job_group( + "edge_products_group", + description="Wait for safe admission when both installation limits are one.", + jobs=[edge_products], + concurrency_groups=("edge_products_limit",), +) +edge_health = sensor( + "edge_platform_orders", + description="Catch up retained platform observations with a stopped group sensor.", + when=platform_event( + kinds=[ + PlatformEventKind.TICK_OVERDUE, + PlatformEventKind.LOCATION_BROKEN, + PlatformEventKind.LAUNCH_FAILED, + ] + ), + run=edge_group, + minimum_interval_seconds=7, + default_status="paused", +) +edge_factory_results.append(edge_health) + +edge_job_outcomes = sensor( + "edge_job_outcomes", + description="Keep warning and partial outcomes distinct from success.", + when=run_event(job=edge_orders, kinds=[RunEventKind.WARN, RunEventKind.PARTIAL]), + run=edge_inventory, + default_status="paused", +) +edge_group_outcomes = sensor( + "edge_group_outcomes", + description="Observe group completion including cancellation and presumed failure.", + when=run_event( + job=edge_group, + kinds=[RunEventKind.COMPLETED], + executed_successfully=("tag:edge_case",), + ), + run=edge_inventory, + default_status="paused", +) +edge_factory_results.extend([edge_job_outcomes, edge_group_outcomes]) diff --git a/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.csv b/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.csv new file mode 100644 index 0000000000..9e70ca2ea3 --- /dev/null +++ b/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.csv @@ -0,0 +1,3 @@ +quantity +2 +7 diff --git a/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.yml b/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.yml new file mode 100644 index 0000000000..de890aa0b8 --- /dev/null +++ b/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.yml @@ -0,0 +1,7 @@ +seeds: + - name: edge_quantity_steps + description: Quantities whose seed destination inherits the target namespace. + tags: [edge_case] + columns: + - name: quantity + type: INTEGER diff --git a/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_scheduled_function.sql b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_scheduled_function.sql new file mode 100644 index 0000000000..8db1011dad --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_scheduled_function.sql @@ -0,0 +1,15 @@ +TEST (name "rpt_edge_scheduled_function__advances_the_inherited_function_quantity"); + +WITH __seed__edge_quantity_steps AS ( + SELECT v.quantity + FROM VALUES + (-2), + (23) AS v(quantity) +), + +__expected__rpt_edge_scheduled_function AS ( + SELECT v.quantity + FROM VALUES + (-1), + (24) AS v(quantity) +) diff --git a/clients/typescript/.gitignore b/clients/typescript/.gitignore new file mode 100644 index 0000000000..c2658d7d1b --- /dev/null +++ b/clients/typescript/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/clients/typescript/README.md b/clients/typescript/README.md new file mode 100644 index 0000000000..107e957f3d --- /dev/null +++ b/clients/typescript/README.md @@ -0,0 +1,112 @@ +# Control API client + +The client uses the native server's OpenAPI request and response types with +`openapi-fetch`. It adds no permissions, credential storage, retries or sync policy. +This package is private and consumed from source; it is not published to npm. + +Rebuild the native package after changing Rust API types, then run from the repository root: + +```sh +uv run --no-sync npm --prefix clients/typescript ci +uv run --no-sync npm --prefix clients/typescript run generate +uv run --no-sync npm --prefix clients/typescript run check +uv run --no-sync npm --prefix clients/typescript test +``` + +Generation invokes `sqb server --openapi` without opening a control database or +importing project Python. Set `SQLBUILD_EXECUTABLE` to test another built binary. +The check regenerates in memory and fails if the committed types differ. Tests use +a real native server, disposable SQLite store and CLI-issued location credential. + +```ts +import { createControlClient } from "@sqlbuild/control-client"; + +const client = createControlClient({ + baseUrl: serverUrl, + headers: { Authorization: `Bearer ${locationCredential}` }, +}); +const { data, error, response } = await client.GET( + "/api/locations/{location_id}/targets/{target}/runs", + { params: { path: { location_id: locationId, target }, query: { limit: 100 } } }, +); +``` + +Inspect `response.status` and `error` before using `data`. Pass an `AbortSignal` +for cancellation. For the scoped changes endpoint, set `parseAs: "stream"` and +consume SSE `changed` frames. An `all` topic, a reconnect, or a lost stream requires +refetching the authorized REST resources; the stream is not durable history. +Browser `EventSource` cannot set a bearer header: use a fetch-based SSE consumer +for the existing machine-authenticated endpoint. Human browser sessions are a +separate authentication contract, not an inferred privilege of this client. + +## Authenticate through a trusted proxy + +Create a person with `sqb admin create-principal orders-reader`, then put the +returned `principal_id` in a server-owned JSON file. Subjects are exact ASCII +upstream identities, not inferred local usernames or email addresses: + +```json +{ + "origin": "https://control.example.com", + "trusted_peers": ["127.0.0.1"], + "identities": { + "external-orders-reader": "00000000-0000-0000-0000-000000000001" + } +} +``` + +Replace the example UUID with the created person's ID, then start +`sqb server --trusted-proxy-config proxy.json`. The proxy must terminate HTTPS, +remove any client-supplied `X-SQB-User`, and replace it with the authenticated +upstream subject. Restrict access to the HTTP backend to the configured proxy +peers. Trusting loopback also trusts other local processes on that host; +`Forwarded` and `X-Forwarded-For` never establish peer trust. + +Through that proxy, each person request authenticates as the bound person; it +issues no cookie or bearer and grants no machine or resource authority. Requests +that change state must carry the configured `Origin`. Unbound subjects and missing +or disabled people are denied; accounts are not auto-provisioned. Password and +trusted-proxy modes cannot be combined. No default role or SSO group assignment +is inferred from the subject. + +## Verify an OIDC identity + +Register an authorization-code client with your HTTPS identity provider, using +`https://control.example.com/api/auth/oidc/callback` as its redirect URI. Create +each person through the local control-store-owner administration path and put +their UUID and exact provider subject in a server-owned JSON configuration: + +```json +{ + "origin": "https://control.example.com", + "issuer": "https://identity.example.com", + "client_id": "orders-control", + "identities": { + "external-orders-reader": "00000000-0000-0000-0000-000000000001" + } +} +``` + +Start `sqb server --oidc-config oidc.json` behind an HTTPS-terminating proxy that +preserves the configured Host and restricts access to the HTTP backend. For a +confidential client, add `client_secret_env` naming the server environment +variable holding its secret; never put the secret in the JSON file. To trust a +private provider CA, add `certificate_authority` pointing to a PEM bundle. +Relative certificate paths resolve from the server process's working directory. +TLS verification remains mandatory and upstream HTTP redirects are not followed. +Password, proxy, disabled-auth and OIDC human-authentication modes are exclusive. + +Navigate the browser to `/api/auth/oidc/login`. The server redirects to the +provider using S256 PKCE and sets a Secure, HttpOnly, host-only, five-minute +SameSite=Lax flow cookie. The provider's GET callback must return through that +same browser. State is single-use across server replicas; a changed server +identity configuration requires starting a fresh flow. The server validates +the ID token's signature, issuer, audience, expiry and nonce, and an access-token +hash when supplied. Provider keys are refreshed for each exchange. + +The callback returns only `principal_id` and clears the flow cookie. Unbound, +missing or disabled people are denied. This identity-only +endpoint does not create an account, browser/CLI session, bearer or role grant, +and cannot authenticate machine routes. It is not a complete browser SSO or +`sqb login` workflow. Authentication denial and successful identity verification +are audited; flow starts share a 20-per-minute real-peer limit across replicas. diff --git a/clients/typescript/package-lock.json b/clients/typescript/package-lock.json new file mode 100644 index 0000000000..342284a966 --- /dev/null +++ b/clients/typescript/package-lock.json @@ -0,0 +1,431 @@ +{ + "name": "@sqlbuild/control-client", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@sqlbuild/control-client", + "dependencies": { + "openapi-fetch": "0.17.0" + }, + "devDependencies": { + "@types/node": "22.20.5", + "openapi-typescript": "7.13.0", + "typescript": "5.9.3" + }, + "engines": { + "node": ">=22.18.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.29.7", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@redocly/ajv": { + "version": "8.11.2", + "resolved": "https://registry.npmjs.org/@redocly/ajv/-/ajv-8.11.2.tgz", + "integrity": "sha512-io1JpnwtIcvojV7QKDUSIuMN/ikdOUd1ReEnUnMKGfDVridQZ31J0MmIuqwuRjWDZfmvr+Q0MqCcfHM2gTivOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js-replace": "^1.0.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@redocly/config": { + "version": "0.22.0", + "resolved": "https://registry.npmjs.org/@redocly/config/-/config-0.22.0.tgz", + "integrity": "sha512-gAy93Ddo01Z3bHuVdPWfCwzgfaYgMdaZPcfL7JZ7hWJoK9V0lXDbigTWkhiPFAaLWzbOJ+kbUQG1+XwIm0KRGQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@redocly/openapi-core": { + "version": "1.34.22", + "resolved": "https://registry.npmjs.org/@redocly/openapi-core/-/openapi-core-1.34.22.tgz", + "integrity": "sha512-dEyx4uHh6Pc6ZkdU9Xe8rA4h12Odq6QKU/fqseBn07xwVwskhHiF3KHhqmw5BR+jAiFTvKTmwCafFql/k1pSzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@redocly/ajv": "8.11.2", + "@redocly/config": "0.22.0", + "colorette": "1.4.0", + "https-proxy-agent": "7.0.6", + "js-levenshtein": "1.1.6", + "js-yaml": "4.3.2", + "minimatch": "5.1.9", + "pluralize": "8.0.0", + "yaml-ast-parser": "0.0.43" + }, + "engines": { + "node": ">=18.17.0", + "npm": ">=9.5.0" + } + }, + "node_modules/@types/node": { + "version": "22.20.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.5.tgz", + "integrity": "sha512-U2+DNr+wSjpsTS/wZGYHq7GcwfuSmKiKvoPvK22zwTlRhU91yOniN4qRR5KhIjvif7ysw/dz/hKmfDH0Ris4aA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ansi-colors": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", + "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.7.tgz", + "integrity": "sha512-uZbew1NqdmPDTMJ8ah1y+b+9QEJrfkXFk3RcTQw3X0jW/xRUvFKsg1CfQdSYGdTbXZWExtU3J3ccxtnfw1Fi0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/change-case": { + "version": "5.4.4", + "resolved": "https://registry.npmjs.org/change-case/-/change-case-5.4.4.tgz", + "integrity": "sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w==", + "dev": true, + "license": "MIT" + }, + "node_modules/colorette": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.4.0.tgz", + "integrity": "sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g==", + "dev": true, + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/index-to-position": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/index-to-position/-/index-to-position-1.2.0.tgz", + "integrity": "sha512-Yg7+ztRkqslMAS2iFaU+Oa4KTSidr63OsFGlOrJoW981kIYO3CGCS3wA95P1mUi/IVSJkn0D479KTJpVpvFNuw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/js-levenshtein": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/js-levenshtein/-/js-levenshtein-1.1.6.tgz", + "integrity": "sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/minimatch": { + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", + "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/openapi-fetch": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/openapi-fetch/-/openapi-fetch-0.17.0.tgz", + "integrity": "sha512-PsbZR1wAPcG91eEthKhN+Zn92FMHxv+/faECIwjXdxfTODGSGegYv0sc1Olz+HYPvKOuoXfp+0pA2XVt2cI0Ig==", + "license": "MIT", + "dependencies": { + "openapi-typescript-helpers": "^0.1.0" + } + }, + "node_modules/openapi-typescript": { + "version": "7.13.0", + "resolved": "https://registry.npmjs.org/openapi-typescript/-/openapi-typescript-7.13.0.tgz", + "integrity": "sha512-EFP392gcqXS7ntPvbhBzbF8TyBA+baIYEm791Hy5YkjDYKTnk/Tn5OQeKm5BIZvJihpp8Zzr4hzx0Irde1LNGQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@redocly/openapi-core": "^1.34.6", + "ansi-colors": "^4.1.3", + "change-case": "^5.4.4", + "parse-json": "^8.3.0", + "supports-color": "^10.2.2", + "yargs-parser": "^21.1.1" + }, + "bin": { + "openapi-typescript": "bin/cli.js" + }, + "peerDependencies": { + "typescript": "^5.x" + } + }, + "node_modules/openapi-typescript-helpers": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/openapi-typescript-helpers/-/openapi-typescript-helpers-0.1.0.tgz", + "integrity": "sha512-OKTGPthhivLw/fHz6c3OPtg72vi86qaMlqbJuVJ23qOvQ+53uw1n7HdmkJFibloF7QEjDrDkzJiOJuockM/ljw==", + "license": "MIT" + }, + "node_modules/parse-json": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-8.3.0.tgz", + "integrity": "sha512-ybiGyvspI+fAoRQbIPRddCcSTV9/LsJbf0e/S85VLowVGzRmokfneg2kwVW/KU5rOXrPSbF1qAKPMgNTqqROQQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.26.2", + "index-to-position": "^1.1.0", + "type-fest": "^4.39.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/pluralize": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz", + "integrity": "sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/supports-color": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz", + "integrity": "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/type-fest": { + "version": "4.41.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", + "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/uri-js-replace": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/uri-js-replace/-/uri-js-replace-1.0.2.tgz", + "integrity": "sha512-VeBBt/nMxedczht3KDL2yBBMJqIxpCQLMZp3UJ1otcWwUHdQW4IiyHpJivExzSK+FdxTowRfGL4FmwDqH9+S8Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/yaml-ast-parser": { + "version": "0.0.43", + "resolved": "https://registry.npmjs.org/yaml-ast-parser/-/yaml-ast-parser-0.0.43.tgz", + "integrity": "sha512-2PTINUwsRqSd+s8XxKaJWQlUuEMHJQyEuh2edBbW8KNJz0SJPwUSD2zRWqezFEdN7IzAgeuYHFUCF7o8zRdZ0A==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + } + } +} diff --git a/clients/typescript/package.json b/clients/typescript/package.json new file mode 100644 index 0000000000..cb648a1b4f --- /dev/null +++ b/clients/typescript/package.json @@ -0,0 +1,20 @@ +{ + "name": "@sqlbuild/control-client", + "private": true, + "type": "module", + "engines": { "node": ">=22.18.0" }, + "exports": "./src/client.ts", + "scripts": { + "generate": "node scripts/generate.mjs", + "check": "node scripts/generate.mjs --check && tsc --noEmit", + "test": "node --experimental-strip-types --test tests/*.test.ts" + }, + "dependencies": { + "openapi-fetch": "0.17.0" + }, + "devDependencies": { + "@types/node": "22.20.5", + "openapi-typescript": "7.13.0", + "typescript": "5.9.3" + } +} diff --git a/clients/typescript/scripts/generate.mjs b/clients/typescript/scripts/generate.mjs new file mode 100644 index 0000000000..07774645ac --- /dev/null +++ b/clients/typescript/scripts/generate.mjs @@ -0,0 +1,20 @@ +import { execFileSync } from "node:child_process"; +import { readFile, writeFile } from "node:fs/promises"; +import openapiTS, { astToString } from "openapi-typescript"; + +const destination = new URL("../src/schema.d.ts", import.meta.url); +const document = execFileSync(process.env.SQLBUILD_EXECUTABLE ?? "sqb", ["server", "--openapi"], { + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, +}); +const generated = "// Generated from sqb server --openapi. Run npm run generate; do not edit.\n" + + astToString(await openapiTS(JSON.parse(document))); +if (process.argv.includes("--check")) { + if (await readFile(destination, "utf8") !== generated) { + throw new Error("Control client is stale; rebuild sqb and run npm run generate."); + } + console.log("Control client matches the native OpenAPI contract."); +} else { + await writeFile(destination, generated); + console.log("Control client generated from the native OpenAPI contract."); +} diff --git a/clients/typescript/src/client.ts b/clients/typescript/src/client.ts new file mode 100644 index 0000000000..ad9a19251f --- /dev/null +++ b/clients/typescript/src/client.ts @@ -0,0 +1,9 @@ +import createClient, { type ClientOptions } from "openapi-fetch"; +import type { paths } from "./schema.d.ts"; + +export type { components, operations, paths } from "./schema.d.ts"; + +/** Typed transport only: credentials and request cancellation belong to the caller. */ +export function createControlClient(options: ClientOptions) { + return createClient(options); +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts new file mode 100644 index 0000000000..e08a30bc2e --- /dev/null +++ b/clients/typescript/src/schema.d.ts @@ -0,0 +1,3699 @@ +// Generated from sqb server --openapi. Run npm run generate; do not edit. +export interface paths { + "/api/admin/people/{principal_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["changeAdministrator"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/admin/roles": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["assignBuiltinRole"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/auth/cli/approve": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["approveCliLogin"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/auth/cli/claim": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["claimCliLogin"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/auth/cli/start": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["startCliLogin"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/auth/logout": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["logoutSession"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/auth/oidc/callback": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** @description Validates one browser-bound code and issues a bounded browser session for an explicitly bound person; no provisioning, personal API tokens or implicit grants. */ + get: operations["oidcIdentity"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/auth/oidc/login": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** @description Begins a browser-bound code flow with S256 PKCE; no account provisioning or grants. */ + get: operations["loginOidc"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/auth/password": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["changePassword"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/auth/password/login": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["loginPassword"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/auth/session": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["currentSession"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/audit": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readSecurityAudit"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/locations/{location_id}/targets/{target}/catalog": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPersonCatalog"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/locations/{location_id}/targets/{target}/connection-tests": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["requestConnectionTest"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/locations/{location_id}/targets/{target}/connection-tests/{request_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readConnectionTest"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/locations/{location_id}/targets/{target}/fingerprint": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPersonFingerprint"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/locations/{location_id}/targets/{target}/node-status": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPersonNodeStatus"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/sql": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPersonNodeSql"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/locations/{location_id}/targets/{target}/runs": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["listPersonRuns"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/locations/{location_id}/targets/{target}/statistics": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPersonStatistics"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/runs": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["requestJobRun"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/runs/cancel": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["cancelPersonRun"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/runs/rerun": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["rerunPersonRun"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/triggers": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["changeTrigger"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/health": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["read_location_health"]; + put?: never; + post: operations["report_location_health"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/changes": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["subscribe"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/metadata": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["report_metadata"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/metadata/statistics-due": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["statistics_due"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/metadata/watermark": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["watermark"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/runs": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["list_runs"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/invocations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["invocations"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/nodes": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["nodes"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/recall/{view}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["read_run_recall"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/snapshots": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["current"]; + put?: never; + post: operations["publish"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/snapshots/metadata": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["metadata"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/triggers": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["list_triggers"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/events": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["events"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/ticks": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["ticks"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/work/poll": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["poll"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/locations/{location_id}/work/{request_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["result"]; + put?: never; + post: operations["complete"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/runs/{run_id}/events": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["report_run_event"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/runs/{run_id}/execution": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["execution"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/runs/{run_id}/heartbeat": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["heartbeat"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/runs/{run_id}/output": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["output"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/runs/{run_id}/recall": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["recall"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; +} +export type webhooks = Record; +export interface components { + schemas: { + /** + * @description Administrator membership and person eligibility share the final-admin invariant. + * @enum {string} + */ + AdministratorChange: "grant" | "revoke" | "disable" | "enable"; + ApiErrorBody: { + /** + * Format: uuid + * @description Existing durable denial audit ID; absent when no decision was recorded. + */ + decision_id?: string | null; + error: string; + }; + AuthenticationDenial: { + error: string; + /** Format: uuid */ + request_id: string; + }; + /** @enum {string} */ + BuiltinRole: "viewer" | "operator" | "developer" | "admin"; + /** @description Browser cancellation cannot supply an actor, claim or lifecycle transition. */ + CancelRunRequest: { + /** Format: uuid */ + event_id: string; + reason: string; + /** Format: uuid */ + run_id: string; + }; + CatalogObservation: { + /** Format: uuid */ + event_id: string; + /** Format: date-time */ + observed_at: string; + record: components["schemas"]["WarehouseCatalogRecord"]; + }; + CatalogPage: { + next_after?: null | components["schemas"]["WarehouseMirrorCursor"]; + records: components["schemas"]["CatalogObservation"][]; + }; + /** @description SSE invalidation topics; `all` means refetch the authorized REST resources. */ + ChangeNotice: { + topic: string; + }; + CliLoginApprovalRequest: { + approve: boolean; + browser_code: string; + }; + CliLoginClaimRequest: { + client_secret: string; + /** Format: uuid */ + flow_id: string; + }; + CliLoginSessionResponse: { + bearer: string; + identity: components["schemas"]["SessionIdentity"]; + }; + CliLoginStartRequest: { + client_hash: string; + }; + /** @description A row whose key was already stored with different content; the first row is kept. */ + ConflictEvidence: { + existing_sha256: string; + record_key: string; + rejected_sha256: string; + table: components["schemas"]["Table"]; + }; + ConnectionTestRequest: { + reason: string; + /** Format: uuid */ + request_id: string; + }; + /** @description The current successful snapshot, read with a consistent generation and change set. */ + CurrentSnapshot: { + changes: components["schemas"]["SnapshotChange"][]; + documents: { + [key: string]: unknown; + }; + nodes: { + [key: string]: components["schemas"]["SnapshotNode"]; + }; + record: components["schemas"]["SnapshotRecord"]; + }; + /** @description Location-owned Python returns data; only the fenced scheduler may apply it. */ + CustomSensorOutcome: { + cursor?: string | null; + run_keys: string[]; + /** @enum {string} */ + status: "completed"; + } | { + cursor?: string | null; + reason: string; + /** @enum {string} */ + status: "skipped"; + } | { + /** @enum {string} */ + status: "failed"; + } | { + /** @enum {string} */ + status: "timed_out"; + }; + /** @description Local definition identity only, not upstream data or graph currency. */ + DefinitionEvidence: { + local_hash: string; + /** @enum {string} */ + status: "matched"; + } | { + current_local_hash: string; + recorded_local_hash: string; + /** @enum {string} */ + status: "changed"; + } | { + current_local_hash?: string | null; + reason: string; + /** @enum {string} */ + status: "unavailable"; + }; + ExternalIdentityResponse: { + csrf_token: string; + /** Format: date-time */ + expires_at: string; + /** Format: uuid */ + principal_id: string; + /** Format: uuid */ + session_id: string; + }; + /** @description Build evidence only: no SQL, arbitrary metadata, or inference about live relation existence. */ + FingerprintObservation: { + definition_hash: string; + /** Format: uuid */ + event_id: string; + /** Format: date-time */ + observed_at: string; + run_id: string; + schema_fingerprint: string; + source_event_id: string; + version_hash: string; + }; + HeartbeatResponse: { + cancel_requested: boolean; + }; + /** @description What one insert batch did, per table. */ + InsertOutcome: { + conflicts: components["schemas"]["ConflictEvidence"][]; + /** @description Rows written or already present with equal content, by table. */ + delivered: [ + "invocations" | "output" | "events" | "diagnostics" | "artifacts" | "artifact_refs" | "heartbeats" | "delivery" | "meta" | "conflicts", + number + ][]; + /** @description Tables whose insert failed, so their rows were not written, and the first failure. */ + failed: components["schemas"]["Table"][]; + failure?: string | null; + }; + /** @description A trusted operator names a published job; execution inputs are never supplied by the caller. */ + JobRunRequest: { + job_key: string; + /** Format: uuid */ + location_id: string; + /** Format: int64 */ + priority: number; + reason: string; + /** Format: uuid */ + run_id: string; + target: string; + window?: null | components["schemas"]["RunWindow"]; + }; + /** @description Locations report bounded facts, never connection strings or user exception text. */ + LocationHealth: { + /** @enum {string} */ + kind: "heartbeat"; + } | { + /** Format: int64 */ + generation: number; + /** @enum {string} */ + kind: "compile_failed"; + } | { + /** Format: int64 */ + generation: number; + healthy: boolean; + /** @enum {string} */ + kind: "connection"; + } | { + /** Format: int64 */ + elapsed_millis: number; + failure?: null | components["schemas"]["WarehouseSyncFailure"]; + /** Format: int64 */ + generation: number; + /** @enum {string} */ + kind: "warehouse_sync"; + location: components["schemas"]["WarehouseStateLocation"]; + /** Format: int64 */ + records_sent: number; + table: components["schemas"]["WarehouseHistoryKind"]; + }; + /** @description Receipt time belongs to the server; consumers derive freshness rather than storing availability. */ + LocationHealthObservation: { + /** Format: date-time */ + received_at: string; + report: components["schemas"]["LocationHealthReport"]; + }; + LocationHealthReport: { + /** Format: uuid */ + event_id: string; + health: components["schemas"]["LocationHealth"]; + /** Format: uuid */ + session_id: string; + target: string; + }; + /** @description Only named location operations cross this boundary, never arbitrary SQL or Python. */ + LocationWork: { + /** @enum {string} */ + kind: "check_connection"; + } | { + evaluation: Record; + /** @enum {string} */ + kind: "evaluate_sensor"; + }; + LocationWorkClaim: { + /** Format: uuid */ + claim_id: string; + /** Format: int64 */ + generation: number; + /** Format: date-time */ + lease_expires_at: string; + request: components["schemas"]["LocationWorkRequest"]; + /** Format: uuid */ + session_id: string; + }; + LocationWorkOutcome: { + healthy: boolean; + /** @enum {string} */ + kind: "connection"; + } | { + /** @enum {string} */ + kind: "sensor_evaluation"; + outcome: components["schemas"]["CustomSensorOutcome"]; + } | { + /** @enum {string} */ + kind: "superseded"; + }; + LocationWorkPoll: { + /** Format: uuid */ + session_id: string; + target: string; + }; + /** @description Expiry is derived from the deadline; only requests, leases and completed results are stored. */ + LocationWorkRecord: { + /** Format: date-time */ + completed_at?: string | null; + /** Format: date-time */ + expires_at: string; + /** Format: int64 */ + generation: number; + request_json: components["schemas"]["LocationWorkRequest"]; + /** Format: date-time */ + requested_at: string; + requested_by: string; + result_json?: null | components["schemas"]["LocationWorkResult"]; + }; + LocationWorkRequest: { + /** Format: uuid */ + location_id: string; + reason: string; + /** Format: uuid */ + request_id: string; + target: string; + work: components["schemas"]["LocationWork"]; + }; + LocationWorkResult: { + /** Format: uuid */ + claim_id: string; + outcome: components["schemas"]["LocationWorkOutcome"]; + /** Format: uuid */ + session_id: string; + }; + /** @description One bounded page of one state table from the registered reader of a published generation. */ + MetadataBatch: { + /** Format: int64 */ + generation: number; + kind: components["schemas"]["WarehouseHistoryKind"]; + records: Record[]; + scan?: null | components["schemas"]["MetadataScan"]; + /** Format: uuid */ + session_id: string; + }; + /** @description A full scan's complete page removes the namespace's rows that the scan did not deliver. */ + MetadataScan: { + complete: boolean; + location: components["schemas"]["WarehouseStateLocation"]; + /** Format: uuid */ + scan_id: string; + }; + MutationResponse: { + applied: boolean; + }; + /** @enum {string} */ + NodeEventKind: "success" | "failed" | "recovered"; + /** @enum {string} */ + NodeOutcome: "success" | "warn" | "failed" | "skipped" | "cancelled"; + /** @description SQL inspection never returns Python source, execution artifacts or unrelated node metadata. */ + NodeSql: { + compiled_sql?: string | null; + node_key: string; + raw_sql?: string | null; + /** Format: uuid */ + snapshot_id: string; + }; + NodeStatus: { + build?: null | components["schemas"]["FingerprintObservation"]; + definition: components["schemas"]["DefinitionEvidence"]; + node_key: string; + physical: components["schemas"]["PhysicalEvidence"]; + /** Format: uuid */ + snapshot_id: string; + source_data: components["schemas"]["SourceDataStatus"]; + }; + /** @description Output of one stream that was not recorded because it exceeded `max_output_bytes`. */ + OmittedOutput: { + /** Format: int64 */ + bytes: number; + /** + * Format: int64 + * @description Sequences this gap accounts for, its own included. + */ + sequences: number; + }; + /** @description One exact-byte output chunk. */ + OutputChunk: { + /** + * Format: int64 + * @description Bytes of the same stream before this chunk. + */ + byte_offset: number; + data: number[]; + /** + * Format: int64 + * @description Output records the queue dropped just before this one. + */ + dropped_before: number; + occurred_at: string; + omitted?: null | components["schemas"]["OmittedOutput"]; + /** + * Format: int64 + * @description Per invocation, from 0, shared by both streams. + */ + sequence: number; + stream: components["schemas"]["OutputStream"]; + }; + /** @description Differing retries retain conflict evidence while preserving the first output bytes. */ + OutputPublication: { + /** Format: int32 */ + conflicts: number; + /** Format: int32 */ + duplicates: number; + /** Format: int32 */ + inserted: number; + }; + /** + * @description A process output stream. + * @enum {string} + */ + OutputStream: "stdout" | "stderr"; + PasswordChangeRequest: { + current_password: string; + new_password: string; + }; + PasswordLoginRequest: { + password: string; + username: string; + }; + PasswordSessionResponse: { + csrf_token: string; + identity: components["schemas"]["SessionIdentity"]; + }; + /** @description Observed physical evidence is not a live warehouse-existence guarantee. */ + PhysicalEvidence: { + observation: components["schemas"]["CatalogObservation"]; + /** @enum {string} */ + status: "observed"; + } | { + reason: string; + /** @enum {string} */ + status: "unavailable"; + }; + /** @enum {string} */ + PlatformEventKind: "heartbeat" | "scheduler_no_leader" | "tick_overdue" | "calendar_horizon_low" | "location_broken" | "launch_failed"; + /** @description One invocation's records; the server owns its claim namespace, start time and version. */ + RecallBatch: { + invocation_id: string; + records: components["schemas"]["RecallRecord"][]; + }; + RecallRecord: { + table: components["schemas"]["Table"]; + values: { + [key: string]: components["schemas"]["RecallValue"]; + }; + }; + /** @description One stored value. */ + RecallValue: { + /** @enum {string} */ + kind: "null"; + } | { + /** @enum {string} */ + kind: "integer"; + /** Format: int64 */ + value: number; + } | { + /** @enum {string} */ + kind: "real"; + /** Format: double */ + value: number; + } | { + /** @enum {string} */ + kind: "text"; + value: string; + } | { + /** @enum {string} */ + kind: "blob"; + value: number[]; + }; + /** @description A retry derives its execution scope from a terminal run, never from caller selectors. */ + RerunRequest: { + from_failure: boolean; + /** Format: int64 */ + priority: number; + reason: string; + /** Format: uuid */ + run_id: string; + /** Format: uuid */ + source_run_id: string; + }; + RoleAssignmentRequest: { + /** Format: uuid */ + location_id?: string | null; + /** Format: uuid */ + principal_id: string; + revoke?: boolean; + role: components["schemas"]["RoleKey"]; + target?: string | null; + }; + /** @description A built-in role by name, or a custom role a location publishes. */ + RoleKey: components["schemas"]["BuiltinRole"] | string; + /** @description This is a server-owned instruction, not an arbitrary status supplied by a worker. */ + RunChange: { + container_id?: string | null; + /** @enum {string} */ + kind: "launched"; + } | { + code: string; + /** @enum {string} */ + kind: "launch_failed"; + } | { + /** @enum {string} */ + kind: "started"; + } | { + /** @enum {string} */ + kind: "node_started"; + node_key: string; + } | { + error_code?: string | null; + /** @enum {string} */ + kind: "node_finished"; + node_key: string; + outcome: components["schemas"]["NodeOutcome"]; + /** Format: int64 */ + rows_affected?: number | null; + skipped_because_of?: string | null; + } | { + /** @enum {string} */ + kind: "check_started"; + node_key: string; + } | { + /** @enum {string} */ + kind: "check_finished"; + node_key: string; + outcome: components["schemas"]["NodeOutcome"]; + } | { + /** @enum {string} */ + kind: "unresponsive"; + } | { + /** @enum {string} */ + kind: "recovered"; + } | { + actor: string; + /** @enum {string} */ + kind: "cancel_requested"; + reason?: string; + } | { + /** Format: int64 */ + exit_code?: number | null; + /** @enum {string} */ + kind: "finished"; + outcome: components["schemas"]["RunOutcome"]; + reason: string; + }; + RunChangeRequest: { + change: components["schemas"]["RunChange"]; + /** Format: uuid */ + claim_id?: string | null; + /** Format: uuid */ + event_id: string; + /** Format: uuid */ + run_id: string; + }; + /** @description The database owns the claim before a launcher sees the pinned request. */ + RunClaim: { + /** Format: uuid */ + claim_id: string; + request: components["schemas"]["RunRequest"]; + }; + /** @enum {string} */ + RunEventKind: "completed" | "success" | "warn" | "partial" | "failed" | "cancelled"; + /** @description Workers compile locally and compare against this immutable snapshot, not the current deploy. */ + RunExecution: { + claim: components["schemas"]["RunClaim"]; + environment_hash: string; + node_hashes: { + [key: string]: string; + }; + /** @description Derived from completed sibling facts and pinned lineage, never a mutable request rewrite. */ + skipped_nodes?: { + [key: string]: string; + }; + sqb_version: string; + }; + /** @enum {string} */ + RunKind: "job" | "group" | "adhoc"; + RunNodeRecord: { + /** Format: date-time */ + ended_at?: string | null; + error_code?: string | null; + node_key: string; + /** Format: int64 */ + rows_affected?: number | null; + skipped_because_of?: string | null; + /** Format: date-time */ + started_at?: string | null; + status: string; + }; + /** @enum {string} */ + RunOrigin: "manual" | "schedule" | "sensor" | "rerun" | "backfill"; + /** + * @description Terminal outcomes cannot be confused with a request to change an active status. + * @enum {string} + */ + RunOutcome: "success" | "warn" | "partial" | "failed" | "cancelled"; + RunRecord: { + /** Format: date-time */ + ended_at?: string | null; + job_key?: string | null; + /** Format: int64 */ + last_event_storage_id: number; + /** Format: date-time */ + launched_at?: string | null; + /** Format: uuid */ + location_id: string; + node_counts_json: { + [key: string]: number; + }; + /** Format: date-time */ + queued_at: string; + /** Format: uuid */ + run_id: string; + selection_json: string[]; + /** Format: uuid */ + snapshot_id: string; + /** Format: date-time */ + started_at?: string | null; + status: components["schemas"]["RunStatus"]; + target: string; + }; + /** @description The authorized selection and pinned execution inputs are immutable once requested. */ + RunRequest: { + concurrency_groups: string[]; + image_id?: string | null; + job_key?: string | null; + kind: components["schemas"]["RunKind"]; + /** Format: uuid */ + location_id: string; + origin: components["schemas"]["RunOrigin"]; + /** Format: uuid */ + parent_run_id?: string | null; + /** Format: int64 */ + priority: number; + requested_by: string; + /** Format: uuid */ + rerun_of_run_id?: string | null; + /** Format: uuid */ + root_run_id: string; + /** Format: uuid */ + run_id: string; + selection: string[]; + settings: { + [key: string]: unknown; + }; + /** Format: uuid */ + snapshot_id: string; + target: string; + /** Format: uuid */ + tick_id?: string | null; + /** Format: uuid */ + trigger_id?: string | null; + window?: null | components["schemas"]["RunWindow"]; + write_set: string[]; + }; + /** @enum {string} */ + RunStatus: "queued" | "launched" | "running" | "unresponsive" | "success" | "warn" | "partial" | "failed" | "cancelled"; + /** @description Inclusive typed CLI cursor bounds; the native planner derives exclusive execution ends. */ + RunWindow: { + /** Format: date-time */ + end: string; + /** @enum {string} */ + kind: "timestamp"; + /** Format: date-time */ + start: string; + } | { + /** Format: int64 */ + end: number; + /** @enum {string} */ + kind: "integer"; + /** Format: int64 */ + start: number; + }; + Schedule: { + expression: string; + /** @enum {string} */ + kind: "cron"; + timezone: string; + } | { + /** @enum {string} */ + kind: "calendar"; + /** Format: date-time */ + refresh_at: string; + times: string[]; + }; + /** @description Durable security facts with an exclusive storage-order cursor. */ + SecurityAuditEvent: { + action: string; + actor: string; + after_json: unknown; + before_json: unknown; + /** Format: uuid */ + event_id: string; + granting_role: string; + ip_address?: string | null; + /** Format: date-time */ + occurred_at: string; + /** Format: uuid */ + request_id: string; + scope_json: unknown; + /** Format: int64 */ + storage_id: number; + }; + SensorCondition: { + /** @enum {string} */ + kind: "platform_event"; + kinds: components["schemas"]["PlatformEventKind"][]; + } | { + executed_successfully?: string[] | null; + job_key: string; + /** @enum {string} */ + kind: "run_event"; + kinds: components["schemas"]["RunEventKind"][]; + } | { + /** @enum {string} */ + kind: "node_event"; + kinds: components["schemas"]["NodeEventKind"][]; + selection: string[]; + } | { + /** @enum {string} */ + kind: "source_new_data"; + selection: string[]; + } | { + /** Format: int32 */ + after_seconds: number; + /** @enum {string} */ + kind: "source_stale"; + selection: string[]; + } | { + code_hash: string; + function: string; + /** @enum {string} */ + kind: "custom"; + }; + SensorDefinition: { + default_status?: components["schemas"]["TriggerStatus"]; + description: string; + job_key: string; + /** Format: int32 */ + minimum_interval_seconds: number; + when: components["schemas"]["SensorCondition"]; + }; + SessionIdentity: { + /** Format: date-time */ + expires_at: string; + /** Format: uuid */ + principal_id: string; + /** Format: uuid */ + session_id: string; + }; + /** @description Node-level changes do not duplicate unchanged definitions or source text. */ + SnapshotChange: { + after_hash?: string | null; + before_hash?: string | null; + change_kind: string; + subject_key: string; + }; + /** @description A location's successful compile for one target. */ + SnapshotInput: { + documents: { + [key: string]: unknown; + }; + environment_hash: string; + /** Format: int64 */ + generation: number; + git_author?: string | null; + git_dirty: boolean; + git_revision?: string | null; + image_id?: string | null; + /** Format: uuid */ + location_id: string; + nodes: { + [key: string]: components["schemas"]["SnapshotNode"]; + }; + sqb_version: string; + target: string; + }; + /** @description A compiled node and its display-only source. SQL and Python are never run by the server. */ + SnapshotNode: { + compiled_sql?: string | null; + definition: unknown; + kind: string; + python_source?: string | null; + raw_sql?: string | null; + }; + /** @description Immutable publication metadata. Generations increase independently per location and target. */ + SnapshotRecord: { + /** Format: int64 */ + generation: number; + /** Format: uuid */ + location_id: string; + manifest_digest: string; + /** Format: date-time */ + published_at: string; + /** Format: uuid */ + snapshot_id: string; + target: string; + }; + SourceDataStatus: { + reason: string; + /** @enum {string} */ + status: "unknown"; + }; + StartedLogin: { + browser_code: string; + /** Format: date-time */ + expires_at: string; + }; + /** @description Eligibility comes from each relation's committed hour, never a namespace watermark. */ + StatisticsDueRead: { + /** Format: uuid */ + after?: string | null; + /** Format: int64 */ + generation: number; + limit: number; + location: components["schemas"]["WarehouseStateLocation"]; + /** Format: uuid */ + session_id: string; + }; + StatisticsObservation: { + /** Format: uuid */ + event_id: string; + record: components["schemas"]["WarehouseStatisticsRecord"]; + }; + StatisticsPage: { + next_after?: null | components["schemas"]["WarehouseMirrorCursor"]; + records: components["schemas"]["StatisticsObservation"][]; + }; + /** @description One bounded catalog page; sampled and unsampled relations both advance its cursor. */ + StatisticsRead: { + location: components["schemas"]["WarehouseStateLocation"]; + names: string[]; + /** Format: uuid */ + next_after?: string | null; + }; + /** + * @description The tables Recall writes. + * @enum {string} + */ + Table: "invocations" | "output" | "events" | "diagnostics" | "artifacts" | "artifact_refs" | "heartbeats" | "delivery" | "meta" | "conflicts"; + /** @enum {string} */ + TickOutcome: "requested" | "covered"; + TickRecord: { + /** Format: uuid */ + covered_by_run_id?: string | null; + /** Format: int64 */ + epoch: number; + /** Format: date-time */ + evaluated_at: string; + outcome: components["schemas"]["TickOutcome"]; + /** Format: uuid */ + run_id?: string | null; + /** Format: date-time */ + scheduled_for: string; + /** Format: uuid */ + tick_id: string; + /** Format: uuid */ + trigger_id: string; + }; + /** @description Operator changes are immutable facts; clearing an override restores the published code default. */ + TriggerChange: { + /** @enum {string} */ + kind: "enable"; + } | { + /** @enum {string} */ + kind: "pause"; + } | { + /** @enum {string} */ + kind: "reset_default"; + } | { + /** @enum {string} */ + kind: "set_cursor"; + /** Format: date-time */ + time: string; + } | { + /** @enum {string} */ + kind: "reset_cursor"; + }; + TriggerChangeRecord: { + /** Format: date-time */ + occurred_at: string; + request_json: components["schemas"]["TriggerChangeRequest"]; + /** Format: uuid */ + snapshot_id: string; + }; + TriggerChangeRequest: { + actor: string; + change: components["schemas"]["TriggerChange"]; + /** Format: uuid */ + event_id: string; + reason: string; + /** Format: uuid */ + trigger_id: string; + }; + /** @description Legacy schedule documents retain their stored shape; sensors add a distinct condition shape. */ + TriggerDeclaration: components["schemas"]["TriggerDefinition"] | components["schemas"]["SensorDefinition"]; + TriggerDefinition: { + default_status?: components["schemas"]["TriggerStatus"]; + description: string; + job_key: string; + schedule: components["schemas"]["Schedule"]; + /** Format: date-time */ + starts_at?: string | null; + }; + TriggerRecord: { + definition_json: components["schemas"]["TriggerDeclaration"]; + job_key: string; + /** Format: date-time */ + last_tick_for?: string | null; + name: string; + /** Format: date-time */ + next_due_at?: string | null; + status: components["schemas"]["TriggerStatus"]; + /** Format: uuid */ + trigger_id: string; + }; + /** @enum {string} */ + TriggerStatus: "running" | "paused"; + WarehouseCatalogColumn: { + data_type: string; + name: string; + }; + /** @description Physical identity and ordered columns come from the adapter catalog, not fingerprints. */ + WarehouseCatalogRecord: { + columns: components["schemas"]["WarehouseCatalogColumn"][]; + name: string; + relation_type: string; + state_database?: string | null; + state_schema: string; + }; + /** + * @description Catalog observations require full scans; only immutable history supports a watermark. + * @enum {string} + */ + WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches" | "catalog" | "statistics"; + WarehouseMirrorCursor: { + /** Format: uuid */ + event_id: string; + /** Format: date-time */ + observed_at: string; + }; + /** @description Physical namespace of SQLBuild's source-freshness state table, never a source query. */ + WarehouseStateLocation: { + database?: string | null; + schema: string; + }; + /** @description A metadata estimate at collection time, never evidence of source-data currency. */ + WarehouseStatisticsRecord: { + /** Format: int64 */ + byte_count?: number | null; + byte_count_basis?: string | null; + detail?: string | null; + name: string; + /** Format: date-time */ + observed_at: string; + /** Format: int64 */ + row_count?: number | null; + state_database?: string | null; + state_schema: string; + }; + /** + * @description Closed failure stages keep warehouse errors and credentials out of health observations. + * @enum {string} + */ + WarehouseSyncFailure: "watermark" | "read" | "delivery"; + WarehouseSyncResponse: { + applied: boolean; + }; + /** @description Derive progress from committed mirror rows, scoped to the active physical reader. */ + WarehouseWatermarkRead: { + /** Format: int64 */ + generation: number; + kind: components["schemas"]["WarehouseHistoryKind"]; + location: components["schemas"]["WarehouseStateLocation"]; + /** Format: uuid */ + session_id: string; + }; + }; + responses: never; + parameters: never; + requestBodies: never; + headers: never; + pathItems: never; +} +export type $defs = Record; +export interface operations { + changeAdministrator: { + parameters: { + query?: never; + header: { + /** @description Session CSRF token */ + "x-sqb-csrf": string; + }; + path: { + principal_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["AdministratorChange"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + assignBuiltinRole: { + parameters: { + query?: never; + header: { + "x-sqb-csrf": string; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["RoleAssignmentRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + approveCliLogin: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CliLoginApprovalRequest"]; + }; + }; + responses: { + 204: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + }; + }; + claimCliLogin: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CliLoginClaimRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["CliLoginSessionResponse"]; + }; + }; + 202: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + }; + }; + startCliLogin: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CliLoginStartRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StartedLogin"]; + }; + }; + 429: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + }; + }; + logoutSession: { + parameters: { + query?: never; + header: { + /** @description Session CSRF token */ + "x-sqb-csrf": string; + }; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 204: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + }; + }; + oidcIdentity: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ExternalIdentityResponse"]; + }; + }; + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + }; + }; + loginOidc: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 303: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + 429: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + }; + }; + changePassword: { + parameters: { + query?: never; + header: { + /** @description Session CSRF token */ + "x-sqb-csrf": string; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["PasswordChangeRequest"]; + }; + }; + responses: { + 204: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + }; + }; + loginPassword: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["PasswordLoginRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["PasswordSessionResponse"]; + }; + }; + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + 429: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + }; + }; + currentSession: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SessionIdentity"]; + }; + }; + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthenticationDenial"]; + }; + }; + }; + }; + readSecurityAudit: { + parameters: { + query?: { + /** @description Exclusive storage-order cursor */ + after?: number; + limit?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SecurityAuditEvent"][]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + readPersonCatalog: { + parameters: { + query: { + /** @description Exact database; omitted matches SQL NULL */ + database?: string; + /** @description Exact physical schema */ + schema: string; + after_observed_at?: string; + after_event_id?: string; + /** @description Page size; defaults to 100, clamped to 1..100 */ + limit?: number; + }; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["CatalogPage"]; + }; + }; + /** @description Invalid namespace or cursor */ + 400: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + requestConnectionTest: { + parameters: { + query?: never; + header: { + "x-sqb-csrf": string; + }; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ConnectionTestRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + readConnectionTest: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + request_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": null | components["schemas"]["LocationWorkRecord"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + readPersonFingerprint: { + parameters: { + query: { + node_type: string; + node_name: string; + /** @description Exact destination database; omitted matches SQL NULL */ + database?: string; + /** @description Exact destination schema; omitted matches SQL NULL */ + schema?: string; + /** @description Exact destination name; omitted matches SQL NULL */ + name?: string; + }; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": null | components["schemas"]["FingerprintObservation"]; + }; + }; + /** @description Invalid node */ + 400: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + readPersonNodeStatus: { + parameters: { + query: { + /** @description One exact published node key */ + node_key: string; + }; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": null | components["schemas"]["NodeStatus"]; + }; + }; + /** @description Invalid node key */ + 400: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + readPersonNodeSql: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + node_key: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": null | components["schemas"]["NodeSql"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + listPersonRuns: { + parameters: { + query?: { + limit?: number; + }; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RunRecord"][]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + readPersonStatistics: { + parameters: { + query: { + /** @description Exact database; omitted matches SQL NULL */ + database?: string; + schema: string; + name: string; + after_observed_at?: string; + after_event_id?: string; + limit?: number; + }; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatisticsPage"]; + }; + }; + /** @description Invalid relation or cursor */ + 400: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + requestJobRun: { + parameters: { + query?: never; + header: { + "x-sqb-csrf": string; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["JobRunRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + cancelPersonRun: { + parameters: { + query?: never; + header: { + "x-sqb-csrf": string; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CancelRunRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + rerunPersonRun: { + parameters: { + query?: never; + header: { + "x-sqb-csrf": string; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["RerunRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + changeTrigger: { + parameters: { + query?: never; + header: { + "x-sqb-csrf": string; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["TriggerChangeRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + read_location_health: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["LocationHealthObservation"][]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + report_location_health: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["LocationHealthReport"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + subscribe: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description changed events: all, runs, run:, ticks, snapshot:, node_status:. Refetch REST on all/reconnect; no durable cursor. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "text/event-stream": components["schemas"]["ChangeNotice"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + report_metadata: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["MetadataBatch"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WarehouseSyncResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Reader generation fenced */ + 409: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + statistics_due: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["StatisticsDueRead"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatisticsRead"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Reader generation fenced */ + 409: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + watermark: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["WarehouseWatermarkRead"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": string | null; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Reader generation fenced */ + 409: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + list_runs: { + parameters: { + query?: { + /** @description Maximum recent runs; defaults to 100, capped at 1000 */ + limit?: number; + }; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RunRecord"][]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + invocations: { + parameters: { + query?: { + /** @description Maximum invocation namespaces; defaults to 100, capped at 1000 */ + limit?: number; + }; + header?: never; + path: { + location_id: string; + target: string; + run_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown[]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Run not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + nodes: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + run_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RunNodeRecord"][]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Run not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + read_run_recall: { + parameters: { + query: { + /** @description Exact scheduler or worker invocation ID */ + invocation: string; + /** @description Maximum rows; defaults to 100, capped at 1000 */ + limit?: number; + }; + header?: never; + path: { + location_id: string; + target: string; + run_id: string; + view: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown[]; + }; + }; + /** @description Unknown Recall view */ + 400: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Run or invocation not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + current: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["CurrentSnapshot"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description No successful snapshot */ + 404: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + publish: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["SnapshotInput"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SnapshotRecord"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Stale generation */ + 409: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + metadata: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SnapshotRecord"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description No successful snapshot */ + 404: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + list_triggers: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["TriggerRecord"][]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + events: { + parameters: { + query?: { + /** @description Maximum events; defaults to 100, capped at 1000 */ + limit?: number; + }; + header?: never; + path: { + location_id: string; + target: string; + trigger_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["TriggerChangeRecord"][]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Trigger not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + ticks: { + parameters: { + query?: { + /** @description Maximum recent ticks; defaults to 100, capped at 1000 */ + limit?: number; + }; + header?: never; + path: { + location_id: string; + target: string; + trigger_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["TickRecord"][]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Trigger not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + poll: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["LocationWorkPoll"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": null | components["schemas"]["LocationWorkClaim"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + result: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + request_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": null | components["schemas"]["LocationWorkRecord"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + complete: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + request_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["LocationWorkResult"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Work claim expired or was replaced */ + 409: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + report_run_event: { + parameters: { + query?: never; + header?: never; + path: { + run_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["RunChangeRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Run not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + execution: { + parameters: { + query?: never; + header?: never; + path: { + run_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RunExecution"]; + }; + }; + /** @description Inactive claim or invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Run not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + heartbeat: { + parameters: { + query?: never; + header?: never; + path: { + run_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HeartbeatResponse"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Run not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + output: { + parameters: { + query?: never; + header?: never; + path: { + run_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["OutputChunk"][]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["OutputPublication"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Run not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + recall: { + parameters: { + query?: never; + header?: never; + path: { + run_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["RecallBatch"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["InsertOutcome"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Run not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; +} diff --git a/clients/typescript/tests/client.test.ts b/clients/typescript/tests/client.test.ts new file mode 100644 index 0000000000..bd937b2883 --- /dev/null +++ b/clients/typescript/tests/client.test.ts @@ -0,0 +1,162 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { once } from "node:events"; +import { createServer, request as forwardRequest } from "node:http"; +import { test } from "node:test"; +import { createControlClient, type components } from "../src/client.ts"; + +test("generated client preserves native scopes, replay, path encoding, SSE and browser sessions", { timeout: 20000 }, async () => { + const directory = await mkdtemp(join(tmpdir(), "sqlbuild-client-")); + const executable = process.env.SQLBUILD_EXECUTABLE ?? "sqb"; + const env = { ...process.env }; + delete env.SQLBUILD_CONTROL_DATABASE_URL; + delete env.SQLBUILD_PLATFORM_WEBHOOK_URL; + const admin = (...args: string[]) => JSON.parse(execFileSync(executable, ["admin", ...args], { + cwd: directory, env, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], + })); + let baseUrl: string; + // Exercise exact-origin forwarding without asking fetch to override its forbidden Host header. + // This HTTP fixture checks the transport contract, not external TLS termination. + const proxy = createServer((request, response) => { + const forwarded = forwardRequest(`${baseUrl}${request.url}`, { + method: request.method, headers: request.headers, + }, (upstream) => { + response.writeHead(upstream.statusCode!, upstream.headers); + upstream.pipe(response); + }); + forwarded.on("error", () => { response.writeHead(502); response.end(); }); + request.pipe(forwarded); + }); + let server: ReturnType | undefined; + try { + proxy.listen(0, "127.0.0.1"); + await once(proxy, "listening"); + const address = proxy.address(); + assert.ok(address && typeof address !== "string"); + const browserUrl = `http://127.0.0.1:${address.port}`; + const origin = `https://127.0.0.1:${address.port}`; + const { location_id } = admin("create-location", "orders"); + const other = admin("create-location", "inventory").location_id; + const { bearer } = admin("issue-location-token", location_id, "--expires-at", + new Date(Date.now() + 60000).toISOString()); + const { principal_id } = admin("create-principal", "orders-reader"); + const password = " orders-password-café "; + execFileSync(executable, ["admin", "set-password", principal_id], { + cwd: directory, env, input: `${password}\n`, stdio: ["pipe", "pipe", "pipe"], + }); + server = spawn(executable, ["server", "--port", "0", "--no-scheduler", + "--password-origin", origin], { + cwd: directory, env, stdio: ["ignore", "ignore", "pipe"], + }); + const running = server; + // Readiness is emitted on stderr; never log the one-time credential response. + baseUrl = await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("API readiness timed out")), 5000); + let log = ""; + running.once("error", reject); + running.once("exit", () => { clearTimeout(timer); reject(new Error("API exited before readiness")); }); + running.stderr!.on("data", (chunk: Buffer) => { + log += chunk.toString(); + const address = /Control API ready: ([^\r\n]+)/.exec(log); + if (address) { clearTimeout(timer); resolve(`http://${address[1]}`); } + }); + }); + const client = createControlClient({ baseUrl, headers: { Authorization: `Bearer ${bearer}` } }); + const healthPath = "/api/locations/{location_id}/health"; + const params = { path: { location_id } }; + const body: components["schemas"]["LocationHealthReport"] = { + event_id: randomUUID(), session_id: randomUUID(), target: "preview #1", health: { kind: "heartbeat" }, + }; + const written = await client.POST(healthPath, { params, body }); + assert.equal(written.response.status, 200); + assert.deepEqual(written.data, { applied: true }); + const replay = await client.POST(healthPath, { params, body }); + assert.deepEqual(replay.data, { applied: false }); + const read = await client.GET(healthPath, { params }); + assert.equal(read.response.headers.get("cache-control"), "no-store"); + assert.deepEqual(read.data?.map((row) => row.report), [body]); + const forbidden = await client.GET(healthPath, { params: { path: { location_id: other } } }); + assert.equal(forbidden.response.status, 403); + assert.equal(forbidden.data, undefined); + const anonymous = await createControlClient({ baseUrl }).GET(healthPath, { params }); + assert.equal(anonymous.response.status, 401); + const runs = await client.GET("/api/locations/{location_id}/targets/{target}/runs", { + params: { path: { location_id, target: "preview #1" }, query: { limit: 1 } }, + }); + assert.equal(runs.response.status, 200); + assert.deepEqual(runs.data, []); + const changes = await client.GET("/api/locations/{location_id}/targets/{target}/changes", { + params: { path: { location_id, target: "preview #1" } }, parseAs: "stream", + signal: AbortSignal.timeout(3000), + }); + assert.equal(changes.response.headers.get("content-type"), "text/event-stream"); + const reader = changes.data!.pipeThrough(new TextDecoderStream()).getReader(); + try { + let event = ""; + while (!event.includes("\n\n")) { + const chunk = await reader.read(); + assert.equal(chunk.done, false); + event += chunk.value; + } + assert.match(event, /event: changed/); + assert.match(event, /data: \{"topic":"all"\}/); + } finally { + await reader.cancel(); + } + const browser = createControlClient({ baseUrl: browserUrl, headers: { Origin: origin } }); + const login = await browser.POST("/api/auth/password/login", { + body: { username: "orders-reader", password }, + }); + assert.equal(login.response.status, 200); + assert.equal(login.response.headers.get("cache-control"), "no-store"); + assert.equal(login.data?.identity.principal_id, principal_id); + assert.ok(login.data?.csrf_token); + assert.equal("bearer" in login.data, false); + const sessionCookie = login.response.headers.getSetCookie() + .find((cookie) => cookie.startsWith("__Host-sqb-session=")); + assert.ok(sessionCookie); + assert.match(sessionCookie, /; Secure;/); + assert.match(sessionCookie, /; HttpOnly$/); + const headers = { Cookie: sessionCookie.split(";")[0] }; + // The transport does not persist credentials or turn a person's session into location authority. + assert.equal((await browser.GET("/api/auth/session")).response.status, 401); + const session = await browser.GET("/api/auth/session", { headers }); + assert.deepEqual(session.data, login.data.identity); + const denied = await browser.GET("/api/control/locations/{location_id}/targets/{target}/runs", { + headers, params: { path: { location_id, target: "development" } }, + }); + assert.equal(denied.response.status, 403); + assert.equal(denied.response.headers.get("cache-control"), "no-store"); + const decisionId: string | null | undefined = denied.error?.decision_id; + assert.ok(decisionId); + assert.deepEqual(denied.error, { + error: `missing permission in this scope; all requested nodes denied; decision ${decisionId}`, + decision_id: decisionId, + }); + const machine = await browser.GET(healthPath, { headers, params }); + assert.equal(machine.response.status, 401); + const invalidLogout = await browser.POST("/api/auth/logout", { + headers, params: { header: { "x-sqb-csrf": "incorrect-csrf" } }, + }); + assert.equal(invalidLogout.response.status, 401); + assert.ok(invalidLogout.error?.request_id); + assert.equal((await browser.GET("/api/auth/session", { headers })).response.status, 200); + const logout = await browser.POST("/api/auth/logout", { + headers, params: { header: { "x-sqb-csrf": login.data.csrf_token } }, + }); + assert.equal(logout.response.status, 204); + assert.equal((await browser.GET("/api/auth/session", { headers })).response.status, 401); + } finally { + if (server && server.exitCode === null) { + const exited = once(server, "exit"); + server.kill("SIGTERM"); + await exited; + } + await new Promise((resolve) => proxy.close(() => resolve())); + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/clients/typescript/tests/types.ts b/clients/typescript/tests/types.ts new file mode 100644 index 0000000000..c064d84da8 --- /dev/null +++ b/clients/typescript/tests/types.ts @@ -0,0 +1,94 @@ +import { createControlClient, type components } from "../src/client.ts"; + +// Compile-only consumer contracts: these must fail if generated typing is lost. +const client = createControlClient({ baseUrl: "https://example.com" }); +// @ts-expect-error Unknown paths are not accepted. +void client.GET("/api/not-a-route"); +// @ts-expect-error The location path parameter is required. +void client.GET("/api/locations/{location_id}/health", {}); +// @ts-expect-error Only the native discriminated health variants are accepted. +const invalid: components["schemas"]["LocationHealth"] = { kind: "unknown" }; +void invalid; +const sync: components["schemas"]["LocationHealth"] = { + kind: "warehouse_sync", generation: 1, + location: { database: null, schema: "main" }, table: "fingerprints", + failure: "read", records_sent: 0, elapsed_millis: 43, +}; +void sync; +declare const observation: components["schemas"]["LocationHealthObservation"]; +if (observation.report.health.kind === "warehouse_sync") { + const records: number = observation.report.health.records_sent; + void records; +} +// @ts-expect-error Raw warehouse exceptions cannot be used as a health failure stage. +const invalidFailure: components["schemas"]["WarehouseSyncFailure"] = "warehouse exception text"; +void invalidFailure; +const auditHistory: components["schemas"]["WarehouseHistoryKind"] = "audit_results"; +void auditHistory; +void client.POST("/api/locations/{location_id}/targets/{target}/metadata", { + params: { path: { location_id: "synthetic-location", target: "development" } }, + body: { kind: "janitor_events", session_id: "synthetic-session", generation: 1, records: [] }, +}); +const catalogColumn: components["schemas"]["WarehouseCatalogColumn"] = { + name: "order_id", data_type: "INTEGER", +}; +void catalogColumn; +void client.POST("/api/locations/{location_id}/targets/{target}/metadata", { + params: { path: { location_id: "synthetic-location", target: "development" } }, + body: { + kind: "catalog", session_id: "synthetic-session", generation: 1, records: [], + scan: { scan_id: "synthetic-scan", location: { database: null, schema: "main" }, complete: true }, + }, +}); +// @ts-expect-error Every metadata page names the state table it mirrors. +const unnamedPage: components["schemas"]["MetadataBatch"] = { session_id: "synthetic-session", generation: 1, records: [] }; +void unnamedPage; +void client.GET("/api/control/locations/{location_id}/targets/{target}/catalog", { + params: { + path: { location_id: "synthetic-location", target: "development" }, + query: { schema: "main", after_observed_at: "2026-01-02T03:04:05Z", after_event_id: "synthetic-relation", limit: 100 }, + }, +}); +const catalogPage: components["schemas"]["CatalogPage"] = { records: [], next_after: null }; +void catalogPage; +// @ts-expect-error Catalog reads require an exact physical schema. +void client.GET("/api/control/locations/{location_id}/targets/{target}/catalog", { params: { path: { location_id: "synthetic-location", target: "development" } } }); +const sourceCondition: components["schemas"]["SensorCondition"] = { + kind: "source_new_data", selection: ["source:raw_orders"], +}; +void sourceCondition; +// @ts-expect-error Source conditions select sources, not execution event kinds. +const invalidSourceCondition: components["schemas"]["SensorCondition"] = { kind: "source_new_data", selection: ["source:raw_orders"], kinds: ["success"] }; +void invalidSourceCondition; +const staleCondition: components["schemas"]["SensorCondition"] = { + kind: "source_stale", selection: ["source:raw_orders"], after_seconds: 60, +}; +void staleCondition; +// @ts-expect-error Staleness conditions require an explicit age threshold. +const missingStaleThreshold: components["schemas"]["SensorCondition"] = { kind: "source_stale", selection: ["source:raw_orders"] }; +void missingStaleThreshold; +// @ts-expect-error Session mutations require the CSRF header. +void client.POST("/api/auth/logout", {}); +// @ts-expect-error Changing a password also requires the current password. +void client.POST("/api/auth/password", { params: { header: { "x-sqb-csrf": "synthetic-csrf" } }, body: { new_password: "synthetic-password" } }); +// @ts-expect-error Session responses never expose a browser-readable bearer. +void ({} as components["schemas"]["PasswordSessionResponse"]).bearer; +void client.GET("/api/control/locations/{location_id}/targets/{target}/fingerprint", { + params: { path: { location_id: "orders-location", target: "development" }, + query: { node_type: "model", node_name: "orders", schema: "main", name: "orders" } }, +}); +// @ts-expect-error Build evidence does not contain SQL or arbitrary metadata. +void ({} as components["schemas"]["FingerprintObservation"]).definition; +void client.GET("/api/control/locations/{location_id}/targets/{target}/node-status", { + params: { path: { location_id: "orders-location", target: "development" }, + query: { node_key: "model:orders" } }, +}); +const sourceData: components["schemas"]["SourceDataStatus"] = { + status: "unknown", reason: "consumed-input provenance is not recorded", +}; +void sourceData; +// @ts-expect-error Observation times cannot establish consumed-source currency. +const inferredCurrency: components["schemas"]["SourceDataStatus"] = { status: "behind", reason: "newer observation" }; +void inferredCurrency; +// @ts-expect-error Node status requires one exact published node. +void client.GET("/api/control/locations/{location_id}/targets/{target}/node-status", { params: { path: { location_id: "orders-location", target: "development" } } }); diff --git a/clients/typescript/tsconfig.json b/clients/typescript/tsconfig.json new file mode 100644 index 0000000000..2a2da4290f --- /dev/null +++ b/clients/typescript/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "noEmit": true, + "allowImportingTsExtensions": true, + "erasableSyntaxOnly": true, + "types": ["node"] + }, + "include": ["src/**/*.ts", "tests/**/*.ts"] +} diff --git a/compose.yml b/compose.yml new file mode 100644 index 0000000000..f9decab88a --- /dev/null +++ b/compose.yml @@ -0,0 +1,82 @@ +name: ${COMPOSE_PROJECT_NAME:-sqlbuild} + +services: + postgres: + image: postgres:17.6-alpine3.22 + environment: + POSTGRES_USER: sqlbuild + POSTGRES_DB: sqlbuild + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} + healthcheck: + test: [CMD-SHELL, "pg_isready -U sqlbuild -d sqlbuild"] + interval: 2s + timeout: 2s + retries: 30 + volumes: + - control:/var/lib/postgresql/data + networks: [control] + + server: + image: ${SQLBUILD_SERVER_IMAGE:-sqlbuild-server:local} + build: + context: . + dockerfile: deploy/Dockerfile + target: server + args: + PYTHON_VERSION: ${PYTHON_VERSION:-3.12} + environment: + - SQLBUILD_CONTROL_DATABASE_URL=${SQLBUILD_CONTROL_DATABASE_URL:?Set SQLBUILD_CONTROL_DATABASE_URL} + - SQLBUILD_PLATFORM_WEBHOOK_URL + group_add: ["${DOCKER_GID:?Set DOCKER_GID to the Docker socket group}"] + volumes: + - ${DOCKER_SOCKET:-/var/run/docker.sock}:/var/run/docker.sock + configs: + - source: launchers + target: /etc/sqlbuild/launchers.json + ports: + - "${SQLBUILD_BIND_ADDRESS:-127.0.0.1}:${SQLBUILD_PORT:-8000}:8000" + healthcheck: + test: [CMD, curl, --fail, --silent, --output, /dev/null, "http://localhost:8000/api/openapi.json"] + interval: 2s + timeout: 2s + retries: 30 + depends_on: + postgres: + condition: service_healthy + networks: [control, workers] + + location: + image: ${SQLBUILD_LOCATION_IMAGE:-sqlbuild-orders:local} + container_name: ${COMPOSE_PROJECT_NAME:-sqlbuild}-location + build: + context: . + dockerfile: deploy/Dockerfile + target: location + args: + PYTHON_VERSION: ${PYTHON_VERSION:-3.12} + SQLBUILD_GIT_REVISION: ${SQLBUILD_GIT_REVISION:?Set SQLBUILD_GIT_REVISION} + command: [location, serve, --server, "http://server:8000", --location-id, "${SQLBUILD_LOCATION_ID:?Set SQLBUILD_LOCATION_ID}", --target, development] + environment: + SQLBUILD_LOCATION_TOKEN: ${SQLBUILD_LOCATION_TOKEN:-} + volumes: + - warehouse:/warehouse + cpus: 2 + mem_limit: 1g + depends_on: + server: + condition: service_healthy + networks: [workers] + +configs: + launchers: + content: | + {"docker":[{"location_id":"${SQLBUILD_LOCATION_ID}","target":"development","server_url":"http://server:8000","max_workers":1,"socket":"/var/run/docker.sock","location_container":"${COMPOSE_PROJECT_NAME:-sqlbuild}-location","executable":"/usr/local/bin/sqb"}]} + +volumes: + control: + warehouse: + +networks: + control: + internal: true + workers: diff --git a/crates/README.md b/crates/README.md index 4aaac11815..7f3820572c 100644 --- a/crates/README.md +++ b/crates/README.md @@ -107,6 +107,125 @@ the goldens in `tests/goldens/compiler`, with stage captures and full discovery, analysis coverage required. CI also runs it on Python 3.13 and 3.14, and compiles and builds a playground project on Windows. +## Service credentials + +Principal API tokens require an active service account at issuance and on every authenticated +request. Person sessions are a separate credential mechanism. Existing personal API-token rows +remain in storage for history but cannot authenticate; disabling a service account also +invalidates its tokens across replicas without a process-local cache. Location and run-claim +credentials retain their separate scopes and lifecycle checks. + +## Individual node and check transitions + +`NodeEventKind.FAILED` matches entry into failure; repeated failures do not retrigger it. +`RECOVERED` matches an actual successful execution after a failure of that same node/check, +location and target, independently of its job or whole-run outcome. `SUCCESS` still matches +each genuine successful execution. A prior start in the same worker claim is required: +cached, blocked or unexecuted work cannot create a failure or claim a recovery. + +WARN is not failure. WARN, SKIPPED and CANCELLED do not clear an earlier actual failure; +cancellation remains in execution history but is not itself an active health problem. +Transitions are pure queries over immutable committed start/finish facts in storage order, +using the latest prior decisive execution, not a persisted mutable health flag. Cursor reset +recomputes the same transitions and existing run-key uniqueness prevents duplicate runs. +These consumers require individual committed node facts. Publishing Python checks into +worker snapshots and defining their frozen membership/retry behavior is a separate execution +boundary; the consumers do not infer check results from whole-build exit codes or diagnostic text. + +## Control-plane live updates + +`GET /api/locations/{location_id}/targets/{target}/changes` streams authenticated SSE +`changed` events containing only a `topic`. Location credentials retain their existing scope; +subscribers never receive another location's or target's invalidations. Topics are `runs`, +`run:`, `ticks`, `snapshot:` and `node_status:`. Clients refetch the +corresponding REST resource rather than deriving application state from a notification. + +Invalidations are deduplicated within the writing transaction. PostgreSQL publishes with +transactional `NOTIFY`, with each API replica holding a schema-isolated `LISTEN`; SQLite publishes +through an in-process broadcast immediately after successful COMMIT, even if the awaiting async +caller was cancelled. Rejected transactions publish nothing. Snapshot, run, trigger, sensor and +warehouse-ingestion owners mark their own changes without adding a second durable event log. + +The first event, subscriber lag and restored PostgreSQL subscriptions emit `all`, requiring a +complete authorized REST refetch. There is no durable SSE cursor or replay guarantee. Recoverable +listener failures retry without stopping HTTP serving; fatal configuration/protocol failures remain +errors. Every event and 15-second keepalive revalidates the credential against the store. Revocation +invalidates active streams across replicas; expired or inactive credentials disclose no subsequent +changes. Server shutdown closes live streams instead of waiting indefinitely for their bodies. + +## Local development supervisor + +`sqb dev` runs the native API, scheduler and reloadable code location together. Its default +store is `.sqlbuild/control.sqlite3` in the canonical project directory. `--control-store-url` +selects another control store, `--target` overrides the configured target, `--port 0` chooses a +free port, and `--no-scheduler` leaves publication and reads active without launching runs. +This backend-only command does not serve a UI. + +Authentication is disabled only for the explicit development listener on loopback. Host and Origin +checks reject browser cross-origin and DNS-rebinding requests; ordinary `sqb server` authentication +is unchanged. Development retains an opaque location identity per canonical project and target. +The spawned code location receives no control-store credentials and uses the ordinary HTTP +publication, reload and health paths; workers retain their ordinary run-scoped credentials. + +Triggers start paused before scheduling can begin. Publication pauses newly introduced or revived +triggers in the same transaction as the snapshot, using ordinary audited operator facts rather +than rewriting code definitions. Reload preserves explicit operator changes to retained triggers. +`--triggers` skips these automatic pauses; it does not undo an operator's persisted pause. +SIGINT or SIGTERM shuts down the scheduler, API streams and location interpreter together. + +## Warehouse event identity + +Fingerprint and source-freshness writers persist deterministic `event_id` values in nullable text +columns. Existing tables evolve additively; no predecessor rows are rewritten. Latest-state readers +retain their existing projections. IDs are lowercase SHA-256 hex over compact UTF-8 JSON arrays, +whose fixed namespace is independent of the package version: + +| Table | Array fields, in order | +|---|---| +| `_sqlbuild_fingerprints` | `"sqlbuild.fingerprint.event.v1"`, node kind, node name, target database, target schema, target name, run ID, version hash | +| `_sqlbuild_source_freshness` | `"sqlbuild.source-freshness.event.v1"`, source name, target database, target schema, target name, run ID, observation time, strategy, value kind, value hash | + +Nullable identity fields remain JSON null, distinct from empty strings. Freshness observation time +is normalized to UTC ISO text with six fractional digits and a `+00:00` offset. Retrying the same +observation retains its ID; another run or observation is distinct. Definitions, display values, +and future storage columns do not alter these identities. + +The location history reader pages fingerprint and source-freshness observations without selecting +only the latest row. It is read-only, including local database connections: it neither creates +databases nor evolves tables. A target can select `catalog_connection = "catalog"` from named +connections; project and local settings for that name resolve independently, with normal secret +expansion and adapter defaults. Project, local and target writer credentials are never overlaid or +used to fill missing catalog credentials. Without the override, reads use the target's normal +connection. Local target overrides select the catalog name through the normal target precedence; +unknown names and blank or non-string selections are configuration errors. +Fingerprint definitions and metadata use the existing storage decoder; +raw storage values remain the keyset cursor, separate from decoded UTC timestamps and text. +Fingerprint and source-freshness ingestion preserve warehouse IDs, scoped by location, target and +physical state table. Only missing or NULL IDs use fixed legacy fields, never a hash of an evolving +full row. Bounded pages of at most 100 observations commit atomically under the current location +session and snapshot generation, and repeated pages are idempotent. Fingerprint legacy IDs are +SHA-256 over a compact JSON array: `"sqlbuild.fingerprint.legacy.v1"`, node kind/name, target +database/schema/name, run ID, definition hash, version hash, schema fingerprint, exact definition, +metadata JSON text and UTC observation time (six fractional digits, `+00:00`). + +Fingerprint events retain their warehouse `definition_hash` unchanged. This is a semantic token +hash: SQL with different whitespace, comments or keyword case can share it. The separate +`definition_text_hash` references the existing immutable `sqb_source_texts` store, keyed by SHA-256 +of the exact UTF-8 bytes and using its existing compressed encoding. Identical bytes share a body; +semantic-hash-equivalent SQL variants remain lossless. Event JSON omits the definition body, and +the event and its referenced body commit or roll back together. Rebuilding the mirror by replaying +history preserves source identities and definitions; no warehouse mutation is involved. + +A history read can hold an +inclusive observation-time lower bound fixed across pages, independently of the row-key cursor. +SQLite's SQL prefilter includes the boundary day and its predecessor so legacy separators, variable +fractional precision and writer offsets crossing a UTC date remain readable; decoded UTC timestamps +enforce the exact bound. Other adapters prefilter at whole-second precision. Continuation follows +the raw row count: even an empty decoded page can have more rows to inspect. Ingestion and per-hash +definition storage are separate from this read-only transport. Automatic polling, +watermark/overlap management and deletion reconciliation are separate location-sync responsibilities, +not implemented by the page reader. + ## Working on the crates - Tests live next to the module they cover; run them for one crate with diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs index 791dcc6822..59e8a2328c 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs @@ -12,8 +12,9 @@ pub const COLUMN_SCHEMA_FINGERPRINT: &str = "schema_fingerprint"; pub const COLUMN_DEFINITION_B64: &str = "definition_b64"; pub const COLUMN_METADATA_JSON_B64: &str = "metadata_json_b64"; pub const COLUMN_TIMESTAMP: &str = "ts"; +pub const COLUMN_EVENT_ID: &str = "event_id"; -/// Every stored column, in table order. +/// Legacy columns used by latest-row readers, including readers of tables without an event ID. pub const FINGERPRINT_COLUMNS: [&str; 12] = [ COLUMN_NODE_TYPE, COLUMN_NODE_NAME, diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/_event_id.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/_event_id.rs new file mode 100644 index 0000000000..6ef1ccee43 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/_event_id.rs @@ -0,0 +1,20 @@ +//! Fixed writer identity excludes timestamps, definitions and future storage columns. + +use sha2::{Digest, Sha256}; + +use crate::fingerprint_storage::models::StoredFingerprint; + +/// Stable SHA-256 of the versioned JSON tuple named by the fingerprint storage contract. +pub(super) fn event_id(row: &StoredFingerprint) -> String { + let identity = serde_json::json!([ + "sqlbuild.fingerprint.event.v1", + row.node_type, + row.node_name, + row.target_database, + row.target_schema, + row.target_name, + row.run_id, + row.version_hash, + ]); + format!("{:x}", Sha256::digest(identity.to_string().as_bytes())) +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs index 7c0a00587e..db3a126811 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs @@ -1,9 +1,10 @@ //! Fingerprint table DDL. use crate::fingerprint_storage::constants::{ - COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_METADATA_JSON_B64, COLUMN_NODE_NAME, - COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_SCHEMA_FINGERPRINT, COLUMN_TARGET_DATABASE, - COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, COLUMN_VERSION_HASH, + COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_EVENT_ID, COLUMN_METADATA_JSON_B64, + COLUMN_NODE_NAME, COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_SCHEMA_FINGERPRINT, + COLUMN_TARGET_DATABASE, COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, + COLUMN_VERSION_HASH, }; /// `CREATE TABLE IF NOT EXISTS` for the fingerprint table, with the adapter's column types. @@ -21,6 +22,7 @@ pub fn create_table_sql(qualified_name: &str, string_type: &str, timestamp_type: {COLUMN_SCHEMA_FINGERPRINT} {string_type} NOT NULL, \ {COLUMN_DEFINITION_B64} {string_type} NOT NULL, \ {COLUMN_METADATA_JSON_B64} {string_type} NOT NULL, \ - {COLUMN_TIMESTAMP} {timestamp_type} NOT NULL)" + {COLUMN_TIMESTAMP} {timestamp_type} NOT NULL, \ + {COLUMN_EVENT_ID} {string_type})" ) } diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs index 25e54bf696..dbd785c797 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs @@ -3,7 +3,9 @@ use crate::fingerprint_storage::_helpers::base64::encode; use crate::fingerprint_storage::_helpers::literals::{optional_string_literal, string_literal}; use crate::fingerprint_storage::_helpers::table_columns::column_list; +use crate::fingerprint_storage::constants::COLUMN_EVENT_ID; use crate::fingerprint_storage::errors::FingerprintStorageError; +use crate::fingerprint_storage::main::_event_id::event_id; use crate::fingerprint_storage::models::StoredFingerprint; /// One multi-row `INSERT` for a non-empty batch. @@ -16,14 +18,14 @@ pub fn insert_sql( } let rows: Vec = fingerprints.iter().map(row_values).collect(); Ok(format!( - "INSERT INTO {qualified_name} ({}) VALUES {}", + "INSERT INTO {qualified_name} ({}, {COLUMN_EVENT_ID}) VALUES {}", column_list(), rows.join(", ") )) } fn row_values(fingerprint: &StoredFingerprint) -> String { - let values: [String; 12] = [ + let values: [String; 13] = [ string_literal(&fingerprint.node_type), string_literal(&fingerprint.node_name), optional_string_literal(fingerprint.target_database.as_deref()), @@ -36,6 +38,7 @@ fn row_values(fingerprint: &StoredFingerprint) -> String { format!("'{}'", encode(fingerprint.definition.as_bytes())), format!("'{}'", encode(fingerprint.metadata_json.as_bytes())), format!("'{}'", fingerprint.ts), + string_literal(&event_id(fingerprint)), ]; format!("({})", values.join(", ")) } diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs index 54c1f22e6f..ab782386e7 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs @@ -1,3 +1,4 @@ +mod _event_id; pub mod create_table_sql; pub mod decode_storage; pub mod insert_batch_lengths; diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs index 59c72c1dd6..5595b45108 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs @@ -142,8 +142,9 @@ fn given_rows_when_inserting_then_literals_are_escaped_and_definitions_encoded() expected_sql: Ok( "INSERT INTO main.t (node_type, node_name, target_database, target_schema, \ target_name, run_id, definition_hash, version_hash, schema_fingerprint, \ - definition_b64, metadata_json_b64, ts) VALUES ('model', 'o''rders', NULL, \ - 'main', 'orders', 'run', 'd', '', 's', 'YWI=', 'e30=', '2026-01-01T00:00:00')", + definition_b64, metadata_json_b64, ts, event_id) VALUES ('model', 'o''rders', NULL, \ + 'main', 'orders', 'run', 'd', '', 's', 'YWI=', 'e30=', '2026-01-01T00:00:00', \ + 'b9397954c803e7f99f6a233c2985be772c7ddf2c6f234197d77bc227ae175cbd')", ), }, InsertTestCase { diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/function_hashes.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/function_hashes.rs index d3d966b00b..21ed40cb98 100644 --- a/crates/sqlbuild-analysis/src/planner_changes/_helpers/function_hashes.rs +++ b/crates/sqlbuild-analysis/src/planner_changes/_helpers/function_hashes.rs @@ -2,9 +2,11 @@ use serde_json::Value; -use crate::planner_changes::_helpers::json_payload::{decoded_object, python_equal}; -use crate::planner_changes::constants::LOCAL_FUNCTION_HASHES_METADATA_KEY; use crate::planner_changes::models::IdentityPayload; +use sqlbuild_core::fingerprint_identity::constants::LOCAL_FUNCTION_HASHES_METADATA_KEY; +use sqlbuild_core::fingerprint_identity::main::{ + decoded_object::decoded_object, python_equal::python_equal, +}; /// Python equality of two identity payloads. pub(crate) fn payloads_equal(left: &IdentityPayload, right: &IdentityPayload) -> bool { diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/json_payload.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/json_payload.rs deleted file mode 100644 index 6b7e1ec92b..0000000000 --- a/crates/sqlbuild-analysis/src/planner_changes/_helpers/json_payload.rs +++ /dev/null @@ -1,80 +0,0 @@ -//! Decode fingerprint metadata as Python's `json.loads` does and write it as `json.dumps(sort_keys=True, separators=(",", ":"))`. - -use serde_json::{Map, Value}; -use sqlbuild_core::json::main::dumps::dumps; -use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; - -/// The decoded top-level object, or None when the text is not a JSON object. -pub(crate) fn decoded_object(metadata_json: &str) -> Result>, ()> { - match serde_json::from_str::(metadata_json) { - Ok(Value::Object(object)) => Ok(Some(object)), - Ok(_) => Ok(None), - Err(_) => Err(()), - } -} - -/// `json.dumps(object, sort_keys=True, separators=(",", ":"), default=str)`. -pub(crate) fn canonical_text(object: Map) -> String { - let dialect: JsonDialect = JsonDialect::Stdlib(StdlibJsonOptions { - sort_keys: true, - ..StdlibJsonOptions::new(None).with_separators(",", ":") - }); - dumps(&json_value(Value::Object(object)), &dialect).unwrap_or_default() -} - -/// Python equality of two decoded JSON values: numbers by value and booleans as 0 and 1. -pub(crate) fn python_equal(left: &Value, right: &Value) -> bool { - match (left, right) { - (Value::Array(left), Value::Array(right)) => { - left.len() == right.len() - && left - .iter() - .zip(right) - .all(|(left, right)| python_equal(left, right)) - } - (Value::Object(left), Value::Object(right)) => { - left.len() == right.len() - && left.iter().all(|(key, value)| { - right - .get(key) - .is_some_and(|other| python_equal(value, other)) - }) - } - (Value::Number(left_number), Value::Number(right_number)) - if !left_number.is_f64() && !right_number.is_f64() => - { - left_number == right_number - } - (Value::Number(_) | Value::Bool(_), Value::Number(_) | Value::Bool(_)) => { - numeric(left) == numeric(right) - } - _ => left == right, - } -} - -fn numeric(value: &Value) -> Option { - match value { - Value::Bool(flag) => Some(f64::from(u8::from(*flag))), - Value::Number(number) => number.as_f64(), - _ => None, - } -} - -fn json_value(value: Value) -> JsonValue { - match value { - Value::Null => JsonValue::Null, - Value::Bool(flag) => JsonValue::Bool(flag), - Value::Number(number) => match JsonInteger::parse(&number.to_string()) { - Some(integer) if !number.is_f64() => JsonValue::Integer(integer), - _ => JsonValue::Float(number.as_f64().unwrap_or_default()), - }, - Value::String(text) => JsonValue::String(text), - Value::Array(items) => JsonValue::Array(items.into_iter().map(json_value).collect()), - Value::Object(object) => JsonValue::Object( - object - .into_iter() - .map(|(key, value)| (key, json_value(value))) - .collect(), - ), - } -} diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/mod.rs index f344a81462..093209583e 100644 --- a/crates/sqlbuild-analysis/src/planner_changes/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_changes/_helpers/mod.rs @@ -1,5 +1,4 @@ pub(crate) mod function_hashes; -pub(crate) mod json_payload; pub(crate) mod pick_more_aggressive; pub(crate) mod rename_mappings; pub(crate) mod schema_comparer; diff --git a/crates/sqlbuild-analysis/src/planner_changes/_helpers/with_origin_cursor_input_names.rs b/crates/sqlbuild-analysis/src/planner_changes/_helpers/with_origin_cursor_input_names.rs index 77bd0cc2bc..ba76751400 100644 --- a/crates/sqlbuild-analysis/src/planner_changes/_helpers/with_origin_cursor_input_names.rs +++ b/crates/sqlbuild-analysis/src/planner_changes/_helpers/with_origin_cursor_input_names.rs @@ -2,8 +2,10 @@ use serde_json::{Map, Value}; -use crate::planner_changes::_helpers::json_payload::{canonical_text, decoded_object}; use crate::planner_changes::constants::{CONFIG_METADATA_KEY, CURSOR_INPUTS_CONFIG_KEY}; +use sqlbuild_core::fingerprint_identity::main::{ + canonical_text::canonical_text, decoded_object::decoded_object, +}; /// Canonical metadata JSON whose `config.cursor_inputs` keys use origin names; other text is returned unchanged. pub(crate) fn with_origin_cursor_input_names( diff --git a/crates/sqlbuild-analysis/src/planner_changes/constants.rs b/crates/sqlbuild-analysis/src/planner_changes/constants.rs index 0ff28f52b8..5bd2eae293 100644 --- a/crates/sqlbuild-analysis/src/planner_changes/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_changes/constants.rs @@ -1,13 +1,5 @@ //! Planner change-detection constants shared with the Python planner. -/// Fingerprint metadata key of the recorded audit gate. -pub const AUDIT_GATE_METADATA_KEY: &str = "audit_gate"; -/// Fingerprint metadata key of the migration fingerprint. -pub const MIGRATION_FINGERPRINT_METADATA_KEY: &str = "migration_fingerprint"; -/// Fingerprint metadata key of the declared-columns hash. -pub const DECLARED_COLUMNS_METADATA_KEY: &str = "declared_columns_hash"; -/// Fingerprint metadata key of directly called function hashes. -pub const LOCAL_FUNCTION_HASHES_METADATA_KEY: &str = "local_function_hashes"; /// Fingerprint metadata key of the model config. pub const CONFIG_METADATA_KEY: &str = "config"; /// Model config key of cursor inputs. diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/recorded_declared_columns_hash.rs b/crates/sqlbuild-analysis/src/planner_changes/main/recorded_declared_columns_hash.rs index 6a3fa0c963..44c9a86876 100644 --- a/crates/sqlbuild-analysis/src/planner_changes/main/recorded_declared_columns_hash.rs +++ b/crates/sqlbuild-analysis/src/planner_changes/main/recorded_declared_columns_hash.rs @@ -2,8 +2,8 @@ use serde_json::Value; -use crate::planner_changes::_helpers::json_payload::decoded_object; -use crate::planner_changes::constants::DECLARED_COLUMNS_METADATA_KEY; +use sqlbuild_core::fingerprint_identity::constants::DECLARED_COLUMNS_METADATA_KEY; +use sqlbuild_core::fingerprint_identity::main::decoded_object::decoded_object; /// The recorded declared-columns hash, when the metadata holds one as a string. pub fn recorded_declared_columns_hash(metadata_json: Option<&str>) -> Option { diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs b/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs index 9bceac24a7..62976bbb3b 100644 --- a/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs +++ b/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs @@ -1,11 +1,8 @@ //! The fingerprint metadata that participates in version identity. -use crate::planner_changes::_helpers::json_payload::decoded_object; -use crate::planner_changes::constants::{ - AUDIT_GATE_METADATA_KEY, DECLARED_COLUMNS_METADATA_KEY, LOCAL_FUNCTION_HASHES_METADATA_KEY, - MIGRATION_FINGERPRINT_METADATA_KEY, -}; use crate::planner_changes::models::IdentityPayload; +use sqlbuild_core::fingerprint_identity::main::decoded_object::decoded_object; +use sqlbuild_core::fingerprint_identity::main::normalize_version_identity_metadata::normalize_version_identity_metadata; /// The metadata without audit-gate, migration and declared-column entries; with `without_functions`, also without function hashes. pub fn version_identity_payload( @@ -16,19 +13,10 @@ pub fn version_identity_payload( return IdentityPayload::Absent; }; match decoded_object(text) { - Err(()) => IdentityPayload::Raw(text.to_owned()), + Err(_) => IdentityPayload::Raw(text.to_owned()), Ok(None) => IdentityPayload::Absent, Ok(Some(mut object)) => { - for key in [ - AUDIT_GATE_METADATA_KEY, - MIGRATION_FINGERPRINT_METADATA_KEY, - DECLARED_COLUMNS_METADATA_KEY, - ] { - object.shift_remove(key); - } - if without_functions { - object.shift_remove(LOCAL_FUNCTION_HASHES_METADATA_KEY); - } + normalize_version_identity_metadata(&mut object, without_functions); IdentityPayload::Object(object) } } diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/with_declared_columns_hash.rs b/crates/sqlbuild-analysis/src/planner_changes/main/with_declared_columns_hash.rs index c3508647a9..db131c7302 100644 --- a/crates/sqlbuild-analysis/src/planner_changes/main/with_declared_columns_hash.rs +++ b/crates/sqlbuild-analysis/src/planner_changes/main/with_declared_columns_hash.rs @@ -2,8 +2,10 @@ use serde_json::Value; -use crate::planner_changes::_helpers::json_payload::{canonical_text, decoded_object}; -use crate::planner_changes::constants::DECLARED_COLUMNS_METADATA_KEY; +use sqlbuild_core::fingerprint_identity::constants::DECLARED_COLUMNS_METADATA_KEY; +use sqlbuild_core::fingerprint_identity::main::{ + canonical_text::canonical_text, decoded_object::decoded_object, +}; /// Canonical metadata JSON that also records `declared_columns_hash`; text that is not an object is returned unchanged. pub fn with_declared_columns_hash(metadata_json: &str, declared_columns_hash: &str) -> String { diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/without_declared_columns_hash.rs b/crates/sqlbuild-analysis/src/planner_changes/main/without_declared_columns_hash.rs index a01da9c91e..01f2e9f785 100644 --- a/crates/sqlbuild-analysis/src/planner_changes/main/without_declared_columns_hash.rs +++ b/crates/sqlbuild-analysis/src/planner_changes/main/without_declared_columns_hash.rs @@ -1,7 +1,9 @@ //! Drop the declared-columns hash from fingerprint metadata. -use crate::planner_changes::_helpers::json_payload::{canonical_text, decoded_object}; -use crate::planner_changes::constants::DECLARED_COLUMNS_METADATA_KEY; +use sqlbuild_core::fingerprint_identity::constants::DECLARED_COLUMNS_METADATA_KEY; +use sqlbuild_core::fingerprint_identity::main::{ + canonical_text::canonical_text, decoded_object::decoded_object, +}; /// Canonical metadata JSON without the declared-columns hash, or the text unchanged when it has none. pub fn without_declared_columns_hash(metadata_json: &str) -> String { diff --git a/crates/sqlbuild-analysis/src/planner_identity/_helpers/compose_graph_identity.rs b/crates/sqlbuild-analysis/src/planner_identity/_helpers/compose_graph_identity.rs index a8e54ec089..9850656bfc 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/_helpers/compose_graph_identity.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/_helpers/compose_graph_identity.rs @@ -1,6 +1,6 @@ //! A node identity composed from its local hash and the hashes of its available upstreams. -use crate::planner_identity::main::stable_identity_hash::stable_identity_hash; +use sqlbuild_core::fingerprint_identity::main::stable_identity_hash::stable_identity_hash; /// The hash of the local hash and each upstream hash, one per line. pub(crate) fn compose_graph_identity(local_hash: &str, upstream_hashes: &[&str]) -> String { diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/mod.rs b/crates/sqlbuild-analysis/src/planner_identity/main/mod.rs index 94c535cd51..a7d42cd6e9 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/main/mod.rs @@ -5,7 +5,5 @@ pub mod function_fingerprint_sql; pub mod graph_write_identity_hashes; pub mod missing_upstream_message; pub mod missing_upstreams; -pub mod model_local_identity_hash; -pub mod stable_identity_hash; pub mod stale_upstream_warning_message; pub mod topological_order; diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs index 4ce46945f6..6426f00d87 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs @@ -7,8 +7,6 @@ use crate::planner_identity::main::function_fingerprint_sql::function_fingerprin use crate::planner_identity::main::graph_write_identity_hashes::graph_write_identity_hashes; use crate::planner_identity::main::missing_upstream_message::missing_upstream_message; use crate::planner_identity::main::missing_upstreams::missing_upstreams; -use crate::planner_identity::main::model_local_identity_hash::model_local_identity_hash; -use crate::planner_identity::main::stable_identity_hash::stable_identity_hash; use crate::planner_identity::main::stale_upstream_warning_message::stale_upstream_warning_message; use crate::planner_identity::main::topological_order::topological_order; use crate::planner_identity::models::{ @@ -25,6 +23,8 @@ use crate::planner_identity::tests::test_types::{ WarningTextTestCase, }; use crate::planner_identity::types::IdentityHashes; +use sqlbuild_core::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; +use sqlbuild_core::fingerprint_identity::main::stable_identity_hash::stable_identity_hash; #[test] fn given_persisted_identity_inputs_when_hashing_then_matches_pinned_python_hashes() { diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/stored_migration_fingerprint.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/stored_migration_fingerprint.rs index c6f07fd0d5..34add163f5 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/main/stored_migration_fingerprint.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/stored_migration_fingerprint.rs @@ -1,6 +1,6 @@ //! Read persisted migration hashes without changing their representation. -use crate::planner_changes::constants::MIGRATION_FINGERPRINT_METADATA_KEY; +use sqlbuild_core::fingerprint_identity::constants::MIGRATION_FINGERPRINT_METADATA_KEY; /// Read only string migration fingerprints from object metadata. pub fn stored_migration_fingerprint(metadata: &str) -> Option { diff --git a/crates/sqlbuild-cache/src/stage_capture/constants.rs b/crates/sqlbuild-cache/src/stage_capture/constants.rs index 816052a6f1..8b773cc563 100644 --- a/crates/sqlbuild-cache/src/stage_capture/constants.rs +++ b/crates/sqlbuild-cache/src/stage_capture/constants.rs @@ -255,6 +255,7 @@ pub const PROJECTED_ATTRIBUTES: [(&str, &[&str]); 15] = [ "external_sql_reference_resolver", "scope_index", "sql_expansions", + "python_execution_inputs", ], ), ( diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index 4a2253fb56..1f24eb892c 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -17,7 +17,11 @@ path = "src/bin/sqlbuild.rs" [dependencies] blake3.workspace = true +chrono.workspace = true getrandom.workspace = true +keyring = { workspace = true, features = ["apple-native", "windows-native", "sync-secret-service", "crypto-rust", "vendored"] } +reqwest.workspace = true +rpassword.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["arbitrary_precision"] } sqlbuild-build-features.workspace = true @@ -26,15 +30,24 @@ sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-cursor.workspace = true sqlbuild-model-config.workspace = true +sqlbuild-orchestration.workspace = true +sqlbuild-recall.workspace = true sqlbuild-rules.workspace = true +tokio.workspace = true +uuid.workspace = true [target.'cfg(unix)'.dependencies] rustix.workspace = true +[target.'cfg(target_os = "linux")'.dependencies] +command-fds.workspace = true + [target.'cfg(windows)'.dependencies] ctrlc.workspace = true +same-file.workspace = true [dev-dependencies] +rusqlite.workspace = true tempfile.workspace = true [lints] diff --git a/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs b/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs new file mode 100644 index 0000000000..19bb1563e3 --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs @@ -0,0 +1,128 @@ +//! Render one authored setting and check emitted options against the build parser. + +use std::collections::BTreeMap; + +use serde_json::Value as Json; + +use crate::argv::_helpers::table::root_parser; +use crate::argv::constants::CONCURRENCY_DEST; +use crate::argv::models::{ActionKind, ActionSpec, Default, ParserSpec, Value}; + +pub(crate) fn setting_argument( + parser: &ParserSpec, + name: &str, + value: &Json, +) -> Result, String> { + if matches!( + name, + "help" + | "target" + | "project_dir" + | "select" + | "exclude" + | "select_file" + | "positional_select" + | "json" + | "json_output" + | "event_output" + | "manifest" + | "dbt_project_dir" + | "dbt_profiles_dir" + | "dbt_target" + | "dbt_target_path" + ) { + return Err(format!("job setting {name} is owned by the run request")); + } + let actions: Vec<&ActionSpec> = parser + .actions + .iter() + .filter(|action| action.dest.as_deref() == Some(name)) + .collect(); + let Some(action) = actions.first() else { + return Err(format!("unknown build setting {name}")); + }; + if matches!(action.kind, ActionKind::StoreTrue | ActionKind::StoreFalse) { + return boolean_argument(name, value, &actions); + } + let option = &action.option_strings[0]; + let text = setting_text(name, value, action)?; + if action + .choices + .as_ref() + .is_some_and(|choices| !choices.contains(&text)) + { + return Err(format!("job setting {name} is not an accepted choice")); + } + Ok(Some(format!("{option}={text}"))) +} + +fn boolean_argument( + name: &str, + value: &Json, + actions: &[&ActionSpec], +) -> Result, String> { + let action = actions[0]; + let flag = value + .as_bool() + .ok_or_else(|| format!("job setting {name} must be boolean"))?; + if let Some(chosen) = actions + .iter() + .find(|action| action.constant == Value::Bool(flag)) + { + return Ok(Some(chosen.option_strings[0].clone())); + } + if action.default != Default::Value(Value::Bool(flag)) + && !(action.default == Default::Suppress + && root_parser().actions.iter().any(|global| { + global.dest.as_deref() == Some(name) + && global.default == Default::Value(Value::Bool(flag)) + })) + { + return Err(format!("job setting {name} cannot be set to {flag}")); + } + Ok(None) +} + +fn setting_text(name: &str, value: &Json, action: &ActionSpec) -> Result { + match action.converter.as_deref() { + Some("int" | "_positive_integer") => { + let integer = value + .as_i64() + .ok_or_else(|| format!("job setting {name} must be an integer"))?; + if integer < 1 + && (name == CONCURRENCY_DEST + || action.converter.as_deref() == Some("_positive_integer")) + { + return Err(format!("job setting {name} must be positive")); + } + Ok(integer.to_string()) + } + Some("parse_cli_vars") => { + if !value.is_object() { + return Err(format!("job setting {name} must be a JSON object")); + } + Ok(value.to_string()) + } + _ => value + .as_str() + .ok_or_else(|| format!("job setting {name} must be text")) + .map(str::to_owned), + } +} + +pub(crate) fn validate_mutex(parser: &ParserSpec, arguments: &[String]) -> Result<(), String> { + let mut mutex_groups: BTreeMap = BTreeMap::new(); + for argument in arguments { + let option = argument + .split_once('=') + .map_or(argument.as_str(), |(option, _)| option); + if let Some(group) = parser.actions[parser.option_index[option]].mutex_group + && let Some(previous) = mutex_groups.insert(group, option) + { + return Err(format!( + "job settings {previous} and {option} are mutually exclusive" + )); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/argv/_helpers/mod.rs b/crates/sqlbuild-cli/src/argv/_helpers/mod.rs index 22763ff51e..1d2d932e20 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/mod.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod dbt_execution; pub(crate) mod formatting; +pub(crate) mod job_settings; pub(crate) mod namespace; pub(crate) mod normalization; pub(crate) mod parsing; diff --git a/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs b/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs index 311d9ef353..b53bf9bf04 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs @@ -77,24 +77,10 @@ impl Run<'_> { .map_err(|message| self.error(parser, &message))?; namespace.set("concurrency", concurrency); } - if CURSOR_COMMANDS.contains(&command) { - let untyped: bool = ["start_cursor", "end_cursor"] - .iter() - .any(|name| namespace.value(name) != Value::None); - let typed: bool = [ - "start_cursor_ts", - "end_cursor_ts", - "start_cursor_int", - "end_cursor_int", - ] - .iter() - .any(|name| namespace.value(name) != Value::None); - if untyped && typed { - return Err(self.error( - parser, - "--start-cursor/--end-cursor cannot be combined with the -ts or -int cursor flags", - )); - } + if CURSOR_COMMANDS.contains(&command) + && let Some(message) = cursor_error(|name| namespace.value(name) != Value::None) + { + return Err(self.error(parser, message)); } if command == RENAME_COMMAND && let Some(message) = rename_error(&namespace) @@ -156,6 +142,24 @@ impl Run<'_> { } } +/// The command line and authored job settings share the same cursor-family restriction. +pub(crate) fn cursor_error(present: impl Fn(&str) -> bool) -> Option<&'static str> { + let untyped = ["start_cursor", "end_cursor"] + .iter() + .any(|name| present(name)); + let typed = [ + "start_cursor_ts", + "end_cursor_ts", + "start_cursor_int", + "end_cursor_int", + ] + .iter() + .any(|name| present(name)); + (untyped && typed).then_some( + "--start-cursor/--end-cursor cannot be combined with the -ts or -int cursor flags", + ) +} + /// The first rule `--since` and `--explain` break, on the commands that accept them. fn since_error(namespace: &Namespace) -> Option<&'static str> { match namespace.value("since") { diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index d2b8b61be1..21d2ec5463 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -78,6 +78,1116 @@ "nargs": "A...", "group": 0, "parsers": [ + { + "name": "control", + "parser": { + "prog": "sqb control", + "description": "Read one exact node's current published SQL using your saved sqb login session. Results are JSON; progress goes to stderr. No project or warehouse connection is opened.", + "groups": [{"title": "positional arguments"}, {"title": "options"}], + "actions": [ + {"kind": "help", "options": ["-h", "--help"], "dest": "help", "nargs": 0, "default": {"suppress": true}, "help": "show this help message and exit", "group": 1}, + {"kind": "store", "dest": "person_action", "choices": ["sql"], "help": "inspect published raw and compiled SQL", "group": 0}, + {"kind": "store", "dest": "person_node", "help": "exact published node key, for example model:orders (not a selector)", "group": 0}, + {"kind": "store", "options": ["--server"], "dest": "login_server", "required": true, "help": "HTTPS SQLBuild server origin used by sqb login", "group": 1}, + {"kind": "store", "options": ["--ca-cert"], "dest": "login_ca", "help": "additional trusted PEM CA certificate", "group": 1}, + {"kind": "store", "options": ["--location-id"], "dest": "person_location", "required": true, "help": "published location UUID", "group": 1}, + {"kind": "store", "options": ["--target"], "dest": "target", "required": true, "help": "exact published target", "group": 1} + ] + } + }, + { + "name": "login", + "parser": { + "prog": "sqb login", + "description": "Sign in through your browser, or verify a saved live CLI session. Credentials stay in your user credential store, never in the project or terminal output.", + "groups": [{"title": "positional arguments"}, {"title": "options"}], + "actions": [ + {"kind": "help", "options": ["-h", "--help"], "dest": "help", "nargs": 0, "default": {"suppress": true}, "help": "show this help message and exit", "group": 1}, + {"kind": "store", "dest": "login_action", "nargs": "?", "choices": ["status"], "help": "verify the saved credential with the live server", "group": 0}, + {"kind": "store", "options": ["--server"], "dest": "login_server", "required": true, "help": "HTTPS SQLBuild server origin", "group": 1}, + {"kind": "store", "options": ["--ca-cert"], "dest": "login_ca", "help": "additional trusted PEM CA certificate", "group": 1}, + {"kind": "store_true", "options": ["--no-browser"], "dest": "login_no_browser", "nargs": 0, "const": true, "default": false, "help": "print the confirmation URL without opening a browser", "group": 1} + ] + } + }, + { + "name": "logout", + "parser": { + "prog": "sqb logout", + "description": "Revoke the saved CLI session and remove local credentials. Remote failures are reported, never silently treated as successful revocation.", + "groups": [{"title": "positional arguments"}, {"title": "options"}], + "actions": [ + {"kind": "help", "options": ["-h", "--help"], "dest": "help", "nargs": 0, "default": {"suppress": true}, "help": "show this help message and exit", "group": 1}, + {"kind": "store", "options": ["--server"], "dest": "login_server", "required": true, "help": "HTTPS SQLBuild server origin", "group": 1}, + {"kind": "store", "options": ["--ca-cert"], "dest": "login_ca", "help": "additional trusted PEM CA certificate", "group": 1} + ] + } + }, + { + "name": "dev", + "parser": { + "prog": "sqb dev", + "description": "Run the local control API, scheduler and reloadable code location. Authentication is disabled only on loopback; triggers start paused. No UI is served.", + "groups": [{"title": "positional arguments"}, {"title": "options"}], + "actions": [ + {"kind": "help", "options": ["-h", "--help"], "dest": "help", "nargs": 0, "default": {"suppress": true}, "help": "show this help message and exit", "group": 1}, + {"kind": "store", "options": ["--project-dir", "--sqb-project-dir"], "dest": "project_dir", "default": {"suppress": true}, "help": "SQLBuild project directory", "group": 1}, + {"kind": "store", "options": ["--target"], "dest": "target", "help": "target to compile and run (default: configured target)", "group": 1}, + {"kind": "store", "options": ["--port"], "dest": "control_port", "default": 8000, "type": "int", "help": "loopback TCP port, or 0 to choose a free port (default: 8000)", "group": 1}, + {"kind": "store", "options": ["--control-store-url"], "dest": "development_store_url", "help": "control store URL or path (default: .sqlbuild/control.sqlite3 in the project)", "group": 1}, + {"kind": "store_true", "options": ["--no-scheduler"], "dest": "control_no_scheduler", "nargs": 0, "const": true, "default": false, "help": "serve and reload without executing scheduled work", "group": 1}, + {"kind": "store_true", "options": ["--triggers"], "dest": "development_triggers", "nargs": 0, "const": true, "default": false, "help": "preserve trigger defaults and operator overrides instead of pausing on startup", "group": 1} + ] + } + }, + { + "name": "location", + "parser": { + "prog": "sqb location", + "description": "Compile and publish a project from its own Python environment. Uses SQLBUILD_LOCATION_TOKEN for scoped HTTP access, never the control database. Results are JSON; progress goes to stderr.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": ["-h", "--help"], + "dest": "help", + "nargs": 0, + "default": {"suppress": true}, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "location_command", + "required": true, + "choices": ["publish", "serve", "runtime", "execute"], + "help": "publish one snapshot, watch and reload, or run the supervisor's stdio compiler", + "group": 0 + }, + { + "kind": "store", + "options": ["--server"], + "dest": "location_server_url", + "required": true, + "help": "control API HTTP or HTTPS origin, without credentials", + "group": 1 + }, + { + "kind": "store", + "options": ["--location-id"], + "dest": "location_id", + "required": true, + "help": "registered location UUID", + "group": 1 + }, + { + "kind": "store", + "options": ["--target"], + "dest": "target", + "required": true, + "help": "configured project target to compile", + "group": 1 + }, + { + "kind": "store", + "options": ["--generation"], + "dest": "location_generation", + "type": "int", + "help": "required for publish; serve derives the next generation from the server", + "group": 1 + }, + { + "kind": "store", + "options": ["--image-id"], + "dest": "location_image_id", + "help": "immutable deployment image identity to pin for workers", + "group": 1 + }, + { + "kind": "store", + "options": ["--run-id"], + "dest": "location_run_id", + "help": "frozen run UUID for execute; requires SQLBUILD_RUN_TOKEN", + "group": 1 + }, + { + "kind": "store", + "options": ["--claim-id"], + "dest": "location_claim_id", + "help": "active claim UUID for execute", + "group": 1 + }, + { + "kind": "store", + "options": ["--project-dir", "--sqb-project-dir"], + "dest": "project_dir", + "default": {"suppress": true}, + "help": "SQLBuild project directory (default: the current directory)", + "group": 1 + } + ] + } + }, + { + "name": "admin", + "parser": { + "prog": "sqb admin", + "description": "Privileged local administration using direct control database access. Results are JSON; issued bearer credentials appear only once on stdout.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--database-env" + ], + "dest": "control_database_env", + "default": "SQLBUILD_CONTROL_DATABASE_URL", + "help": "environment variable holding the control database URL (default: SQLBUILD_CONTROL_DATABASE_URL; SQLite when unset)", + "group": 1 + }, + { + "kind": "parsers", + "dest": "admin_command", + "nargs": "A...", + "required": true, + "group": 0, + "parsers": [ + { + "name": "create-location", + "parser": { + "prog": "sqb admin create-location", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "NAME", + "required": true, + "help": "location label; identity is assigned by the control store", + "group": 0 + } + ] + } + }, + { + "name": "create-principal", + "parser": { + "prog": "sqb admin create-principal", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "NAME", + "required": true, + "help": "principal label; creation grants no permissions", + "group": 0 + } + ] + } + }, + { + "name": "set-password", + "parser": { + "prog": "sqb admin set-password", + "description": "Set or reset an active person's password through direct control-store ownership. Reads hidden, confirmed terminal input or one line from standard input; never grants roles.", + "groups": [{"title": "positional arguments"}, {"title": "options"}], + "actions": [ + {"kind": "help", "options": ["-h", "--help"], "dest": "help", "nargs": 0, "default": {"suppress": true}, "help": "show this help message and exit", "group": 1}, + {"kind": "store", "options": [], "dest": "admin_subject", "metavar": "PRINCIPAL_ID", "required": true, "help": "active person UUID; password is read from the terminal or standard input, never an argument", "group": 0} + ] + } + }, + { + "name": "create-user", + "parser": { + "prog": "sqb admin create-user", + "description": "Initialize the first installation administrator through local control-store ownership. Available once only; losing administrators never reopens bootstrap. Password provisioning is separate.", + "groups": [{"title": "positional arguments"}, {"title": "options"}], + "actions": [ + {"kind": "help", "options": ["-h", "--help"], "dest": "help", "nargs": 0, "default": {"suppress": true}, "help": "show this help message and exit", "group": 1}, + {"kind": "store", "options": [], "dest": "admin_subject", "metavar": "NAME", "required": true, "help": "first administrator's person name", "group": 0} + ] + } + }, + { + "name": "grant-admin", + "parser": { + "prog": "sqb admin grant-admin", + "description": "Restore an existing person as installation administrator through local control-store ownership, for example after the last administrator is lost. Enables the person and revokes their sessions; never reopens bootstrap. Password provisioning is separate.", + "groups": [{"title": "positional arguments"}, {"title": "options"}], + "actions": [ + {"kind": "help", "options": ["-h", "--help"], "dest": "help", "nargs": 0, "default": {"suppress": true}, "help": "show this help message and exit", "group": 1}, + {"kind": "store", "options": [], "dest": "admin_subject", "metavar": "PRINCIPAL_ID", "required": true, "help": "existing person UUID to enable and grant installation admin", "group": 0} + ] + } + }, + { + "name": "issue-location-token", + "parser": { + "prog": "sqb admin issue-location-token", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "LOCATION_ID", + "required": true, + "help": "location UUID", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--expires-at" + ], + "dest": "admin_expires_at", + "required": true, + "help": "future expiry as an RFC3339 timestamp with timezone", + "group": 1 + } + ] + } + }, + { + "name": "revoke-token", + "parser": { + "prog": "sqb admin revoke-token", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "TOKEN_ID", + "required": true, + "help": "credential UUID, never its bearer secret", + "group": 0 + } + ] + } + }, + { + "name": "rotate-token", + "parser": { + "prog": "sqb admin rotate-token", + "description": "Replace a live credential atomically through direct control-store ownership, retaining its exact scope and expiry. Returns the new bearer once as JSON; the old credential immediately stops working.", + "groups": [{"title": "positional arguments"}, {"title": "options"}], + "actions": [ + {"kind": "help", "options": ["-h", "--help"], "dest": "help", "nargs": 0, "default": {"suppress": true}, "help": "show this help message and exit", "group": 1}, + {"kind": "store", "options": [], "dest": "admin_subject", "metavar": "TOKEN_ID", "required": true, "help": "live credential UUID, never its bearer secret", "group": 0} + ] + } + }, + { + "name": "rerun-run", + "parser": { + "prog": "sqb admin rerun-run", + "description": "Retry a terminal run using its pinned snapshot and settings, with an atomic security audit.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": ["-h", "--help"], + "dest": "help", + "nargs": 0, + "default": {"suppress": true}, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "SOURCE_RUN_ID", + "required": true, + "help": "terminal source run UUID", + "group": 0 + }, + { + "kind": "store", + "options": ["--run-id"], + "dest": "admin_event_id", + "required": true, + "help": "new run UUID; reuse it with identical arguments after an uncertain result", + "group": 1 + }, + { + "kind": "store_true", + "options": ["--from-failure"], + "dest": "admin_from_failure", + "nargs": 0, + "const": true, + "default": false, + "help": "retry exactly the failed and skipped nodes, excluding successful nodes", + "group": 1 + }, + { + "kind": "store", + "options": ["--priority"], + "dest": "admin_priority", + "type": "int", + "help": "signed queue priority; higher values run first (default: 0)", + "group": 1 + }, + { + "kind": "store", + "options": ["--reason"], + "dest": "admin_reason", + "required": true, + "help": "nonblank reason recorded with the security audit", + "group": 1 + } + ] + } + }, + { + "name": "request-run", + "parser": { + "prog": "sqb admin request-run", + "description": "Queue a published job using privileged database access. Selection, image, settings and write claims come from the snapshot; the request and audit commit together.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "LOCATION_ID", + "required": true, + "help": "location UUID", + "group": 0 + }, + { + "kind": "store", + "options": [], + "dest": "admin_job", + "metavar": "JOB", + "required": true, + "help": "published job name", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--target" + ], + "dest": "admin_target", + "required": true, + "help": "published target name", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--run-id" + ], + "dest": "admin_event_id", + "required": true, + "help": "run UUID; reuse it with identical arguments after an uncertain result", + "group": 1 + }, + { + "kind": "store", + "options": ["--start-cursor-ts"], + "dest": "admin_start_cursor_ts", + "help": "backfill timestamp start (RFC3339); requires --end-cursor-ts", + "group": 1 + }, + { + "kind": "store", + "options": ["--end-cursor-ts"], + "dest": "admin_end_cursor_ts", + "help": "inclusive backfill timestamp end (RFC3339); requires --start-cursor-ts", + "group": 1 + }, + { + "kind": "store", + "options": ["--start-cursor-int"], + "dest": "admin_start_cursor_int", + "help": "backfill integer start; requires --end-cursor-int", + "group": 1 + }, + { + "kind": "store", + "options": ["--end-cursor-int"], + "dest": "admin_end_cursor_int", + "help": "inclusive backfill integer end; requires --start-cursor-int", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--priority" + ], + "dest": "admin_priority", + "type": "int", + "help": "signed queue priority; higher values run first (default: 0)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--reason" + ], + "dest": "admin_reason", + "required": true, + "help": "nonblank reason recorded with the security audit", + "group": 1 + } + ] + } + }, + { + "name": "check-location", + "parser": { + "prog": "sqb admin check-location", + "description": "Request an outbound connection check on a published location using privileged database access.", + "groups": [ + {"title": "positional arguments"}, + {"title": "options"} + ], + "actions": [ + { + "kind": "help", + "options": ["-h", "--help"], + "dest": "help", + "nargs": 0, + "default": {"suppress": true}, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "LOCATION_ID", + "required": true, + "help": "location UUID", + "group": 0 + }, + { + "kind": "store", + "options": ["--target"], + "dest": "admin_target", + "required": true, + "help": "published target name", + "group": 1 + }, + { + "kind": "store", + "options": ["--request-id"], + "dest": "admin_event_id", + "required": true, + "help": "request UUID; reuse it with identical arguments after an uncertain result", + "group": 1 + }, + { + "kind": "store", + "options": ["--reason"], + "dest": "admin_reason", + "required": true, + "help": "nonblank reason recorded with the security audit", + "group": 1 + } + ] + } + }, + { + "name": "location-work", + "parser": { + "prog": "sqb admin location-work", + "description": "Read an outbound location request and its immutable result using privileged database access.", + "groups": [ + {"title": "positional arguments"}, + {"title": "options"} + ], + "actions": [ + { + "kind": "help", + "options": ["-h", "--help"], + "dest": "help", + "nargs": 0, + "default": {"suppress": true}, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "REQUEST_ID", + "required": true, + "help": "location work request UUID", + "group": 0 + } + ] + } + }, + { + "name": "cancel-run", + "parser": { + "prog": "sqb admin cancel-run", + "description": "Request cancellation through privileged database access; the fact, projection and security audit commit together. Active warehouse claims remain held until execution ends.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "RUN_ID", + "required": true, + "help": "run UUID", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--event-id" + ], + "dest": "admin_event_id", + "required": true, + "help": "request UUID; reuse it with identical arguments after an uncertain result", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--reason" + ], + "dest": "admin_reason", + "required": true, + "help": "nonblank reason recorded with the cancellation fact and security audit", + "group": 1 + } + ] + } + }, + { + "name": "change-trigger", + "parser": { + "prog": "sqb admin change-trigger", + "description": "Change runtime trigger state using privileged database access; the fact, projection and security audit commit together.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "TRIGGER_ID", + "required": true, + "help": "current trigger UUID", + "group": 0 + }, + { + "kind": "store", + "options": [], + "dest": "admin_trigger_change", + "metavar": "CHANGE", + "choices": [ + "enable", + "pause", + "reset-default", + "set-cursor", + "reset-cursor" + ], + "required": true, + "help": "runtime change; reset-default restores the published status default", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--event-id" + ], + "dest": "admin_event_id", + "required": true, + "help": "request UUID; reuse it with identical arguments after an uncertain result", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--reason" + ], + "dest": "admin_reason", + "required": true, + "help": "nonblank reason recorded with the operator fact and security audit", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--cursor" + ], + "dest": "admin_cursor", + "help": "RFC3339 timestamp with offset; required only for set-cursor", + "group": 1 + } + ] + } + } + ], + "choices_help": [ + { + "dest": "create-location", + "metavar": "create-location", + "help": "register a location with an atomic security audit record" + }, + { + "dest": "create-principal", + "metavar": "create-principal", + "help": "create a person without granting permissions" + }, + { + "dest": "set-password", + "metavar": "set-password", + "help": "set an active person's password without granting permissions" + }, + { + "dest": "create-user", + "metavar": "create-user", + "help": "initialize the first installation administrator once through local store ownership" + }, + { + "dest": "issue-location-token", + "metavar": "issue-location-token", + "help": "issue a bounded location credential, returned once as JSON" + }, + { + "dest": "revoke-token", + "metavar": "revoke-token", + "help": "revoke a credential by UUID" + }, + { + "dest": "rotate-token", + "metavar": "rotate-token", + "help": "replace a live credential without changing its scope or expiry" + }, + { + "dest": "request-run", + "metavar": "request-run", + "help": "queue a pinned published job with an atomic security audit" + }, + { + "dest": "rerun-run", + "metavar": "rerun-run", + "help": "retry a terminal run or exactly its failed and skipped nodes" + }, + { + "dest": "cancel-run", + "metavar": "cancel-run", + "help": "request run cancellation with an atomic audit and retry identity" + }, + { + "dest": "check-location", + "metavar": "check-location", + "help": "request an outbound location connection check with an atomic audit" + }, + { + "dest": "location-work", + "metavar": "location-work", + "help": "read a location work request and its result" + }, + { + "dest": "change-trigger", + "metavar": "change-trigger", + "help": "change trigger status or cursor with an atomic audit and retry identity" + } + ] + } + ] + } + }, + { + "name": "server", + "parser": { + "prog": "sqb server", + "description": "Run the native control API and scheduler without importing project Python.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--database-env" + ], + "dest": "control_database_env", + "default": "SQLBUILD_CONTROL_DATABASE_URL", + "help": "environment variable holding the control database URL (default: SQLBUILD_CONTROL_DATABASE_URL; SQLite when unset)", + "group": 1 + }, + { + "kind": "store", + "options": ["--launchers"], + "dest": "control_launchers", + "help": "installation-owned JSON file of local location/target launchers and concurrency limits", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--host" + ], + "dest": "control_host", + "default": "127.0.0.1", + "help": "IP address to listen on (default: 127.0.0.1)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--port" + ], + "dest": "control_port", + "default": 8000, + "type": "int", + "help": "TCP port, or 0 to choose a free port (default: 8000)", + "group": 1 + }, + { + "kind": "store", + "options": ["--password-origin"], + "dest": "control_password_origin", + "help": "enable password sessions for this exact external HTTPS origin; requires HTTPS termination in front of the HTTP listener", + "group": 1 + }, + { + "kind": "store", + "options": ["--default-role"], + "dest": "control_default_role", + "choices": ["none", "viewer"], + "default": "none", + "help": "ambient metadata access for authenticated people: none (default) or installation-wide viewer; never grants SQL, evidence, execution or administration", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--statistics-retention-days" + ], + "dest": "control_statistics_retention_days", + "default": 90, + "type": "int", + "help": "days of hourly warehouse statistics to keep; older samples are hidden and then deleted (default: 90)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--audit-retention-days" + ], + "dest": "control_audit_retention_days", + "default": 400, + "type": "int", + "help": "days of security audit records to keep; older records are deleted once every configured audit webhook has received them and no retained run's retry needs them (default: 400)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--snapshot-retention-count" + ], + "dest": "control_snapshot_retention_count", + "default": 10, + "type": "int", + "help": "published snapshots to keep per location and target; older snapshots are deleted unless a run, trigger, job or other retained record still references them (default: 10)", + "group": 1 + }, + { + "kind": "store", + "options": ["--trusted-proxy-config"], + "dest": "control_trusted_proxy_config", + "help": "JSON origin, trusted_peers and explicit subject-to-person identities; proxy must replace X-SQB-User and terminate HTTPS; no provisioning or grants", + "group": 1 + }, + { + "kind": "store", + "options": ["--oidc-config"], + "dest": "control_oidc_config", + "help": "JSON HTTPS origin/issuer, client_id and explicit subject-to-person identities; native code flow with PKCE; no provisioning or grants", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--no-scheduler" + ], + "dest": "control_no_scheduler", + "nargs": 0, + "const": true, + "default": false, + "help": "serve the API without the embedded scheduler", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--openapi" + ], + "dest": "control_openapi", + "nargs": 0, + "const": true, + "default": false, + "help": "print the OpenAPI contract as JSON without opening a database", + "group": 1 + } + ] + } + }, + { + "name": "scheduler", + "parser": { + "prog": "sqb scheduler", + "description": "Run the lease-guarded native scheduler against the server's control database.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--database-env" + ], + "dest": "control_database_env", + "default": "SQLBUILD_CONTROL_DATABASE_URL", + "help": "environment variable holding the control database URL (default: SQLBUILD_CONTROL_DATABASE_URL; SQLite when unset)", + "group": 1 + }, + { + "kind": "store", + "options": ["--launchers"], + "dest": "control_launchers", + "help": "installation-owned JSON file of local location/target launchers and concurrency limits", + "group": 1 + } + ] + } + }, { "name": "compile", "parser": { @@ -7592,6 +8702,34 @@ } ], "choices_help": [ + {"dest": "control", "metavar": "control", "help": "inspect published SQL with your saved person session"}, + {"dest": "login", "metavar": "login", "help": "sign in through a browser or verify a saved CLI session"}, + {"dest": "logout", "metavar": "logout", "help": "revoke the CLI session and remove local credentials"}, + { + "dest": "dev", + "metavar": "dev", + "help": "run the loopback development API, scheduler and reloadable code location" + }, + { + "dest": "location", + "metavar": "location", + "help": "compile and publish code-location snapshots over HTTP" + }, + { + "dest": "admin", + "metavar": "admin", + "help": "administer locations and credentials through privileged database access" + }, + { + "dest": "server", + "metavar": "server", + "help": "run the native control API and scheduler" + }, + { + "dest": "scheduler", + "metavar": "scheduler", + "help": "run a standalone lease-guarded scheduler" + }, { "dest": "cost", "metavar": "cost", @@ -7620,4 +8758,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index ef1b833fbb..3c079c4086 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -36,6 +36,8 @@ pub(crate) const MAXIMUM_BLANK_RUN: usize = 2; /// `CliNamespace` attributes the post-parse rules read and write. pub(crate) const COMMAND_DEST: &str = "command"; +pub(crate) const BUILD_COMMAND: &str = "build"; +pub(crate) const CONCURRENCY_DEST: &str = "concurrency"; pub(crate) const DBT_COMMAND: &str = "dbt"; pub(crate) const DBT_DEBUG_COMMAND: &str = "debug"; pub(crate) const DBT_PASSTHROUGH_COMMANDS: [&str; 4] = ["plan", "run", "build", "debug"]; diff --git a/crates/sqlbuild-cli/src/argv/main/job_settings_arguments.rs b/crates/sqlbuild-cli/src/argv/main/job_settings_arguments.rs new file mode 100644 index 0000000000..46d7b8ba46 --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/main/job_settings_arguments.rs @@ -0,0 +1,32 @@ +//! Validate frozen build settings against the same table as the command-line parser. + +use std::collections::BTreeMap; + +use serde_json::Value as Json; + +use crate::argv::_helpers::job_settings::{setting_argument, validate_mutex}; +use crate::argv::_helpers::normalization::cursor_error; +use crate::argv::_helpers::table::root_parser; +use crate::argv::constants::BUILD_COMMAND; + +/// Produce argument tokens, never shell text; execution identity is owned by the run request. +pub fn job_settings_arguments(settings: &BTreeMap) -> Result, String> { + let parser = root_parser() + .actions + .iter() + .flat_map(|action| &action.parsers) + .find(|(name, _)| name == BUILD_COMMAND) + .map(|(_, parser)| parser) + .ok_or_else(|| "build command is unavailable".to_owned())?; + let mut arguments: Vec = Vec::new(); + for (name, value) in settings { + if let Some(argument) = setting_argument(parser, name, value)? { + arguments.push(argument); + } + } + validate_mutex(parser, &arguments)?; + if let Some(message) = cursor_error(|name| settings.contains_key(name)) { + return Err(format!("job settings: {message}")); + } + Ok(arguments) +} diff --git a/crates/sqlbuild-cli/src/argv/main/mod.rs b/crates/sqlbuild-cli/src/argv/main/mod.rs index 0dabda8f2b..d456903240 100644 --- a/crates/sqlbuild-cli/src/argv/main/mod.rs +++ b/crates/sqlbuild-cli/src/argv/main/mod.rs @@ -1,3 +1,4 @@ pub mod cli_namespace_defaults; +pub mod job_settings_arguments; pub mod parse_cli_arguments; pub mod parse_dbt_execution_arguments; diff --git a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs index d47a6702c5..2524c91185 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs @@ -1,6 +1,14 @@ +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::argv::main::job_settings_arguments::job_settings_arguments; use crate::argv::models::{ArgvOutcome, Value}; use crate::argv::tests::helpers::{parse, strings}; -use crate::argv::tests::test_types::{ExitTestCase, HelpTextTestCase, ParsedValueTestCase}; +use crate::argv::tests::test_types::{ + ExitTestCase, HelpTextTestCase, JobSettingsRejectionTestCase, JobSettingsTestCase, + ParsedValueTestCase, +}; #[test] fn given_accepted_arguments_when_parsing_then_attributes_hold_the_parsed_values() { @@ -41,6 +49,47 @@ fn given_accepted_arguments_when_parsing_then_attributes_hold_the_parsed_values( attribute: "contract_command", expected_value: Value::Str("diff".to_owned()), }, + ParsedValueTestCase { + description: "server grants no ambient role by default", + argv: &["server"], + attribute: "control_default_role", + expected_value: Value::Str("none".into()), + }, + ParsedValueTestCase { + description: "viewer metadata access requires an explicit server option", + argv: &["server", "--default-role", "viewer"], + attribute: "control_default_role", + expected_value: Value::Str("viewer".into()), + }, + ParsedValueTestCase { + description: "an explicit failed-node rerun never becomes a full rerun", + argv: &[ + "admin", + "rerun-run", + "00000000-0000-0000-0000-000000000001", + "--run-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + "Retry failed orders", + "--from-failure", + ], + attribute: "admin_from_failure", + expected_value: Value::Bool(true), + }, + ParsedValueTestCase { + description: "a full rerun is the default without failure narrowing", + argv: &[ + "admin", + "rerun-run", + "00000000-0000-0000-0000-000000000001", + "--run-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + "Retry all orders", + ], + attribute: "admin_from_failure", + expected_value: Value::Bool(false), + }, ]; for test_case in test_cases { @@ -172,3 +221,91 @@ fn given_command_help_when_parsing_then_hidden_options_are_left_out() { assert_eq!(absent, test_case.expected_absent, "{help}"); } } + +#[test] +fn given_job_settings_when_validating_then_values_survive_the_build_parser() { + let test_cases = [ + JobSettingsTestCase { + description: "inverse flags and option-like values survive the build parser", + settings: BTreeMap::from([ + ("concurrency".into(), json!(3)), + ("load_sources".into(), json!(false)), + ("reload".into(), json!(false)), + ("run_tests".into(), json!(false)), + ("full_refresh".into(), json!(true)), + ("warehouse".into(), json!("--full-refresh")), + ]), + expected_values: vec![ + ("concurrency", Value::Int("3".into())), + ("load_sources", Value::Bool(false)), + ("reload", Value::Bool(false)), + ("run_tests", Value::Bool(false)), + ("full_refresh", Value::Bool(true)), + ("warehouse", Value::Str("--full-refresh".into())), + ], + expected_empty: false, + }, + JobSettingsTestCase { + description: "default-valued flags do not become explicit options", + settings: BTreeMap::from([ + ("run_tests".into(), json!(true)), + ("full_refresh".into(), json!(false)), + ("debug".into(), json!(false)), + ]), + expected_values: vec![], + expected_empty: true, + }, + ]; + for test_case in test_cases { + let settings = job_settings_arguments(&test_case.settings).expect(test_case.description); + assert_eq!( + settings.is_empty(), + test_case.expected_empty, + "{}", + test_case.description + ); + let mut arguments: Vec = vec!["build".to_owned()]; + arguments.extend(settings); + let arguments: Vec<&str> = arguments.iter().map(String::as_str).collect(); + let parsed = parse(&arguments).parsed.expect(test_case.description); + for (name, expected_value) in test_case.expected_values { + assert_eq!( + parsed.get(name), + Some(&expected_value), + "{}", + test_case.description + ); + } + } +} + +#[test] +fn given_conflicting_job_settings_when_validating_then_options_are_rejected() { + let test_cases = [ + JobSettingsRejectionTestCase { + description: "no-load conflicts with reload", + settings: BTreeMap::from([ + ("load_sources".into(), json!(false)), + ("reload".into(), json!(true)), + ]), + expected_error: "mutually exclusive", + }, + JobSettingsRejectionTestCase { + description: "load conflicts with reload", + settings: BTreeMap::from([ + ("load_sources".into(), json!(true)), + ("reload".into(), json!(true)), + ]), + expected_error: "mutually exclusive", + }, + ]; + for test_case in test_cases { + assert!( + job_settings_arguments(&test_case.settings) + .expect_err(test_case.description) + .contains(test_case.expected_error), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/argv/tests/test_types.rs b/crates/sqlbuild-cli/src/argv/tests/test_types.rs index 4452d1689e..793e4a87a3 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_types.rs @@ -1,3 +1,6 @@ +use std::collections::BTreeMap; + +use serde_json::Value as Json; use sqlbuild_core::text::main::python_str_repr::python_str_repr; use sqlbuild_core::dbt_cli::models::DbtParsedFlags; @@ -63,3 +66,18 @@ pub(super) struct DbtExecutionTestCase { pub(super) args: &'static [&'static str], pub(super) expected_parsed: Result, } + +/// Authored settings and the build namespace or validation failure they produce. +pub(super) struct JobSettingsTestCase { + pub(super) description: &'static str, + pub(super) settings: BTreeMap, + pub(super) expected_values: Vec<(&'static str, Value)>, + pub(super) expected_empty: bool, +} + +/// Mutually exclusive authored options rejected before a snapshot is published. +pub(super) struct JobSettingsRejectionTestCase { + pub(super) description: &'static str, + pub(super) settings: BTreeMap, + pub(super) expected_error: &'static str, +} diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 550b31ebe3..247242fd45 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -515,6 +515,27 @@ pub(crate) const DEBUG_ROUTE: Route = routed( }, ); +const LOCATION_ARGUMENTS: [Field; 9] = [ + field("project_dir", Source::EffectiveProjectDir), + SELECTED_TARGET, + same("location_id"), + attr("server_url", "location_server_url"), + attr("generation", "location_generation"), + attr("image_id", "location_image_id"), + attr("action", "location_command"), + attr("run_id", "location_run_id"), + attr("claim_id", "location_claim_id"), +]; +pub(crate) const LOCATION_ROUTE: Route = routed( + "run_location", + "run_location_command", + &LOCATION_ARGUMENTS, + CallShape { + slot_positional: 0, + implementation_keywords: true, + }, +); + const JANITOR_FIELDS: [Field; 11] = [ PROJECT_DIR, NO_COLOR, diff --git a/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs b/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs index 29de9f0374..bb91c6290a 100644 --- a/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs +++ b/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs @@ -6,9 +6,9 @@ use crate::dispatch::_helpers::subcommands::{ use crate::dispatch::constants::{ AUDIT_ROUTE, BUILD_ROUTE, CHECK_ROUTE, CLONE_ROUTE, COMPILE_ROUTE, CONTRACT_ROUTE, COST_ROUTE, DAG_ROUTE, DEBUG_ROUTE, DIFF_ROUTE, FORMAT_ROUTE, FRESHNESS_ROUTE, INIT_ROUTE, JANITOR_ROUTE, - LINEAGE_ROUTE, LOAD_ROUTE, PLAN_ROUTE, PLAYGROUND_ROUTE, QUERY_ROUTE, RECALL_COMMAND_NAME, - RECALL_ROUTE, REFACTOR_ROUTE, REVIEW_ROUTE, RULES_ROUTE, SCOPE_ROUTE, SEED_ROUTE, SKILLS_ROUTE, - TEST_ROUTE, + LINEAGE_ROUTE, LOAD_ROUTE, LOCATION_ROUTE, PLAN_ROUTE, PLAYGROUND_ROUTE, QUERY_ROUTE, + RECALL_COMMAND_NAME, RECALL_ROUTE, REFACTOR_ROUTE, REVIEW_ROUTE, RULES_ROUTE, SCOPE_ROUTE, + SEED_ROUTE, SKILLS_ROUTE, TEST_ROUTE, }; use crate::dispatch::models::{CommandFacts, Dispatch, DispatchFailure, Route}; @@ -55,6 +55,7 @@ pub fn command_dispatch(facts: &CommandFacts) -> Dispatch { RECALL_COMMAND_NAME => run(&RECALL_ROUTE), "cost" => run(&COST_ROUTE), "debug" => run(&DEBUG_ROUTE), + "location" => run(&LOCATION_ROUTE), "janitor" => run(&JANITOR_ROUTE), "init" => run(&INIT_ROUTE), "playground" => run(&PLAYGROUND_ROUTE), diff --git a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs index e6f689c242..4b4165481c 100644 --- a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs +++ b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs @@ -10,7 +10,13 @@ use crate::launcher::_helpers::messages::{print_version, report_missing_python_c use crate::launcher::constants::{ HANDOFF_FAILURE_EXIT_CODE, LAUNCH_RECORD_OPTION, VERSION_OPTIONS, }; +use crate::native_commands::main::parse_native_invocation::parse_native_invocation; +use crate::native_commands::main::run_control_invocation::run_control_invocation; +use crate::native_commands::main::run_location_invocation::run_location_invocation; +use crate::native_commands::main::run_login_invocation::run_login_invocation; use crate::native_commands::main::run_native_invocation::run_native_invocation; +use crate::native_commands::main::run_person_invocation::run_person_invocation; +use crate::native_commands::models::NativeParse; use crate::replay::main::serve_reused_compile::serve_reused_compile; /// The launch record path and key for these arguments, when the directory is known. @@ -38,11 +44,35 @@ pub(crate) fn launch(arguments: Vec, started: (Instant, SystemTime)) - print_version(); return 0; } + let native = parse_native_invocation(&arguments); + if let Ok(NativeParse::Exit(code)) = &native { + return *code; + } + if let Ok(NativeParse::Invocation { namespace, .. }) = &native + && let Some(code) = run_login_invocation(namespace) + { + return code; + } + if let Ok(NativeParse::Invocation { namespace, .. }) = &native + && let Some(code) = run_person_invocation(namespace) + { + return code; + } + if let Ok(NativeParse::Invocation { namespace, .. }) = &native + && let Some(code) = run_control_invocation(namespace) + { + return code; + } let program: PathBuf = std::env::current_exe().unwrap_or_default(); let Some(script) = python_cli(&program) else { report_missing_python_cli(&program); return HANDOFF_FAILURE_EXIT_CODE; }; + if let Ok(NativeParse::Invocation { namespace, .. }) = &native + && let Some(code) = run_location_invocation(namespace, &script) + { + return code; + } let mut handed: Vec = Vec::with_capacity(arguments.len() + 1); let record: Option<(PathBuf, String)> = launch_record(&program, &script, &arguments); if let Some((path, key)) = &record @@ -50,7 +80,12 @@ pub(crate) fn launch(arguments: Vec, started: (Instant, SystemTime)) - { return exit_code(code); } - if let Ok(code) = run_native_invocation(&arguments, &script) { + if let Ok(NativeParse::Invocation { + namespace, + use_color, + }) = native + && let Ok(code) = run_native_invocation(&namespace, &script, use_color) + { return code; } if let Some((path, _)) = record { diff --git a/crates/sqlbuild-cli/src/lib.rs b/crates/sqlbuild-cli/src/lib.rs index 538dec3263..ce5297b160 100644 --- a/crates/sqlbuild-cli/src/lib.rs +++ b/crates/sqlbuild-cli/src/lib.rs @@ -9,4 +9,6 @@ pub mod diff_report; pub mod dispatch; pub mod launcher; pub mod native_commands; +pub(crate) mod person_login; pub mod replay; +pub(crate) mod workers; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 1ab59675b6..cbce05d41e 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -1,18 +1,33 @@ -//! Which parsed invocations run without Python, decided from the parsed values alone. +//! Resolve native invocations and control-service settings from parsed arguments. +use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; +use chrono::{DateTime, Utc}; +use serde_json::json; +use sqlbuild_orchestration::api::models::{ApiAuthentication, HumanLogin}; +use sqlbuild_orchestration::credentials::models::{AuditRetention, CredentialScope}; +use sqlbuild_orchestration::declarations::models::TriggerChange; +use sqlbuild_orchestration::platform::models::{LocationWork, LocationWorkRequest}; +use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest, RunWindow}; +use sqlbuild_orchestration::snapshots::models::SnapshotRetention; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::warehouse_metadata::models::StatisticsRetention; +use uuid::Uuid; + use crate::argv::constants::{JSON_OPTION, REVIEW_PROG}; use crate::argv::models::{ArgvOutcome, Namespace, Value}; use crate::dispatch::main::invocation_policy::invocation_policy; use crate::dispatch::models::{CommandFacts, InvocationPolicy}; +use crate::native_commands::_helpers::terminal::read_password; use crate::native_commands::constants::{ CLEAN_COMMAND, INIT_COMMAND, RULES_COMMAND, RULES_LIST_ACTION, RULES_SHOW_ACTION, SKILLS_COMMAND, }; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::{ - CostOptions, NativeCommand, NativeInvocation, RulesAction, RulesOptions, SkillsOptions, + ControlAdmin, CostOptions, NativeCommand, NativeInvocation, RulesAction, RulesOptions, + SkillsOptions, }; use sqlbuild_core::cost_command::constants::DESCENDING_ORDER; @@ -25,17 +40,336 @@ pub(crate) fn review_json_failure(arguments: &[String], outcome: &ArgvOutcome) - .is_some_and(|failure| failure.prog == REVIEW_PROG) } -fn text(namespace: &Namespace, name: &str) -> Option { +pub(crate) fn text(namespace: &Namespace, name: &str) -> Option { match namespace.get(name) { Some(Value::Str(text) | Value::Path(text)) => Some(text.clone()), _ => None, } } -fn flag(namespace: &Namespace, name: &str) -> bool { +pub(crate) fn flag(namespace: &Namespace, name: &str) -> bool { namespace.get(name) == Some(&Value::Bool(true)) } +pub(crate) fn control_admin(namespace: &Namespace) -> Result { + let value = text(namespace, "admin_subject").unwrap_or_default(); + let action = text(namespace, "admin_command"); + if action.as_deref() == Some("create-user") { + if value.trim().is_empty() { + return Err(ControlError::Invalid("user name is required".into())); + } + return Ok(ControlAdmin::BootstrapUser(value)); + } + if action.as_deref() == Some("create-location") { + if value.trim().is_empty() { + return Err(ControlError::Invalid("location name is required".into())); + } + return Ok(ControlAdmin::CreateLocation(value)); + } + if action.as_deref() == Some("create-principal") { + if value.trim().is_empty() { + return Err(ControlError::Invalid("principal name is required".into())); + } + return Ok(ControlAdmin::CreatePrincipal(value)); + } + let id = Uuid::parse_str(&value) + .map_err(|_| ControlError::Invalid("a valid subject UUID is required".into()))?; + match action.as_deref() { + Some("location-work") => Ok(ControlAdmin::LocationWork(id)), + Some("check-location") => { + let request_id = + Uuid::parse_str(&text(namespace, "admin_event_id").unwrap_or_default()) + .map_err(|_| ControlError::Invalid("--request-id must be a UUID".into()))?; + let target = text(namespace, "admin_target").unwrap_or_default(); + let reason = text(namespace, "admin_reason").unwrap_or_default(); + if target.trim().is_empty() || reason.trim().is_empty() { + return Err(ControlError::Invalid( + "target and reason must not be blank".into(), + )); + } + Ok(ControlAdmin::RequestLocationWork(LocationWorkRequest { + request_id, + location_id: id, + target, + reason, + work: LocationWork::CheckConnection {}, + })) + } + Some("change-trigger" | "cancel-run" | "request-run" | "rerun-run") => { + let identity_option = if matches!(action.as_deref(), Some("request-run" | "rerun-run")) + { + "--run-id" + } else { + "--event-id" + }; + let event_id = Uuid::parse_str(&text(namespace, "admin_event_id").unwrap_or_default()) + .map_err(|_| ControlError::Invalid(format!("{identity_option} must be a UUID")))?; + let reason = text(namespace, "admin_reason").unwrap_or_default(); + if reason.trim().is_empty() { + return Err(ControlError::Invalid("--reason must not be blank".into())); + } + if matches!(action.as_deref(), Some("request-run" | "rerun-run")) { + let priority = match namespace.get("admin_priority") { + Some(Value::Int(value)) => value.parse::().map_err(|_| { + ControlError::Invalid("--priority must fit a signed 64-bit integer".into()) + })?, + _ => 0, + }; + if action.as_deref() == Some("rerun-run") { + return Ok(ControlAdmin::Rerun(RerunRequest { + run_id: event_id, + source_run_id: id, + from_failure: flag(namespace, "admin_from_failure"), + reason, + priority, + })); + } + let target = text(namespace, "admin_target").unwrap_or_default(); + let job_key = text(namespace, "admin_job").unwrap_or_default(); + if target.trim().is_empty() || job_key.trim().is_empty() { + return Err(ControlError::Invalid( + "target and job must not be blank".into(), + )); + } + let window = match ( + text(namespace, "admin_start_cursor_ts"), + text(namespace, "admin_end_cursor_ts"), + text(namespace, "admin_start_cursor_int"), + text(namespace, "admin_end_cursor_int"), + ) { + (None, None, None, None) => None, + (Some(start), Some(end), None, None) => Some(RunWindow::Timestamp { + start: DateTime::parse_from_rfc3339(&start) + .map_err(|_| { + ControlError::Invalid( + "backfill start must be an RFC3339 timestamp with timezone" + .into(), + ) + })? + .with_timezone(&Utc), + end: DateTime::parse_from_rfc3339(&end) + .map_err(|_| { + ControlError::Invalid( + "backfill end must be an RFC3339 timestamp with timezone" + .into(), + ) + })? + .with_timezone(&Utc), + }), + (None, None, Some(start), Some(end)) => Some(RunWindow::Integer { + start: start.parse().map_err(|_| { + ControlError::Invalid( + "backfill start must be a signed 64-bit integer".into(), + ) + })?, + end: end.parse().map_err(|_| { + ControlError::Invalid( + "backfill end must be a signed 64-bit integer".into(), + ) + })?, + }), + _ => { + return Err(ControlError::Invalid( + "backfill requires both start and end of exactly one cursor type" + .into(), + )); + } + }; + return Ok(ControlAdmin::RequestJobRun(JobRunRequest { + run_id: event_id, + location_id: id, + target, + job_key, + reason, + priority, + window, + })); + } + if action.as_deref() == Some("cancel-run") { + return Ok(ControlAdmin::CancelRun { + event_id, + run_id: id, + reason, + }); + } + let mut change = json!({ + "kind": text(namespace, "admin_trigger_change").unwrap_or_default().replace('-', "_"), + }); + let cursor = text(namespace, "admin_cursor"); + if let Some(cursor) = &cursor { + change["time"] = json!(cursor); + } + let change: TriggerChange = serde_json::from_value(change) + .map_err(|error| ControlError::Invalid(format!("trigger change: {error}")))?; + if cursor.is_some() && !matches!(change, TriggerChange::SetCursor { .. }) { + return Err(ControlError::Invalid( + "--cursor is only valid for set-cursor".into(), + )); + } + Ok(ControlAdmin::ChangeTrigger { + event_id, + trigger_id: id, + change, + reason, + }) + } + Some("issue-location-token") => { + let expiry = DateTime::parse_from_rfc3339( + &text(namespace, "admin_expires_at").unwrap_or_default(), + ) + .map_err(|_| ControlError::Invalid("--expires-at must be an RFC3339 timestamp".into()))? + .with_timezone(&Utc); + if expiry <= Utc::now() { + return Err(ControlError::Invalid( + "token expiry must be in the future".into(), + )); + } + Ok(ControlAdmin::IssueToken( + CredentialScope::Location { location_id: id }, + expiry, + )) + } + Some("revoke-token") => Ok(ControlAdmin::RevokeToken(id)), + Some("rotate-token") => Ok(ControlAdmin::RotateToken(id)), + Some("set-password") => Ok(ControlAdmin::SetPassword(id, read_password()?)), + Some("grant-admin") => Ok(ControlAdmin::GrantAdmin(id)), + _ => Err(ControlError::Invalid( + "an administration command is required".into(), + )), + } +} + +pub(crate) fn control_address( + namespace: &Namespace, + standalone: bool, +) -> Result, ControlError> { + if standalone { + return Ok(None); + } + let host: IpAddr = text(namespace, "control_host") + .unwrap_or_else(|| "127.0.0.1".into()) + .parse() + .map_err(|_| ControlError::Invalid("--host must be an IP address".into()))?; + let port = match namespace.get("control_port") { + Some(Value::Int(value)) => value + .parse::() + .map_err(|_| ControlError::Invalid("--port must be between 0 and 65535".into()))?, + _ => 8000, + }; + Ok(Some(SocketAddr::new(host, port))) +} + +/// API settings are validated together before any store is opened. +pub(crate) fn control_api_settings( + namespace: &Namespace, +) -> Result< + ( + ApiAuthentication, + (StatisticsRetention, AuditRetention, SnapshotRetention), + ), + ControlError, +> { + let statistics = match retention_setting(namespace, "statistics-retention-days")? { + Some(days) => StatisticsRetention::days(days)?, + None => StatisticsRetention::default(), + }; + let audit = match retention_setting(namespace, "audit-retention-days")? { + Some(days) => AuditRetention::days(days)?, + None => AuditRetention::default(), + }; + let snapshots = match retention_setting(namespace, "snapshot-retention-count")? { + Some(count) => SnapshotRetention::count(count)?, + None => SnapshotRetention::default(), + }; + Ok(( + control_authentication(namespace)?, + (statistics, audit, snapshots), + )) +} + +/// Zero and out-of-range values name the flag before any type-specific check runs. +fn retention_setting(namespace: &Namespace, flag: &str) -> Result, ControlError> { + match namespace.get(&format!("control_{}", flag.replace('-', "_"))) { + Some(Value::Int(value)) => match value.parse::() { + Ok(setting) if setting > 0 => Ok(Some(setting)), + _ => Err(ControlError::Invalid(format!( + "--{flag} must be between 1 and 65535" + ))), + }, + _ => Ok(None), + } +} + +fn control_authentication(namespace: &Namespace) -> Result { + let password = text(namespace, "control_password_origin"); + let proxy = authentication_settings( + text(namespace, "control_trusted_proxy_config"), + ("trusted-proxy", "origin, trusted_peers and identities"), + )?; + let oidc = authentication_settings( + text(namespace, "control_oidc_config"), + ("OIDC", "origin, issuer, client_id and identities"), + )?; + let login = match (password, proxy, oidc) { + (None, None, None) => HumanLogin::None, + (Some(origin), None, None) => HumanLogin::Password(origin), + (None, Some(settings), None) => HumanLogin::TrustedProxy(settings), + (None, None, Some(settings)) => HumanLogin::Oidc(settings), + (_, Some(_), _) => { + return Err(ControlError::Invalid( + "trusted-proxy authentication cannot be combined with other human authentication modes".into(), + )); + } + _ => { + return Err(ControlError::Invalid( + "OIDC authentication cannot be combined with other human authentication modes" + .into(), + )); + } + }; + Ok(ApiAuthentication { + login, + default_role: serde_json::from_value(json!( + text(namespace, "control_default_role").unwrap_or_else(|| "none".into()) + )) + .map_err(|_| ControlError::Invalid("default role must be none or viewer".into()))?, + }) +} + +fn authentication_settings( + path: Option, + (mechanism, fields): (&str, &str), +) -> Result, ControlError> { + path.map(|path| { + let document = std::fs::read_to_string(path).map_err(|_| { + ControlError::Invalid(format!("could not read {mechanism} configuration")) + })?; + serde_json::from_str(&document).map_err(|_| { + ControlError::Invalid(format!( + "{mechanism} configuration requires valid JSON with {fields}" + )) + }) + }) + .transpose() +} + +pub(crate) fn control_database(namespace: &Namespace) -> Result { + let variable = text(namespace, "control_database_env") + .unwrap_or_else(|| "SQLBUILD_CONTROL_DATABASE_URL".into()); + match std::env::var(&variable) { + Ok(value) => Ok(value), + Err(std::env::VarError::NotPresent) => Ok(PathBuf::from( + text(namespace, "project_dir").unwrap_or_else(|| ".".into()), + ) + .join(".sqlbuild/control.sqlite3") + .to_string_lossy() + .into_owned()), + Err(std::env::VarError::NotUnicode(_)) => Err(ControlError::Invalid( + "control database variable must be UTF-8".into(), + )), + } +} + /// The strings of a list attribute, or `None` when it holds anything else. fn texts(namespace: &Namespace, name: &str) -> Option> { match namespace.get(name) { diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 90297029d9..3b5ba6cb46 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -1,29 +1,864 @@ //! Running a Python-free command: its checks, then its output with the observed services. +use std::future::Future; use std::io; use std::path::{Path, PathBuf}; -use std::time::SystemTime; +use std::process::Stdio; +use std::time::{Duration, SystemTime}; -use crate::argv::models::Stream; +use chrono::Utc; +use serde_json::json; +use sqlbuild_orchestration::api::constants::BODY_BYTES; +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::api::main::serve_api::serve_api; +use sqlbuild_orchestration::api::models::{ + ApiAuthentication, DevelopmentScope, DockerImageSource, HumanLogin, +}; +use sqlbuild_orchestration::credentials::models::AuditRetention; +use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE; +use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; +use sqlbuild_orchestration::platform::main::run_maintenance::run_maintenance; +use sqlbuild_orchestration::platform::models::{ + LocationHealth, LocationHealthReport, LocationWorkClient, LocationWorkPoll, + WarehouseSyncFailure, +}; +use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; +use sqlbuild_orchestration::scheduler::models::SchedulerState; +use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation; +use sqlbuild_orchestration::snapshots::models::SnapshotRetention; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::open_store::open_store; +use sqlbuild_orchestration::store::models::ControlStore; +use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use sqlbuild_orchestration::warehouse_metadata::models::{ + MetadataBatch, MetadataScan, StatisticsDueRead, StatisticsRead, StatisticsRetention, + WarehouseHistoryKind, WarehouseHistoryPage, WarehouseStateLocation, +}; +use sqlbuild_orchestration::webhooks::constants::SECURITY_AUDIT_WEBHOOK_VARIABLE; +use sqlbuild_orchestration::webhooks::main::{ + connect_webhook::connect_webhook, run_webhook::run_webhook, +}; +use sqlbuild_orchestration::webhooks::models::{Webhook, WebhookSource}; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; +use tokio::process::{Child, Command}; +use tokio::sync::{mpsc, oneshot, watch}; +use tokio::task::JoinSet; +use tokio::time::{Instant, MissedTickBehavior}; +use uuid::Uuid; + +use crate::argv::models::{Namespace, Stream, Value}; use crate::compute_logs::constants::DEFAULT_RETENTION_COUNT; use crate::compute_logs::main::abandon_capture::abandon_capture; use crate::compute_logs::main::finish_capture::finish_capture; use crate::compute_logs::main::open_log_root::open_log_root; use crate::compute_logs::main::start_capture::start_capture; use crate::compute_logs::models::{CaptureEnd, CaptureStart, OpenCapture}; +use crate::native_commands::_helpers::decision::{control_database, flag, text}; use crate::native_commands::_helpers::init::run_init; use crate::native_commands::_helpers::package::installed_package; -use crate::native_commands::_helpers::paths::project_location; +use crate::native_commands::_helpers::paths::{location_database_files, project_location}; use crate::native_commands::_helpers::project_facts::has_no_runtime_extensions; use crate::native_commands::_helpers::rules::run_rules; use crate::native_commands::_helpers::skills::{run_skills, skill_freshness_notice}; -use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output}; -use crate::native_commands::constants::{UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND}; +use crate::native_commands::_helpers::terminal::{ + finish_lost_stdout, terminal_columns, write_output, +}; +use crate::native_commands::constants::{ + ADMIN_COMMAND, COLUMNS_VARIABLE, FALLBACK_COLUMNS, METADATA_POLL_SECONDS, + METADATA_RECONCILE_SECONDS, SCHEDULER_COMMAND, UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND, +}; use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::main::_run_admin::run_admin; +use crate::native_commands::main::_run_control::run_control; use crate::native_commands::main::_run_cost::run_cost; use crate::native_commands::models::{ - CommandOutput, InstalledPackage, LostStdout, NativeCommand, NativeInvocation, ProjectLocation, + CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, + LocationMetadataPage, LocationMetadataSync, LocationServe, LostStdout, NativeCommand, + NativeInvocation, ProjectLocation, }; +use crate::native_commands::types::LocationPublication; +use crate::workers::main::docker_image_sources::docker_image_sources; +use crate::workers::main::open_development_workers::open_development_workers; +use crate::workers::main::open_workers::open_workers; +use crate::workers::main::terminate_child::terminate_child; +use crate::workers::models::Workers; + +/// Own the API, scheduler and outbound location as one development process tree. +pub(crate) async fn run_development_roles( + (store, scope): (ControlStore, DevelopmentScope), + (project, executable): (PathBuf, PathBuf), + listener: tokio::net::TcpListener, + scheduler: bool, +) -> Result<(), ControlError> { + let address = listener.local_addr()?; + let url = format!("http://{address}"); + let location_id = scope.location_id; + let target = scope.target.clone(); + let triggers = scope.triggers; + let workers = open_development_workers( + (location_id, target.clone()), + project.clone(), + executable.clone(), + url.clone(), + )?; + let (stop, shutdown) = watch::channel(false); + let mut tasks = JoinSet::new(); + tasks.spawn(serve_api( + store.clone(), + listener, + (ApiAuthentication { + login: HumanLogin::Development(scope), + ..ApiAuthentication::default() + }, Vec::new(), Default::default()), + (move || { + drop(write_output(&[(Stream::Stderr, &format!( + "Development API ready: {address}\nDevelopment location: {location_id}\nAuthentication disabled on loopback; triggers {}\n", + if triggers { "follow code defaults and operator overrides" } else { "start paused" } + ))])); + }, development_stopped(shutdown.clone())), + )); + start_platform_roles(&mut tasks, (&store, &shutdown), Default::default())?; + tasks.spawn(development_location_role( + (&project, &executable), + (location_id, &target), + (&url, &store), + shutdown.clone(), + )?); + if scheduler { + drop(write_output(&[( + Stream::Stderr, + "Starting development scheduler...\n", + )])); + let scheduler_store = store.clone(); + tasks.spawn(async move { + run_scheduler( + &scheduler_store, + SchedulerState::new(Uuid::now_v7()), + shutdown, + workers, + ) + .await + }); + drop(write_output(&[( + Stream::Stderr, + "Development scheduler started\n", + )])); + } + supervise_control_roles(tasks, stop).await +} + +async fn development_stopped(mut shutdown: watch::Receiver) { + while !*shutdown.borrow_and_update() { + if shutdown.changed().await.is_err() { + break; + } + } +} + +/// Hand off only the local file exclusion, never control-store connection credentials. +fn development_location_role( + (project, executable): (&Path, &Path), + (location_id, target): (Uuid, &str), + (url, store): (&str, &ControlStore), + shutdown: watch::Receiver, +) -> Result> + Send + 'static, ControlError> { + let mut command = Command::new(executable); + command + .args(["--project-dir"]) + .arg(project) + .args([ + "location", + "serve", + "--server", + url, + "--location-id", + &location_id.to_string(), + "--target", + target, + ]) + .env("SQLBUILD_LOCATION_TOKEN", "local-development") + .env_remove("SQLBUILD_CONTROL_DATABASE_URL") + .env_remove("SQLBUILD_PLATFORM_WEBHOOK_URL") + .env_remove(SECURITY_AUDIT_WEBHOOK_VARIABLE) + .env_remove("SQLBUILD_RUN_TOKEN") + .env_remove("SQLBUILD_DEVELOPMENT_CONTROL_FILE") + .stdin(Stdio::null()) + .kill_on_drop(true); + if let ControlStore::Sqlite(path) = store { + command.env( + "SQLBUILD_DEVELOPMENT_CONTROL_FILE", + std::env::current_dir()?.join(path), + ); + } + let mut child = command.spawn()?; + Ok(async move { + tokio::select! { + result = child.wait() => { + result?; + Err(ControlError::Invalid("development code location stopped unexpectedly".into())) + }, + () = development_stopped(shutdown) => { + terminate_child(&mut child)?; + match tokio::time::timeout(Duration::from_secs(10), child.wait()).await { + Ok(result) => { result?; }, + Err(_) => child.kill().await?, + } + Ok(()) + }, + } + }) +} + +/// Request one publication without transferring interpreter ownership to a cancellable future. +pub(crate) async fn compile_generation( + requests: &mpsc::Sender, + generation: i64, + sources: &[sqlbuild_cache::project_snapshot::models::PathStamp], + stopped: impl Future>, +) -> Result, Vec)>, ControlError> { + let (reply, publication) = oneshot::channel(); + requests + .send(LocationCompileRequest { + generation, + python_sources: sources + .iter() + .filter(|stamp| stamp.relative_path.ends_with(".py")) + .cloned() + .collect(), + reply, + }) + .await + .map_err(|_| ControlError::Invalid("location compiler stopped".into()))?; + tokio::pin!(stopped); + let result = tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + result = publication => result.map_err(|_| ControlError::Invalid("location compiler stopped before replying".into()))?, + }; + let record = match result { + Ok(record) => record, + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!( + "Location compiler failed; generation {generation} may already be published: {error}\n" + ), + )])); + None + } + }; + let published = record.is_some(); + let mut refresh_at = None; + let mut locations: Vec = Vec::new(); + if let Some((record, refresh, state_locations)) = record { + refresh_at = refresh + .map(|deadline| Instant::now() + (deadline - Utc::now()).to_std().unwrap_or_default()); + locations = state_locations; + let output = serde_json::to_string(&record) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if write_output(&[(Stream::Stdout, &format!("{output}\n"))]).is_some() { + return Err(ControlError::Invalid(format!( + "Snapshot {} was published, but result delivery failed", + record.snapshot_id + ))); + } + } + Ok(Some((published, refresh_at, locations))) +} + +/// The child is retained by its owning task while this exchange can be cancelled. +pub(crate) async fn exchange_location_generation( + compiler: &mut LocationCompiler, + generation: i64, +) -> Result, ControlError> { + compiler + .stdin + .write_all(format!("{generation}\n").as_bytes()) + .await?; + compiler.stdin.flush().await?; + let mut line = String::new(); + if compiler.stdout.read_line(&mut line).await? == 0 { + return Err(ControlError::Invalid( + "location interpreter exited before replying".into(), + )); + } + serde_json::from_str(&line).map_err(|_| { + ControlError::Invalid("location interpreter returned an invalid publication record".into()) + }) +} + +/// The caller retains and reaps the child while a cancellable bounded exchange owns its pipes. +pub(crate) fn location_result( + (options, script): (&LocationServe, &Path), + (request, maximum_bytes): (Vec, u64), +) -> Result<(Child, impl Future>), ControlError> { + let mut child = location_runtime_child(options, script)?; + let mut stdin = child + .stdin + .take() + .ok_or_else(|| ControlError::Invalid("location input is unavailable".into()))?; + let stdout = child + .stdout + .take() + .ok_or_else(|| ControlError::Invalid("location output is unavailable".into()))?; + Ok((child, async move { + stdin.write_all(&request).await?; + stdin.write_all(b"\n").await?; + stdin.shutdown().await?; + drop(stdin); + let mut bytes: Vec = Vec::new(); + stdout + .take(maximum_bytes + 1) + .read_to_end(&mut bytes) + .await?; + if bytes.len() as u64 > maximum_bytes { + return Err(ControlError::Invalid( + "location reply exceeds its byte limit".into(), + )); + } + serde_json::from_slice(&bytes) + .map_err(|_| ControlError::Invalid("invalid location reply".into())) + })) +} + +pub(crate) fn metadata_sync( + (locations, generation): (Vec, i64), + reconcile_seconds: u32, +) -> LocationMetadataSync { + LocationMetadataSync { + locations, + generation, + index: 0, + scan_id: Uuid::now_v7(), + source_after: None, + observed_since: None, + watermark_loaded: false, + statistics: None, + reconcile: true, + reconcile_at: Instant::now(), + reconcile_seconds, + limit: MAX_HISTORY_RECORDS, + pending: None, + next_at: Instant::now(), + } +} + +pub(crate) fn history_request(state: &LocationMetadataSync) -> Result, ControlError> { + if state.index == 0 && state.source_after.is_none() && state.limit == MAX_HISTORY_RECORDS { + drop(write_output(&[( + Stream::Stderr, + "Reading warehouse history for automatic synchronization...\n", + )])); + } + let kind = state.kind().as_str(); + let request = if state.kind() == WarehouseHistoryKind::Statistics { + json!({(kind): state.statistics.as_ref().ok_or_else(|| ControlError::Invalid("statistics eligibility page missing".into()))?}) + } else { + json!({(kind): { + "location":state.location(),"after":state.source_after,"observed_since":state.observed_since,"limit":state.limit + }}) + }; + serde_json::to_vec(&request).map_err(|error| ControlError::Invalid(error.to_string())) +} + +pub(crate) async fn statistics_page( + (client, target): (&LocationWorkClient, &str), + (state, poll): (&LocationMetadataSync, &LocationWorkPoll), +) -> Result { + let after = state + .source_after + .as_ref() + .and_then(|values| values.first()) + .and_then(|value| value.as_deref()) + .map(Uuid::parse_str) + .transpose() + .map_err(|_| ClientError::Configuration("invalid statistics eligibility cursor"))?; + client + .statistics_due( + target, + &StatisticsDueRead { + session_id: poll.session_id, + generation: state.generation, + location: state.location().clone(), + after, + limit: state.limit, + }, + ) + .await +} + +/// Freeze one bounded page; a full scan's last page also removes rows the scan did not deliver. +pub(crate) fn freeze_history_page( + mut state: LocationMetadataSync, + page: WarehouseHistoryPage, + poll: &LocationWorkPoll, +) -> Result { + let batch = MetadataBatch { + kind: state.kind(), + session_id: poll.session_id, + generation: state.generation, + records: page + .records + .iter() + .map(serde_json::to_value) + .collect::>() + .map_err(|error| ControlError::Invalid(error.to_string()))?, + scan: state.reconciling().then(|| MetadataScan { + scan_id: state.scan_id, + location: state.location().clone(), + complete: page.next_after.is_none(), + }), + }; + if serde_json::to_vec(&batch) + .map_err(|error| ControlError::Invalid(error.to_string()))? + .len() + > BODY_BYTES + { + state.statistics = None; + if state.limit > 1 { + state.limit = (state.limit / 2).max(1); + } else { + state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); + drop(write_output(&[( + Stream::Stderr, + "Warehouse history row exceeds the request bound; synchronization incomplete, no stale rows removed.\n", + )])); + } + } else { + state.pending = Some(LocationMetadataPage { + batch, + next_source_after: page.next_after, + }); + } + Ok(state) +} + +pub(crate) fn acknowledge_metadata_page(mut state: LocationMetadataSync) -> LocationMetadataSync { + let Some(pending) = state.pending.take() else { + return state; + }; + state.source_after = pending.next_source_after; + if state.kind() == WarehouseHistoryKind::Statistics { + state.statistics = None; + } + if state.source_after.is_none() { + state.index += 1; + state.scan_id = Uuid::now_v7(); + state.observed_since = None; + state.watermark_loaded = false; + state.limit = MAX_HISTORY_RECORDS; + if state.index == state.locations.len() * WarehouseHistoryKind::ALL.len() { + state.index = 0; + if state.reconcile { + state.reconcile_at = + Instant::now() + Duration::from_secs(u64::from(state.reconcile_seconds)); + } + state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); + drop(write_output(&[( + Stream::Stderr, + "Warehouse history synchronization completed.\n", + )])); + } + } + state +} + +/// Operational cost is elapsed IO time, not an estimate of warehouse billing. +pub(crate) async fn metadata_health( + state: &LocationMetadataSync, + (client, poll): (&LocationWorkClient, &LocationWorkPoll), + (started, failure): (Instant, Option), +) { + let report = LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: poll.session_id, + target: poll.target.clone(), + health: LocationHealth::WarehouseSync { + generation: state.generation, + location: state.location().clone(), + table: state.kind(), + failure, + records_sent: state + .pending + .as_ref() + .map_or(0, |page| page.batch.records.len()) as u64, + elapsed_millis: started.elapsed().as_millis() as u64, + }, + }; + if let Err(error) = client.report_health(&report).await { + drop(write_output(&[( + Stream::Stderr, + &format!("Warehouse synchronization health report failed: {error}\n"), + )])); + } +} + +/// Health IO remains native even while a user-code child holds its interpreter's GIL. +pub(crate) async fn report_location_health( + options: &LocationServe, + bearer: &str, + session_id: Uuid, + health: LocationHealth, +) { + let report = LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: options.target.clone(), + health, + }; + if let Err(error) = + report_remote_location_health((&options.server_url, bearer), options.location_id, &report) + .await + { + drop(write_output(&[( + Stream::Stderr, + &format!("Location health report failed: {error}\n"), + )])); + } +} + +pub(crate) async fn location_heartbeats( + options: &LocationServe, + bearer: &str, + session_id: Uuid, +) -> Result<(), ControlError> { + let mut clock = tokio::time::interval(Duration::from_secs(30)); + clock.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + clock.tick().await; + report_location_health(options, bearer, session_id, LocationHealth::Heartbeat {}).await; + } +} + +pub(crate) fn location_options(namespace: &Namespace) -> Result { + if namespace + .get("location_generation") + .is_some_and(|value| *value != Value::None) + { + return Err(ControlError::Invalid( + "serve derives --generation from the server; do not supply it".into(), + )); + } + let project_dir = text(namespace, "project_dir") + .map_or_else(std::env::current_dir, |directory| { + std::fs::canonicalize(directory) + })?; + if !project_dir.is_dir() { + return Err(ControlError::Invalid("project directory must exist".into())); + } + let location_id = text(namespace, "location_id") + .unwrap_or_default() + .parse() + .map_err(|_| ControlError::Invalid("location ID must be a UUID".into()))?; + let target = text(namespace, "target").unwrap_or_default(); + if target.trim().is_empty() { + return Err(ControlError::Invalid("target must not be blank".into())); + } + let metadata_reconcile_seconds = std::env::var("SQLBUILD_METADATA_RECONCILE_SECONDS") + .map_or(Ok(METADATA_RECONCILE_SECONDS), |value| { + value + .parse::() + .map(std::num::NonZeroU32::get) + }) + .map_err(|_| { + ControlError::Invalid( + "SQLBUILD_METADATA_RECONCILE_SECONDS must be a positive integer".into(), + ) + })?; + Ok(LocationServe { + project_dir, + server_url: text(namespace, "location_server_url").unwrap_or_default(), + location_id, + target, + image_id: text(namespace, "location_image_id"), + metadata_reconcile_seconds, + }) +} + +/// Reconcile from the server after restarts or lost acknowledgements; transient outages retry. +pub(crate) async fn location_generation( + options: &LocationServe, + bearer: &str, +) -> Result, ControlError> { + match next_remote_generation( + &options.server_url, + bearer, + options.location_id, + &options.target, + ) + .await + { + Ok(generation) => Ok(Some(generation)), + Err(error) => { + if matches!(&error, ClientError::Configuration(_)) + || matches!(&error, ClientError::Rejected(status) if matches!(status.as_u16(), 401 | 403)) + { + return Err(ControlError::Invalid(error.to_string())); + } + drop(write_output(&[( + Stream::Stderr, + &format!( + "Location generation lookup failed; retaining last good snapshot and retrying: {error}\n" + ), + )])); + Ok(None) + } + } +} + +/// The installed interpreter owns user imports; the supervisor never imports Python. +pub(crate) fn location_runtime_child(options: &LocationServe, script: &Path) -> io::Result { + let mut command = location_child(options, script); + command + .args([ + "location", + "runtime", + "--server", + &options.server_url, + "--location-id", + ]) + .arg(options.location_id.to_string()) + .args(["--target", &options.target]); + if let Some(image) = &options.image_id { + command.args(["--image-id", image]); + } + command.stdin(Stdio::piped()).stdout(Stdio::piped()).spawn() +} + +pub(crate) async fn check_published_location( + options: &LocationServe, + script: &Path, + (bearer, session_id, generation): (&str, Uuid, i64), + stopped: impl Future>, +) -> Result>, ControlError> { + drop(write_output(&[( + Stream::Stderr, + "Checking location connection with sqb debug...\n", + )])); + tokio::pin!(stopped); + let Some((connected, files)) = + check_location_connection(options, script, stopped.as_mut()).await? + else { + return Ok(None); + }; + tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = report_location_health(options, bearer, session_id, LocationHealth::Connection { generation, healthy: connected }) => {}, + } + drop(write_output(&[( + Stream::Stderr, + if connected { + "Code location ready; connection check passed; watching for changes.\n" + } else { + "Code location published; connection check failed; watching for changes.\n" + }, + )])); + Ok(Some(files)) +} + +/// Capture the ordinary debug check's exact local file paths without contaminating publication stdout. +pub(crate) async fn check_location_connection( + options: &LocationServe, + script: &Path, + stopped: impl Future>, +) -> Result)>, ControlError> { + let mut child = location_child(options, script) + .args(["debug", "--target", &options.target, "--json"]) + .stdout(Stdio::piped()) + .spawn()?; + let mut stdout = child + .stdout + .take() + .ok_or_else(|| ControlError::Invalid("location debug output is unavailable".into()))?; + let mut output: Vec = Vec::new(); + tokio::pin!(stopped); + let connected = tokio::select! { + result = &mut stopped => { + child.kill().await?; + result?; + return Ok(None); + }, + result = async { tokio::try_join!(child.wait(), stdout.read_to_end(&mut output)) } => result?.0.success(), + }; + let files: Vec = match serde_json::from_slice::(&output) + .and_then(|report| serde_json::from_value(report["local_files"].clone())) + { + Ok(files) => files, + Err(_) => { + drop(write_output(&[( + Stream::Stderr, + "Location connection check did not produce a valid report; retaining the published snapshot.\n", + )])); + return Ok(Some((false, Vec::new()))); + } + }; + Ok(Some(( + connected, + location_database_files(&options.project_dir, files), + ))) +} + +fn location_child(options: &LocationServe, script: &Path) -> Command { + let mut command = Command::new(script); + command.arg("--project-dir").arg(&options.project_dir); + if terminal_columns().is_err() { + command.env(COLUMNS_VARIABLE, FALLBACK_COLUMNS.to_string()); + } + command + .stdin(Stdio::null()) + .stderr(Stdio::inherit()) + .kill_on_drop(true); + command +} + +/// Health production and delivery remain native and independent of scheduler leadership. +pub(crate) fn start_platform_roles( + tasks: &mut JoinSet>, + (store, shutdown): (&ControlStore, &watch::Receiver), + retention: (AuditRetention, StatisticsRetention, SnapshotRetention), +) -> Result<(), ControlError> { + let mut sinks: Vec<(Webhook, &str)> = Vec::new(); + for (variable, source, label) in [ + ( + PLATFORM_WEBHOOK_VARIABLE, + WebhookSource::Platform, + "Platform", + ), + ( + SECURITY_AUDIT_WEBHOOK_VARIABLE, + WebhookSource::SecurityAudit, + "Security audit", + ), + ] { + match std::env::var(variable) { + Ok(endpoint) => sinks.push((connect_webhook(&endpoint, source)?, label)), + Err(std::env::VarError::NotPresent) => {} + Err(_) => { + return Err(ControlError::Invalid(format!( + "{label} webhook configuration must be UTF-8" + ))); + } + } + } + drop(write_output(&[( + Stream::Stderr, + "Starting platform maintenance...\n", + )])); + let maintenance_store = store.clone(); + let maintenance_shutdown = shutdown.clone(); + let audit_sinks: Vec = sinks.iter().map(|(sink, _)| sink.clone()).collect(); + tasks.spawn(async move { + let report = |retained: i64| { + drop(write_output(&[( + Stream::Stderr, + &format!("Security audit retention: retained past horizon: {retained}\n"), + )])); + }; + run_maintenance( + (&maintenance_store, Uuid::now_v7()), + (retention, &audit_sinks), + (report, maintenance_shutdown), + ) + .await + }); + drop(write_output(&[( + Stream::Stderr, + "Platform maintenance started\n", + )])); + for (sink, label) in sinks { + drop(write_output(&[( + Stream::Stderr, + &format!("Starting {label} webhook...\n"), + )])); + let sink_store = store.clone(); + let sink_shutdown = shutdown.clone(); + tasks.spawn(async move { run_webhook(&sink_store, sink, sink_shutdown).await }); + drop(write_output(&[( + Stream::Stderr, + &format!("{label} webhook started\n"), + )])); + } + Ok(()) +} + +/// Dispatch maintenance actions without implicitly initializing or upgrading their store. +pub(crate) async fn run_control_action( + namespace: &Namespace, + command: &str, +) -> Result, ControlError> { + if command == ADMIN_COMMAND { + return run_admin(namespace).await.map(Some); + } + run_control(namespace, command == SCHEDULER_COMMAND) + .await + .map(|()| None) +} + +/// Open and migrate the privileged store, reporting progress without contaminating JSON stdout. +pub(crate) async fn open_control_store( + namespace: &Namespace, +) -> Result { + let url = control_database(namespace)?; + drop(write_output(&[( + Stream::Stderr, + "Opening control store...\n", + )])); + let store = open_store(&url).await?; + migrate_store(&store).await?; + drop(write_output(&[(Stream::Stderr, "Control store ready\n")])); + Ok(store) +} + +/// Decode privileged CLI configuration before handing process ownership to the workers domain. +pub(crate) fn open_control_workers( + namespace: &Namespace, +) -> Result<(Workers, Vec), ControlError> { + let workers = open_workers( + text(namespace, "control_launchers").as_deref(), + text(namespace, "control_database_env") + .unwrap_or_else(|| "SQLBUILD_CONTROL_DATABASE_URL".into()), + !flag(namespace, "control_no_scheduler"), + )?; + let images = docker_image_sources(&workers); + Ok((workers, images)) +} + +pub(crate) async fn shutdown_signal() -> Result<(), ControlError> { + #[cfg(unix)] + { + let mut terminate = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + tokio::select! { + result = tokio::signal::ctrl_c() => result?, + _ = terminate.recv() => {}, + } + } + #[cfg(not(unix))] + tokio::signal::ctrl_c().await?; + Ok(()) +} + +/// Stop sibling roles on a signal or role failure, then drain their shutdown results. +pub(crate) async fn supervise_control_roles( + mut tasks: JoinSet>, + stop: watch::Sender, +) -> Result<(), ControlError> { + let mut result = tokio::select! { + result = tasks.join_next() => match result { + Some(result) => result.map_err(ControlError::from).and_then(std::convert::identity), + None => Ok(()), + }, + result = shutdown_signal() => result, + }; + stop.send_replace(true); + while let Some(joined) = tasks.join_next().await { + let joined = joined + .map_err(ControlError::from) + .and_then(std::convert::identity); + result = result.and(joined); + } + result +} /// The capture Python's `dispatch_with_compute_logs` opens; a failure only loses the log. fn open_capture( diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs index 302e8a5a66..e64efaba99 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs @@ -1,11 +1,158 @@ //! `pathlib.Path` construction and `Path.resolve()` over native paths. use std::path::{Component, Path, PathBuf}; +use std::time::Duration; +use sqlbuild_cache::digest::constants::{ + EXCLUDED_DIRECTORIES, EXCLUDED_ROOT_DIRECTORIES, PRESENCE_ONLY_FILE_SUFFIXES, +}; +use sqlbuild_cache::project_snapshot::main::snapshot_project_files::snapshot_project_files; +use sqlbuild_cache::project_snapshot::models::{PathStamp, SnapshotRules}; +use sqlbuild_core::git_revisions::main::head_commit::head_commit; use sqlbuild_rules::project_tree::main::project_real_path::project_real_path; +use tokio::time::Instant; use crate::native_commands::errors::PythonNeeded; -use crate::native_commands::models::ProjectLocation; +use crate::native_commands::models::{LocationWatch, ProjectLocation}; +use crate::replay::main::output_file_identities::output_file_identities; + +/// Observe the same project inputs as compiler reuse, excluding redirected command output. +pub(crate) fn location_watch(root: &Path) -> LocationWatch { + let rules = location_rules(); + let files: Vec = std::env::var_os("SQLBUILD_DEVELOPMENT_CONTROL_FILE") + .into_iter() + .map(|path| path.to_string_lossy().into_owned()) + .collect(); + let local_files = location_database_files(root, files); + LocationWatch { + observed: location_inputs(root, &rules, &local_files), + rules, + local_files, + attempted: None, + changed_at: tokio::time::Instant::now(), + retry_at: None, + } +} + +/// Coalesce source edits and retry deadlines without replacing the retained attempted input. +pub(crate) fn observe_location_inputs( + root: &Path, + mut watch: LocationWatch, +) -> (LocationWatch, bool) { + let current = location_inputs(root, &watch.rules, &watch.local_files); + if watch.observed != current { + watch.observed = current; + watch.changed_at = Instant::now(); + } + if watch.attempted.as_ref() == Some(&watch.observed) + && watch.retry_at.is_none_or(|retry| Instant::now() < retry) + { + return (watch, false); + } + if watch.attempted.is_some() && watch.changed_at.elapsed() < Duration::from_millis(250) { + return (watch, false); + } + watch.attempted = Some(watch.observed.clone()); + (watch, true) +} + +fn location_rules() -> SnapshotRules { + SnapshotRules { + excluded: EXCLUDED_DIRECTORIES + .iter() + .map(ToString::to_string) + .collect(), + excluded_root: EXCLUDED_ROOT_DIRECTORIES + .iter() + .map(ToString::to_string) + .collect(), + presence_suffixes: PRESENCE_ONLY_FILE_SUFFIXES + .iter() + .map(ToString::to_string) + .collect(), + output_files: output_file_identities(), + } +} + +/// Git commits matter to publication metadata even when no working-tree file changes. +pub(crate) fn location_inputs( + root: &Path, + rules: &SnapshotRules, + local_files: &[PathBuf], +) -> (Vec, Option) { + let mut stamps = snapshot_project_files(root, rules); + stamps.retain(|stamp| { + !rules + .presence_suffixes + .iter() + .any(|suffix| stamp.relative_path.ends_with(suffix)) + }); + stamps = exclude_location_files(stamps, root, local_files); + #[cfg(windows)] + { + let output: Vec = + [same_file::Handle::stdout(), same_file::Handle::stderr()] + .into_iter() + .flatten() + .filter(|handle| { + handle + .as_file() + .metadata() + .is_ok_and(|metadata| metadata.is_file()) + }) + .collect(); + stamps.retain(|stamp| { + same_file::Handle::from_path(root.join(&stamp.relative_path)) + .map_or(true, |handle| !output.contains(&handle)) + }); + } + stamps.sort_by(|left, right| left.relative_path.cmp(&right.relative_path)); + (stamps, head_commit(root)) +} + +/// Apply resolved exclusions to retained observations, never a fresh snapshot of possible source edits. +pub(crate) fn exclude_location_databases( + root: &Path, + files: Vec, + mut watch: LocationWatch, +) -> LocationWatch { + watch.local_files.extend(files); + watch.local_files.sort(); + watch.local_files.dedup(); + watch.observed.0 = exclude_location_files(watch.observed.0, root, &watch.local_files); + watch.attempted = watch.attempted.map(|(stamps, commit)| { + ( + exclude_location_files(stamps, root, &watch.local_files), + commit, + ) + }); + watch +} + +fn exclude_location_files( + mut stamps: Vec, + root: &Path, + local_files: &[PathBuf], +) -> Vec { + stamps.retain(|stamp| !local_files.contains(&root.join(&stamp.relative_path))); + stamps +} + +pub(crate) fn location_database_files(root: &Path, files: Vec) -> Vec { + let mut paths: Vec = Vec::new(); + for file in files { + let authored = pathlib_path(&root.join(file)); + let resolved = project_real_path(&authored); + for path in [authored, resolved] { + for suffix in ["", ".wal", "-wal", "-shm", "-journal"] { + let mut file = path.as_os_str().to_os_string(); + file.push(suffix); + paths.push(PathBuf::from(file)); + } + } + } + paths +} /// `Path(text)`: repeated separators, `.` parts and trailing separators dropped. pub(crate) fn pathlib_path(text: &Path) -> PathBuf { diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs index e94539aa29..7b06b015ef 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs @@ -2,9 +2,14 @@ use std::path::Path; +use sqlbuild_config::effective::main::effective_target_name::effective_target_name; +use sqlbuild_config::effective::models::TargetChoice; +use sqlbuild_config::models::ConfigValue; +use sqlbuild_config::project::main::load_local_config::load_local_config; use sqlbuild_config::project::main::load_project_config::load_project_config; use sqlbuild_config::project::models::{Built, ConfigPaths}; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_orchestration::store::errors::ControlError; use crate::native_commands::constants::{ CUSTOM_RULES_DIRECTORY, EVENT_EXPORTERS_DIRECTORY, LEGACY_LOCAL_CONFIG_FILENAME, @@ -64,3 +69,41 @@ pub(crate) fn has_no_runtime_extensions(project_dir: &Path) -> bool { pub(crate) fn has_custom_rules(project_dir: &Path) -> bool { project_dir.join(CUSTOM_RULES_DIRECTORY).is_dir() } + +/// Resolve development's target through the same selected/local/default precedence as compilation. +pub(crate) fn development_target( + project_dir: &Path, + selected: Option, +) -> Result { + let paths = config_paths(project_dir); + let invalid = + || ControlError::Invalid("project configuration is invalid; run sqb debug".into()); + let project = load_project_config(&paths).map_err(|_| invalid())?; + let local = load_local_config(&paths).map_err(|_| invalid())?; + let text = |config: &Built, name: &str| match object_field(config, name) { + Some(Built::Value(ConfigValue::String(value))) => Some(value.clone()), + _ => None, + }; + let defined: Vec = [&project, &local] + .into_iter() + .filter_map(|config| object_field(config, "targets")) + .filter_map(|targets| match targets { + Built::Dict(targets) => Some(targets), + _ => None, + }) + .flatten() + .map(|(name, _)| name.clone()) + .collect(); + effective_target_name(TargetChoice { + selected, + local: text(&local, "target"), + default: text(&project, "default_target"), + defined, + }) + .map_err(|_| { + ControlError::Invalid("unknown development target; run sqb debug --target ".into()) + })? + .ok_or_else(|| { + ControlError::Invalid("sqb dev requires --target or a configured default target".into()) + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs index cd354451ed..5476f82e40 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs @@ -1,12 +1,15 @@ //! The process's terminal: its width, colour support and stream writes. -use std::io::{self, IsTerminal, Write}; +use std::io::{self, BufRead, IsTerminal, Read, Write}; + +use sqlbuild_orchestration::store::errors::ControlError; use crate::argv::models::Stream; use crate::native_commands::_helpers::argv_host::python_int; use crate::native_commands::constants::{ BROKEN_PIPE_EXCEPTION, COLUMNS_VARIABLE, EXIT_FLUSH_FAILURE, EXIT_FLUSH_FAILURE_STATUS, - FALLBACK_COLUMNS, NO_COLOR_VARIABLE, PYTHON_STDOUT_BUFFER_SIZE, UNCAUGHT_EXCEPTION_STATUS, + FALLBACK_COLUMNS, NO_COLOR_VARIABLE, PASSWORD_INPUT_BYTES, PYTHON_STDOUT_BUFFER_SIZE, + UNCAUGHT_EXCEPTION_STATUS, }; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::LostStdout; @@ -54,6 +57,37 @@ pub(crate) fn stdout_supports_color() -> bool { std::env::var_os(NO_COLOR_VARIABLE).is_none() && io::stdout().is_terminal() } +/// Passwords never enter parsed arguments, environment variables or command output. +pub(crate) fn read_password() -> Result { + let password = if io::stdin().is_terminal() { + let password = rpassword::prompt_password("Password: ")?; + if password != rpassword::prompt_password("Confirm password: ")? { + return Err(ControlError::Invalid("passwords do not match".into())); + } + password + } else { + let mut password = String::new(); + io::stdin() + .lock() + .take(PASSWORD_INPUT_BYTES) + .read_line(&mut password)?; + if password.len() as u64 == PASSWORD_INPUT_BYTES { + return Err(ControlError::Invalid("password input is too large".into())); + } + if password.ends_with('\n') { + password.pop(); + if password.ends_with('\r') { + password.pop(); + } + } + password + }; + if password.is_empty() { + return Err(ControlError::Invalid("password input was empty".into())); + } + Ok(password) +} + /// `\n` as a Python text stream writes it on this platform. fn console_bytes(text: &str) -> Vec { if cfg!(windows) { diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 867ce8bb82..a3c0d46a69 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -41,6 +41,9 @@ pub(crate) const RUNNING_BUILD_STATUS: &str = "running"; pub(crate) const CLEAN_COMMAND: &str = "clean"; pub(crate) const INIT_COMMAND: &str = "init"; pub(crate) const SKILLS_COMMAND: &str = "skills"; +pub(crate) const SERVER_COMMAND: &str = "server"; +pub(crate) const SCHEDULER_COMMAND: &str = "scheduler"; +pub(crate) const ADMIN_COMMAND: &str = "admin"; pub(crate) const RULES_COMMAND: &str = "rules"; pub(crate) const RULES_LIST_ACTION: &str = "list"; pub(crate) const RULES_SHOW_ACTION: &str = "show"; @@ -87,3 +90,16 @@ pub(crate) const EXIT_FLUSH_FAILURE: &str = "Exception ignored in: <_io.TextIOWrapper name='' mode='w' encoding='utf-8'>"; pub(crate) const EXIT_FLUSH_FAILURE_STATUS: i32 = 120; pub(crate) const UNCAUGHT_EXCEPTION_STATUS: i32 = 1; +/// Reserve half the location work claim for acknowledgement retries after killing a callback. +pub(crate) const SENSOR_EVALUATION_SECONDS: u64 = 30; +pub(crate) const SENSOR_RESULT_BYTES: u64 = 1_048_576; +/// Independent read timeout and delay between automatic history polls. +pub(crate) const METADATA_POLL_SECONDS: u64 = 30; +/// Full key-set reconciliation is independent of the incremental poll and overlap window. +pub(crate) const METADATA_RECONCILE_SECONDS: u32 = 300; +pub(crate) const METADATA_OVERLAP_SECONDS: i64 = 300; +/// A source-key cursor can repeat the last row's base64-encoded definition beside the records. +pub(crate) const METADATA_REPLY_BYTES: u64 = + 3 * sqlbuild_orchestration::api::constants::BODY_BYTES as u64; +/// Bound piped terminal input independently of the credential owner's password policy. +pub(crate) const PASSWORD_INPUT_BYTES: u64 = 4096; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs new file mode 100644 index 0000000000..2a20ef09b2 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs @@ -0,0 +1,83 @@ +//! One task owns a warm interpreter; cancelled requests and source restarts always reap it. + +use std::path::Path; + +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::io::BufReader; +use tokio::sync::mpsc; + +use crate::native_commands::_helpers::execution::{ + exchange_location_generation, location_runtime_child, +}; +use crate::native_commands::models::{LocationCompileRequest, LocationCompiler, LocationServe}; + +pub(crate) async fn run_location_compiler( + options: &LocationServe, + script: &Path, + mut requests: mpsc::Receiver, +) -> Result<(), ControlError> { + let mut compiler: Option = None; + let result = async { + while let Some(mut request) = requests.recv().await { + if compiler + .as_ref() + .is_some_and(|compiler| compiler.python_sources != request.python_sources) + { + stop_compiler(compiler.take()).await?; + } + if compiler.is_none() { + compiler = Some(open_compiler(options, script, request.python_sources)?); + } + let active = compiler + .as_mut() + .ok_or_else(|| ControlError::Invalid("location compiler is unavailable".into()))?; + let result = tokio::select! { + _ = request.reply.closed() => break, + result = exchange_location_generation(active, request.generation) => result, + }; + let failed = result.is_err(); + let _ = request + .reply + .send(result.map_err(|error| error.to_string())); + if failed { + stop_compiler(compiler.take()).await?; + } + } + Ok(()) + } + .await; + stop_compiler(compiler).await?; + result +} + +fn open_compiler( + options: &LocationServe, + script: &Path, + python_sources: Vec, +) -> Result { + let mut child = location_runtime_child(options, script)?; + let stdin = child + .stdin + .take() + .ok_or_else(|| ControlError::Invalid("location interpreter input is unavailable".into()))?; + let stdout = child.stdout.take().ok_or_else(|| { + ControlError::Invalid("location interpreter output is unavailable".into()) + })?; + Ok(LocationCompiler { + child, + stdin, + stdout: BufReader::new(stdout), + python_sources, + }) +} + +async fn stop_compiler(compiler: Option) -> Result<(), ControlError> { + if let Some(mut compiler) = compiler { + if compiler.child.try_wait()?.is_none() { + compiler.child.kill().await?; + } else { + compiler.child.wait().await?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs new file mode 100644 index 0000000000..7ad9339b76 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs @@ -0,0 +1,192 @@ +//! Isolated read-only pages and exact acknowledgement retries share the published input fence. + +use std::future::Future; +use std::path::Path; +use std::time::Duration; + +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::platform::models::{ + LocationWorkClient, LocationWorkPoll, WarehouseSyncFailure, +}; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::warehouse_metadata::models::{ + WarehouseHistoryKind, WarehouseHistoryPage, WarehouseWatermarkRead, +}; +use tokio::time::Instant; + +use crate::argv::models::Stream; +use crate::native_commands::_helpers::execution::{ + acknowledge_metadata_page, freeze_history_page, history_request, location_result, + metadata_health, statistics_page, +}; +use crate::native_commands::_helpers::paths::location_inputs; +use crate::native_commands::_helpers::terminal::write_output; +use crate::native_commands::constants::{ + METADATA_OVERLAP_SECONDS, METADATA_POLL_SECONDS, METADATA_REPLY_BYTES, +}; +use crate::native_commands::models::{LocationMetadataSync, LocationServe, LocationWatch}; + +/// One bounded source or mirror page leaves named work polling a turn between pages. +pub(super) async fn sync_location_metadata( + mut state: LocationMetadataSync, + (options, script, watch): (&LocationServe, &Path, &LocationWatch), + (client, poll): (&LocationWorkClient, &LocationWorkPoll), + stopped: impl Future>, +) -> Result, ControlError> { + if Instant::now() < state.next_at + || state.locations.is_empty() + || location_inputs(&options.project_dir, &watch.rules, &watch.local_files) != watch.observed + { + return Ok(Some(state)); + } + let started = Instant::now(); + tokio::pin!(stopped); + if state.pending.is_none() { + if state.kind() == WarehouseHistoryKind::Statistics && state.statistics.is_none() { + state.statistics = Some(tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + result = statistics_page((client, &options.target), (&state, poll)) => match result { + Ok(page) => page, + Err(error) => return retry_metadata(state, error), + }, + }); + } + if !state.watermark_loaded { + if state.index == 0 { + state.reconcile = Instant::now() >= state.reconcile_at; + } + if !state.reconcile + && !matches!( + state.kind(), + WarehouseHistoryKind::Catalog | WarehouseHistoryKind::Statistics + ) + { + let request = WarehouseWatermarkRead { + session_id: poll.session_id, + generation: state.generation, + location: state.location().clone(), + kind: state.kind(), + }; + let watermark = tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + result = client.metadata_watermark(&options.target, &request) => result, + }; + let watermark = match watermark { + Ok(watermark) => watermark, + Err(error) => { + tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = metadata_health(&state, (client, poll), (started, Some(WarehouseSyncFailure::Watermark))) => {}, + } + return retry_metadata(state, error); + } + }; + state.observed_since = watermark.and_then(|time| { + time.checked_sub_signed(chrono::Duration::seconds(METADATA_OVERLAP_SECONDS)) + }); + } + state.watermark_loaded = true; + } + let Some(page) = read_metadata_page((options, script), &state, stopped.as_mut()).await? + else { + return Ok(None); + }; + let Some(page) = page else { + tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = metadata_health(&state, (client, poll), (started, Some(WarehouseSyncFailure::Read))) => {}, + } + state.retry_read(); + drop(write_output(&[( + Stream::Stderr, + "Warehouse history read failed; retaining mirror and retrying a bounded page.\n", + )])); + return Ok(Some(state)); + }; + if location_inputs(&options.project_dir, &watch.rules, &watch.local_files) != watch.observed + { + return Ok(Some(state)); + } + state = freeze_history_page(state, page, poll)?; + } + let Some(pending) = &state.pending else { + return Ok(Some(state)); + }; + let response = tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + response = client.sync_metadata(&options.target, &pending.batch) => response, + }; + if let Err(error) = response { + tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = metadata_health(&state, (client, poll), (started, Some(WarehouseSyncFailure::Delivery))) => {}, + } + return retry_metadata(state, error); + } + tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = metadata_health(&state, (client, poll), (started, None)) => {}, + } + Ok(Some(acknowledge_metadata_page(state))) +} + +fn retry_metadata( + mut state: LocationMetadataSync, + error: ClientError, +) -> Result, ControlError> { + if matches!(error, ClientError::Configuration(_)) + || matches!(error, ClientError::Rejected(status) if matches!(status.as_u16(), 401 | 403)) + { + return Err(ControlError::Invalid(error.to_string())); + } + state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); + drop(write_output(&[( + Stream::Stderr, + &format!( + "Warehouse history synchronization failed; retaining progress and exact pending page for retry: {error}\n" + ), + )])); + Ok(Some(state)) +} + +/// An outer absence means shutdown; an inner absence is a failed read, never an empty table. +async fn read_metadata_page( + (options, script): (&LocationServe, &Path), + state: &LocationMetadataSync, + stopped: impl Future>, +) -> Result>, ControlError> { + if let Some(statistics) = &state.statistics + && statistics.names.is_empty() + { + return Ok(Some(Some(WarehouseHistoryPage { + records: Vec::new(), + next_after: statistics.next_after.map(|id| vec![Some(id.to_string())]), + }))); + } + let (mut child, reply) = match location_result::>( + (options, script), + (history_request(state)?, METADATA_REPLY_BYTES), + ) { + Ok(exchange) => exchange, + Err(_) => return Ok(Some(None)), + }; + tokio::pin!(stopped); + let outcome = tokio::select! { + result = &mut stopped => { child.kill().await?; result?; return Ok(None); }, + result = tokio::time::timeout(Duration::from_secs(METADATA_POLL_SECONDS), async { + let page = reply.await?; + if !child.wait().await?.success() { + return Err(ControlError::Invalid("location metadata interpreter failed".into())); + } + Ok(page) + }) => result, + }; + if child.try_wait()?.is_none() { + child.kill().await?; + } + let page = match outcome { + Ok(Ok(page)) => page, + Ok(Err(_)) | Err(_) => None, + }; + Ok(Some(page)) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_sensor.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_sensor.rs new file mode 100644 index 0000000000..5c202c077d --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_sensor.rs @@ -0,0 +1,52 @@ +//! A callback gets its own interpreter; timeout and shutdown reap it before reporting a result. + +use std::future::Future; +use std::path::Path; +use std::time::Duration; + +use sqlbuild_orchestration::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; +use sqlbuild_orchestration::store::errors::ControlError; + +use crate::native_commands::_helpers::execution::location_result; +use crate::native_commands::constants::{SENSOR_EVALUATION_SECONDS, SENSOR_RESULT_BYTES}; +use crate::native_commands::models::LocationServe; + +pub(super) async fn evaluate_location_sensor( + options: &LocationServe, + script: &Path, + evaluation: &SensorEvaluationRequest, + stopped: impl Future>, +) -> Result, ControlError> { + let request = + serde_json::to_vec(evaluation).map_err(|error| ControlError::Invalid(error.to_string()))?; + let (mut child, reply) = + match location_result((options, script), (request, SENSOR_RESULT_BYTES)) { + Ok(exchange) => exchange, + Err(_) => return Ok(Some(CustomSensorOutcome::Failed)), + }; + tokio::pin!(stopped); + let outcome = tokio::select! { + result = &mut stopped => { + child.kill().await?; + result?; + return Ok(None); + }, + result = tokio::time::timeout(Duration::from_secs(SENSOR_EVALUATION_SECONDS), async { + let outcome = reply.await?; + if !child.wait().await?.success() { + return Err(ControlError::Invalid("location sensor interpreter failed".into())); + } + Ok(outcome) + }) => { + match result { + Ok(Ok(outcome)) => outcome, + Ok(Err(_)) => CustomSensorOutcome::Failed, + Err(_) => CustomSensorOutcome::TimedOut, + } + }, + }; + if child.try_wait()?.is_none() { + child.kill().await?; + } + Ok(Some(outcome)) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs new file mode 100644 index 0000000000..abf575eb77 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs @@ -0,0 +1,149 @@ +//! Named outbound work shares the watcher's generation boundary, never its interpreter. + +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use chrono::Utc; +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::platform::models::{ + LocationWork, LocationWorkClient, LocationWorkOutcome, LocationWorkPoll, LocationWorkResult, +}; +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::time::Instant; + +use crate::argv::models::Stream; +use crate::native_commands::_helpers::execution::check_location_connection; +use crate::native_commands::_helpers::paths::location_inputs; +use crate::native_commands::_helpers::terminal::write_output; +use crate::native_commands::main::_location_sensor::evaluate_location_sensor; +use crate::native_commands::models::{LocationServe, LocationWatch}; + +/// Retry the exact completed result after uncertain delivery; never rerun it to obtain a new result. +pub(super) async fn run_location_work( + (options, script): (&LocationServe, &Path), + (client, poll, watch): (&LocationWorkClient, &LocationWorkPoll, &LocationWatch), + generation: Option, + stopped: impl Future>, +) -> Result>, ControlError> { + tokio::pin!(stopped); + let claim = tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + result = client.poll(poll) => match result { + Ok(claim) => claim, + Err(error) => { + work_error(&error)?; + return Ok(Some(Vec::new())); + }, + }, + }; + let Some(claim) = claim else { + return Ok(Some(Vec::new())); + }; + let deadline = Instant::now() + + (claim.lease_expires_at - Utc::now()) + .to_std() + .unwrap_or_default(); + let mut files: Vec = Vec::new(); + let outcome = if generation == Some(claim.generation) + && location_inputs(&options.project_dir, &watch.rules, &watch.local_files) == watch.observed + { + let message = match &claim.request.work { + LocationWork::CheckConnection {} => "Checking requested location connection...\n", + LocationWork::EvaluateSensor { .. } => "Evaluating requested custom sensor...\n", + }; + drop(write_output(&[(Stream::Stderr, message)])); + let mut shutting_down = false; + let expires = async { + tokio::select! { + result = &mut stopped => { shutting_down = true; result }, + () = tokio::time::sleep_until(deadline) => Ok(()), + } + }; + let checked = match claim.request.work { + LocationWork::CheckConnection {} => check_location_connection(options, script, expires) + .await? + .map(|(healthy, files)| (LocationWorkOutcome::Connection { healthy }, files)), + LocationWork::EvaluateSensor { evaluation } => { + evaluate_location_sensor(options, script, &evaluation, expires) + .await? + .map(|outcome| { + ( + LocationWorkOutcome::SensorEvaluation { outcome }, + Vec::new(), + ) + }) + } + }; + let Some((outcome, local_files)) = checked else { + drop(write_output(&[( + Stream::Stderr, + "Location work stopped before completion; no result reported.\n", + )])); + return Ok((!shutting_down).then_some(files)); + }; + files = local_files; + let mut expected = watch.observed.clone(); + expected + .0 + .retain(|stamp| !files.contains(&options.project_dir.join(&stamp.relative_path))); + let mut excluded = watch.local_files.clone(); + excluded.extend(files.clone()); + if location_inputs(&options.project_dir, &watch.rules, &excluded) == expected { + outcome + } else { + LocationWorkOutcome::Superseded {} + } + } else { + LocationWorkOutcome::Superseded {} + }; + let result = LocationWorkResult { + claim_id: claim.claim_id, + session_id: poll.session_id, + outcome, + }; + while Instant::now() < deadline { + let completed = tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = tokio::time::sleep_until(deadline) => break, + result = client.complete(claim.request.request_id, &result) => result, + }; + match completed { + Ok(_) => { + drop(write_output(&[( + Stream::Stderr, + "Location work completed.\n", + )])); + return Ok(Some(files)); + } + Err(error) => { + work_error(&error)?; + if matches!(error, ClientError::Rejected(status) if status.is_client_error()) { + break; + } + } + } + tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = tokio::time::sleep(Duration::from_secs(1)) => {}, + } + } + drop(write_output(&[( + Stream::Stderr, + "Location work acknowledgement expired; check the server result before retrying.\n", + )])); + Ok(Some(files)) +} + +fn work_error(error: &ClientError) -> Result<(), ControlError> { + if matches!(error, ClientError::Configuration(_)) + || matches!(error, ClientError::Rejected(status) if matches!(status.as_u16(), 401 | 403)) + { + return Err(ControlError::Invalid(error.to_string())); + } + drop(write_output(&[( + Stream::Stderr, + &format!("Location work request failed: {error}\n"), + )])); + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs new file mode 100644 index 0000000000..c61b8cb137 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -0,0 +1,133 @@ +//! Direct database ownership authorizes bootstrap, not an untrusted HTTP request or supplied role. + +use serde_json::{Value, json}; +use sqlbuild_orchestration::authorization::main::bootstrap_user::bootstrap_user; +use sqlbuild_orchestration::authorization::main::grant_admin::grant_admin; +use sqlbuild_orchestration::credentials::main::identities::create_principal::create_principal; +use sqlbuild_orchestration::credentials::main::locations::create_location::create_location; +use sqlbuild_orchestration::credentials::main::sessions::set_password::set_password; +use sqlbuild_orchestration::credentials::main::tokens::issue_token::issue_token; +use sqlbuild_orchestration::credentials::main::tokens::revoke_token::revoke_token; +use sqlbuild_orchestration::credentials::main::tokens::rotate_token::rotate_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, PasswordAuthority}; +use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; +use sqlbuild_orchestration::declarations::models::TriggerChangeRequest; +use sqlbuild_orchestration::platform::main::location_work::location_work; +use sqlbuild_orchestration::platform::main::request_location_work::request_location_work; +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::request_job_run::request_job_run; +use sqlbuild_orchestration::runs::main::request_rerun::request_rerun; +use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; +use sqlbuild_orchestration::store::errors::ControlError; +use uuid::Uuid; + +use crate::argv::models::Namespace; +use crate::native_commands::_helpers::decision::control_admin; +use crate::native_commands::_helpers::execution::open_control_store; +use crate::native_commands::models::ControlAdmin; + +pub(crate) async fn run_admin(namespace: &Namespace) -> Result { + let action = control_admin(namespace)?; + let store = open_control_store(namespace).await?; + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + match action { + ControlAdmin::BootstrapUser(name) => { + let id = bootstrap_user(&store, &context, &name).await?; + Ok(json!({"principal_id":id,"name":name,"kind":"person","role":"admin"})) + } + ControlAdmin::CreateLocation(name) => { + let id = create_location(&store, &name, &context).await?; + Ok(json!({"location_id": id, "name": name})) + } + ControlAdmin::CreatePrincipal(name) => { + let id = create_principal(&store, &context, &name).await?; + Ok(json!({"principal_id": id, "name": name})) + } + ControlAdmin::IssueToken(scope, expires_at) => { + let token = issue_token(&store, &context, scope, expires_at).await?; + Ok(json!(token)) + } + ControlAdmin::RevokeToken(token_id) => { + let revoked = revoke_token(&store, &context, token_id).await?; + Ok(json!({"token_id": token_id, "revoked": revoked})) + } + ControlAdmin::RotateToken(token_id) => { + Ok(json!(rotate_token(&store, &context, token_id).await?)) + } + ControlAdmin::SetPassword(principal_id, password) => { + set_password( + &store, + &context, + PasswordAuthority::Operator(principal_id), + &password, + ) + .await?; + Ok(json!({"principal_id": principal_id, "password_set": true})) + } + ControlAdmin::GrantAdmin(principal_id) => { + grant_admin(&store, &context, principal_id).await?; + Ok(json!({"principal_id": principal_id, "role": "admin"})) + } + ControlAdmin::RequestJobRun(request) => { + let applied = request_job_run(&store, &context, &request).await?; + Ok(json!({"run_id": request.run_id, "applied": applied})) + } + ControlAdmin::Rerun(request) => { + let applied = request_rerun(&store, (&context, None), &request).await?; + Ok(json!({"run_id": request.run_id, "applied": applied})) + } + ControlAdmin::RequestLocationWork(request) => { + let applied = request_location_work(&store, (&context, None), &request).await?; + Ok(json!({"request_id": request.request_id, "applied": applied})) + } + ControlAdmin::LocationWork(request_id) => { + Ok(json!(location_work(&store, request_id).await?)) + } + ControlAdmin::CancelRun { + event_id, + run_id, + reason, + } => { + let applied = change_run( + &store, + RunAuthority::Operator(&context), + &RunChangeRequest { + event_id, + run_id, + claim_id: None, + change: RunChange::CancelRequested { + actor: context.actor.clone(), + reason, + }, + }, + ) + .await?; + Ok(json!({"event_id": event_id, "run_id": run_id, "applied": applied})) + } + ControlAdmin::ChangeTrigger { + event_id, + trigger_id, + change, + reason, + } => { + let applied = change_trigger( + &store, + &context, + &TriggerChangeRequest { + event_id, + trigger_id, + actor: context.actor.clone(), + reason, + change, + }, + ) + .await?; + Ok(json!({"event_id": event_id, "trigger_id": trigger_id, "applied": applied})) + } + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs new file mode 100644 index 0000000000..d5a88fd9d5 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -0,0 +1,80 @@ +//! Supervise native API/scheduler roles together; a failed role stops its siblings. + +use sqlbuild_orchestration::api::main::serve_api::serve_api; +use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; +use sqlbuild_orchestration::scheduler::models::SchedulerState; +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::sync::watch; +use uuid::Uuid; + +use crate::argv::models::{Namespace, Stream}; +use crate::native_commands::_helpers::decision::{control_address, control_api_settings, flag}; +use crate::native_commands::_helpers::execution::{ + open_control_store, open_control_workers, start_platform_roles, supervise_control_roles, +}; +use crate::native_commands::_helpers::terminal::write_output; + +async fn stopped(mut shutdown: watch::Receiver) { + while !*shutdown.borrow_and_update() { + if shutdown.changed().await.is_err() { + break; + } + } +} + +pub(crate) async fn run_control( + namespace: &Namespace, + standalone: bool, +) -> Result<(), ControlError> { + let address = control_address(namespace, standalone)?; + let (authentication, (statistics, audit, snapshots)) = control_api_settings(namespace)?; + let store = open_control_store(namespace).await?; + let (workers, images) = open_control_workers(namespace)?; + let (stop, shutdown) = watch::channel(false); + let mut tasks = tokio::task::JoinSet::new(); + if let Some(address) = address { + drop(write_output(&[( + Stream::Stderr, + "Starting control API...\n", + )])); + let listener = tokio::net::TcpListener::bind(address).await?; + let address = listener.local_addr()?; + tasks.spawn(serve_api( + store.clone(), + listener, + (authentication, images, statistics), + ( + move || { + drop(write_output(&[( + Stream::Stderr, + &format!("Control API ready: {address}\n"), + )])); + }, + stopped(shutdown.clone()), + ), + )); + start_platform_roles( + &mut tasks, + (&store, &shutdown), + (audit, statistics, snapshots), + )?; + } + if standalone || !flag(namespace, "control_no_scheduler") { + drop(write_output(&[(Stream::Stderr, "Starting scheduler...\n")])); + let scheduler_store = store.clone(); + tasks.spawn(async move { + run_scheduler( + &scheduler_store, + SchedulerState::new(Uuid::now_v7()), + shutdown, + workers, + ) + .await + }); + drop(write_output(&[( + Stream::Stderr, + "Scheduler loop started; leadership is lease-guarded\n", + )])); + } + supervise_control_roles(tasks, stop).await +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_dev.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_dev.rs new file mode 100644 index 0000000000..e852ca458c --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_dev.rs @@ -0,0 +1,82 @@ +//! Collapse the production API, scheduler and outbound location into one local process tree. + +use std::path::Path; + +use sqlbuild_orchestration::api::models::DevelopmentScope; +use sqlbuild_orchestration::credentials::main::locations::development_location::development_location; +use sqlbuild_orchestration::credentials::models::AuditContext; +use sqlbuild_orchestration::declarations::main::pause_development_triggers::pause_development_triggers; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::{migrate_store::migrate_store, open_store::open_store}; +use sqlbuild_orchestration::store::models::ControlStore; +use uuid::Uuid; + +use crate::argv::models::{Namespace, Stream}; +use crate::native_commands::_helpers::decision::{control_address, flag, text}; +use crate::native_commands::_helpers::execution::run_development_roles; +use crate::native_commands::_helpers::project_facts::development_target; +use crate::native_commands::_helpers::terminal::write_output; + +pub(super) async fn run_dev(namespace: &Namespace) -> Result<(), ControlError> { + let project = + std::fs::canonicalize(text(namespace, "project_dir").unwrap_or_else(|| ".".into()))?; + let target = development_target(&project, text(namespace, "target"))?; + let address = control_address(namespace, false)? + .ok_or_else(|| ControlError::Invalid("development address is required".into()))?; + if !address.ip().is_loopback() { + return Err(ControlError::Invalid( + "sqb dev only listens on loopback".into(), + )); + } + let executable = std::env::current_exe()?; + drop(write_output(&[( + Stream::Stderr, + "Starting development control store...\n", + )])); + let (store, scope) = development_store(namespace, &project, &target).await?; + drop(write_output(&[( + Stream::Stderr, + "Development control store ready\nStarting development API...\n", + )])); + let listener = tokio::net::TcpListener::bind(address).await?; + run_development_roles( + (store, scope), + (project, executable), + listener, + !flag(namespace, "control_no_scheduler"), + ) + .await +} + +/// Complete retained identity and startup pauses before starting any scheduling roles. +async fn development_store( + namespace: &Namespace, + project: &Path, + target: &str, +) -> Result<(ControlStore, DevelopmentScope), ControlError> { + let database = text(namespace, "development_store_url").unwrap_or_else(|| { + project + .join(".sqlbuild/control.sqlite3") + .to_string_lossy() + .into_owned() + }); + let store = open_store(&database).await?; + migrate_store(&store).await?; + let context = AuditContext { + actor: "local-development".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let location_id = + development_location(&store, &project.to_string_lossy(), target, &context).await?; + let scope = DevelopmentScope { + location_id, + target: target.into(), + triggers: flag(namespace, "development_triggers"), + }; + if !scope.triggers { + pause_development_triggers(&store, (location_id, target)).await?; + } + Ok((store, scope)) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs new file mode 100644 index 0000000000..43e3a3f2c0 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs @@ -0,0 +1,185 @@ +//! Coalesce source edits and retain the last good publication across isolated compiler failures. + +use std::future::Future; +use std::path::Path; +use std::time::Duration; + +use sqlbuild_orchestration::platform::main::connect_location_work::connect_location_work; +use sqlbuild_orchestration::platform::models::{ + LocationHealth, LocationWorkClient, LocationWorkPoll, +}; +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::sync::mpsc; +use tokio::time::{Instant, MissedTickBehavior}; +use uuid::Uuid; + +use crate::argv::models::Stream; +use crate::native_commands::_helpers::execution::{ + check_published_location, compile_generation, location_generation, location_heartbeats, + metadata_sync, report_location_health, shutdown_signal, +}; +use crate::native_commands::_helpers::paths::{ + exclude_location_databases, location_watch, observe_location_inputs, +}; +use crate::native_commands::_helpers::terminal::write_output; +use crate::native_commands::main::_location_compiler::run_location_compiler; +use crate::native_commands::main::_location_metadata::sync_location_metadata; +use crate::native_commands::main::_location_work::run_location_work; +use crate::native_commands::models::{ + LocationCompileRequest, LocationMetadataSync, LocationServe, LocationWatch, +}; + +pub(super) async fn run_location( + options: LocationServe, + script: &Path, + bearer: &str, +) -> Result<(), ControlError> { + let session_id = Uuid::now_v7(); + let client = connect_location_work((&options.server_url, bearer), options.location_id) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let poll = LocationWorkPoll { + session_id, + target: options.target.clone(), + }; + let (requests, receiver) = mpsc::channel(1); + let serving = async { + tokio::select! { + result = watch_location(&options, script, (bearer, &client, &poll), requests) => result, + result = location_heartbeats(&options, bearer, session_id) => result, + } + }; + let (serving, compiler) = + tokio::join!(serving, run_location_compiler(&options, script, receiver)); + serving.and(compiler) +} + +async fn watch_location( + options: &LocationServe, + script: &Path, + (bearer, client, poll): (&str, &LocationWorkClient, &LocationWorkPoll), + requests: mpsc::Sender, +) -> Result<(), ControlError> { + let mut watch = location_watch(&options.project_dir); + let mut work_at = Instant::now() + Duration::from_secs(2); + let mut published_generation = None; + let mut metadata = None; + let mut clock = tokio::time::interval(Duration::from_millis(250)); + clock.set_missed_tick_behavior(MissedTickBehavior::Skip); + let stopped = shutdown_signal(); + tokio::pin!(stopped); + drop(write_output(&[( + Stream::Stderr, + "Starting code location; watching project inputs...\n", + )])); + loop { + tokio::select! { + result = &mut stopped => return result, + _ = clock.tick() => {}, + } + let ready; + (watch, ready) = observe_location_inputs(&options.project_dir, watch); + if !ready { + let Some(idle) = poll_location( + (options, script), + (client, poll, published_generation), + (watch, work_at, metadata), + stopped.as_mut(), + ) + .await? + else { + return Ok(()); + }; + (watch, work_at, metadata) = idle; + continue; + } + published_generation = None; + let generation = tokio::select! { + result = &mut stopped => return result, + generation = location_generation(options, bearer) => generation?, + }; + let generation = match generation { + Some(generation) => generation, + None => { + watch.retry_at = Some(Instant::now() + Duration::from_secs(5)); + continue; + } + }; + drop(write_output(&[( + Stream::Stderr, + &format!("Compiling location generation {generation} in an isolated interpreter...\n"), + )])); + let Some((published, refresh_at, locations)) = + compile_generation(&requests, generation, &watch.observed.0, stopped.as_mut()).await? + else { + return Ok(()); + }; + if published { + watch.retry_at = refresh_at; + published_generation = Some(generation); + metadata = Some(metadata_sync( + (locations, generation), + options.metadata_reconcile_seconds, + )); + let Some(files) = check_published_location( + options, + script, + (bearer, poll.session_id, generation), + stopped.as_mut(), + ) + .await? + else { + return Ok(()); + }; + watch = exclude_location_databases(&options.project_dir, files, watch); + } else { + watch.retry_at = Some(Instant::now() + Duration::from_secs(30)); + tokio::select! { + result = &mut stopped => return result, + () = report_location_health(options, bearer, poll.session_id, LocationHealth::CompileFailed { generation }) => {}, + } + drop(write_output(&[( + Stream::Stderr, + "Location reload failed; retaining last good snapshot. Retrying on change or in 30 seconds.\n", + )])); + } + } +} + +/// Named work and metadata share one idle turn without transferring interpreter ownership. +async fn poll_location( + (options, script): (&LocationServe, &Path), + (client, poll, generation): (&LocationWorkClient, &LocationWorkPoll, Option), + (mut watch, mut work_at, mut metadata): (LocationWatch, Instant, Option), + stopped: impl Future>, +) -> Result)>, ControlError> { + tokio::pin!(stopped); + if generation.is_some() && watch.attempted.as_ref() == Some(&watch.observed) { + if Instant::now() >= work_at { + let Some(files) = run_location_work( + (options, script), + (client, poll, &watch), + generation, + stopped.as_mut(), + ) + .await? + else { + return Ok(None); + }; + watch = exclude_location_databases(&options.project_dir, files, watch); + work_at = Instant::now() + Duration::from_secs(2); + } + if let Some(state) = metadata.take() { + metadata = sync_location_metadata( + state, + (options, script, &watch), + (client, poll), + stopped.as_mut(), + ) + .await?; + if metadata.is_none() { + return Ok(None); + } + } + } + Ok(Some((watch, work_at, metadata))) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index 034b2c7bcd..c5fb741b92 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -1,3 +1,16 @@ pub(super) mod _cost_report_options; +pub(super) mod _location_compiler; +pub(super) mod _location_metadata; +pub(super) mod _location_sensor; +pub(super) mod _location_work; +pub(super) mod _run_admin; +pub(super) mod _run_control; pub(super) mod _run_cost; +pub(super) mod _run_dev; +pub(super) mod _run_location; +pub(crate) mod parse_native_invocation; +pub(crate) mod run_control_invocation; +pub(crate) mod run_location_invocation; +pub(crate) mod run_login_invocation; pub mod run_native_invocation; +pub(crate) mod run_person_invocation; diff --git a/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs new file mode 100644 index 0000000000..8c2056d976 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs @@ -0,0 +1,59 @@ +//! Parse once with the shared command table before resolving the optional Python CLI. + +use std::ffi::OsString; + +use crate::argv::main::parse_cli_arguments::parse_cli_arguments; +use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace}; +use crate::native_commands::_helpers::argv_host::{NativeArgvHost, namespace_defaults}; +use crate::native_commands::_helpers::decision::{review_json_failure, text}; +use crate::native_commands::_helpers::terminal::{ + finish_rejected_parse, stdout_supports_color, terminal_columns, +}; +use crate::native_commands::constants::{FALLBACK_COLUMNS, NO_COLOR_OPTION}; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::NativeParse; + +pub(crate) fn parse_native_invocation(arguments: &[OsString]) -> Result { + let arguments: Vec = arguments + .iter() + .map(|argument| argument.to_str().map(str::to_owned)) + .collect::>() + .ok_or(PythonNeeded::Arguments)?; + let use_color = + !arguments.iter().any(|argument| argument == NO_COLOR_OPTION) && stdout_supports_color(); + let columns = terminal_columns(); + let needs_python_width = columns.is_err(); + let settings = ArgvSettings { + columns: columns.unwrap_or(FALLBACK_COLUMNS), + use_color, + }; + let mut host = NativeArgvHost::default(); + let outcome: ArgvOutcome = + parse_cli_arguments(&arguments, namespace_defaults(), settings, &mut host); + if needs_python_width + && !outcome.parsed.as_ref().is_ok_and(|namespace| { + text(namespace, "command").as_deref() == Some("location") + && text(namespace, "location_command").as_deref() == Some("serve") + }) + { + return Err(PythonNeeded::Arguments); + } + if host.needs_python { + return Err(PythonNeeded::Arguments); + } + let namespace: Namespace = match outcome.parsed { + Err(_) if review_json_failure(&arguments, &outcome) => return Err(PythonNeeded::Arguments), + Err(code) => { + return Ok(NativeParse::Exit(finish_rejected_parse( + &outcome.writes, + code, + ))); + } + Ok(namespace) if outcome.writes.is_empty() => namespace, + Ok(_) => return Err(PythonNeeded::Arguments), + }; + Ok(NativeParse::Invocation { + namespace, + use_color, + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs new file mode 100644 index 0000000000..a77a31147f --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs @@ -0,0 +1,83 @@ +//! Control roles are served before Python/package discovery, including their JSON contract. + +use sqlbuild_orchestration::api::main::openapi_document::openapi_document; + +use crate::argv::models::{Namespace, Stream}; +use crate::native_commands::_helpers::decision::{flag, text}; +use crate::native_commands::_helpers::execution::run_control_action; +use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output}; +use crate::native_commands::constants::{ADMIN_COMMAND, SCHEDULER_COMMAND, SERVER_COMMAND}; + +pub(crate) fn run_control_invocation(namespace: &Namespace) -> Option { + let command = text(namespace, "command")?; + if !matches!( + command.as_str(), + SERVER_COMMAND | SCHEDULER_COMMAND | ADMIN_COMMAND + ) { + return None; + } + if flag(namespace, "control_openapi") { + return Some(match serde_json::to_string(&openapi_document()) { + Ok(document) => { + let document = document + "\n"; + let lost = write_output(&[(Stream::Stdout, document.as_str())]); + finish_lost_stdout(lost.as_ref(), 0) + } + Err(_) => { + drop(write_output(&[( + Stream::Stderr, + "Control API contract generation failed\n", + )])); + 1 + } + }); + } + let runtime = match tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("Control service failed to start: {error}\n"), + )])); + return Some(1); + } + }; + let admin = command == ADMIN_COMMAND; + let role = if admin { "administration" } else { "service" }; + let result = runtime.block_on(run_control_action(namespace, &command)); + Some(match result { + Ok(output) => { + let lost = + output.and_then(|value| write_output(&[(Stream::Stdout, &format!("{value}\n"))])); + if admin && lost.is_some() { + drop(write_output(&[( + Stream::Stderr, + &format!( + "Control {role} JSON result delivery failed; changes may already be committed\n" + ), + )])); + return Some(1); + } + let exit_code = finish_lost_stdout(lost.as_ref(), 0); + if exit_code == 0 { + let message = if admin { + "Control administration completed successfully\n" + } else { + "Control service stopped successfully\n" + }; + drop(write_output(&[(Stream::Stderr, message)])); + } + exit_code + } + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("Control {role} failed: {error}\n"), + )])); + 1 + } + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_location_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_location_invocation.rs new file mode 100644 index 0000000000..ac052a6b55 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/run_location_invocation.rs @@ -0,0 +1,65 @@ +//! A location supervisor stays native while each compilation uses the installed project interpreter. + +use std::path::Path; + +use crate::argv::models::{Namespace, Stream}; +use crate::native_commands::_helpers::decision::text; +use crate::native_commands::_helpers::execution::location_options; +use crate::native_commands::_helpers::terminal::write_output; +use crate::native_commands::main::_run_dev::run_dev; +use crate::native_commands::main::_run_location::run_location; + +pub(crate) fn run_location_invocation(namespace: &Namespace, script: &Path) -> Option { + let development = text(namespace, "command").as_deref() == Some("dev"); + if !development + && (text(namespace, "command").as_deref() != Some("location") + || text(namespace, "location_command").as_deref() != Some("serve")) + { + return None; + } + let result = (|| { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; + if development { + return runtime.block_on(run_dev(namespace)); + } + let options = location_options(namespace)?; + let bearer = std::env::var("SQLBUILD_LOCATION_TOKEN").map_err(|_| { + std::io::Error::other( + "SQLBUILD_LOCATION_TOKEN must contain a UTF-8 location credential", + ) + })?; + if bearer.trim().is_empty() { + return Err(std::io::Error::other("SQLBUILD_LOCATION_TOKEN must not be blank").into()); + } + runtime.block_on(run_location(options, script, &bearer)) + })(); + Some(match result { + Ok(()) => { + drop(write_output(&[( + Stream::Stderr, + if development { + "Development services stopped successfully\n" + } else { + "Code location stopped successfully\n" + }, + )])); + 0 + } + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!( + "{} failed: {error}\n", + if development { + "Development services" + } else { + "Code location" + } + ), + )])); + 1 + } + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_login_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_login_invocation.rs new file mode 100644 index 0000000000..62b3216e37 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/run_login_invocation.rs @@ -0,0 +1,48 @@ +//! Login resolves its native command options without project discovery or invocation capture. + +use crate::argv::models::{Namespace, Stream}; +use crate::native_commands::_helpers::decision::{flag, text}; +use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output}; +use crate::person_login::main::run_login::run_login; +use crate::person_login::models::{LoginAction, LoginOptions}; + +pub(crate) fn run_login_invocation(namespace: &Namespace) -> Option { + let command = text(namespace, "command")?; + let action = match command.as_str() { + "logout" => LoginAction::Logout, + "login" if text(namespace, "login_action").as_deref() == Some("status") => { + LoginAction::Status + } + "login" => LoginAction::Start, + _ => return None, + }; + Some( + match run_login( + LoginOptions { + action, + server: text(namespace, "login_server").unwrap_or_default(), + ca: text(namespace, "login_ca").map(Into::into), + no_browser: flag(namespace, "login_no_browser"), + }, + |message| { + drop(write_output(&[(Stream::Stderr, message)])); + }, + ) { + Ok(Some(identity)) => match serde_json::to_string(&identity) { + Ok(identity) => { + let lost = write_output(&[(Stream::Stdout, &format!("{identity}\n"))]); + finish_lost_stdout(lost.as_ref(), 0) + } + Err(_) => 1, + }, + Ok(None) => 0, + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("SQLBuild CLI {command} failed: {error}\n"), + )])); + 1 + } + }, + ) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs index 40360a7de8..c92474afef 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs @@ -1,50 +1,19 @@ //! Serve one invocation without Python when its parse and project facts allow it. -use std::ffi::OsString; use std::path::Path; -use crate::argv::main::parse_cli_arguments::parse_cli_arguments; -use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace}; -use crate::native_commands::_helpers::argv_host::{NativeArgvHost, namespace_defaults}; -use crate::native_commands::_helpers::decision::{native_invocation, review_json_failure}; +use crate::argv::models::Namespace; +use crate::native_commands::_helpers::decision::native_invocation; use crate::native_commands::_helpers::execution::run_command; -use crate::native_commands::_helpers::terminal::{ - finish_rejected_parse, stdout_supports_color, terminal_columns, -}; -use crate::native_commands::constants::NO_COLOR_OPTION; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::NativeInvocation; /// The exit status of an invocation `sqb` served, or why the same arguments go to Python. pub fn run_native_invocation( - arguments: &[OsString], + namespace: &Namespace, python_cli: &Path, + use_color: bool, ) -> Result { - let arguments: Vec = arguments - .iter() - .map(|argument| argument.to_str().map(str::to_owned)) - .collect::>() - .ok_or(PythonNeeded::Arguments)?; - let use_color: bool = - !arguments.iter().any(|argument| argument == NO_COLOR_OPTION) && stdout_supports_color(); - let settings: ArgvSettings = ArgvSettings { - columns: terminal_columns()?, - use_color, - }; - let mut host: NativeArgvHost = NativeArgvHost::default(); - let outcome: ArgvOutcome = - parse_cli_arguments(&arguments, namespace_defaults(), settings, &mut host); - if host.needs_python { - return Err(PythonNeeded::Arguments); - } - let namespace: Namespace = match outcome.parsed { - Err(_) if review_json_failure(&arguments, &outcome) => { - return Err(PythonNeeded::Arguments); - } - Err(code) => return Ok(finish_rejected_parse(&outcome.writes, code)), - Ok(namespace) if outcome.writes.is_empty() => namespace, - Ok(_) => return Err(PythonNeeded::Arguments), - }; - let invocation: NativeInvocation = native_invocation(&namespace)?; + let invocation: NativeInvocation = native_invocation(namespace)?; run_command(&invocation, python_cli, use_color) } diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_person_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_person_invocation.rs new file mode 100644 index 0000000000..036e68a8d5 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/run_person_invocation.rs @@ -0,0 +1,60 @@ +//! Person control commands use the saved HTTPS session, never privileged database access. + +use crate::argv::models::{Namespace, Stream}; +use crate::native_commands::_helpers::decision::text; +use crate::native_commands::_helpers::terminal::write_output; +use crate::person_login::main::read_node_sql::read_node_sql; + +pub(crate) fn run_person_invocation(namespace: &Namespace) -> Option { + if text(namespace, "command").as_deref() != Some("control") { + return None; + } + drop(write_output(&[( + Stream::Stderr, + "Reading published SQL…\n", + )])); + let result = (|| { + let location = text(namespace, "person_location") + .unwrap_or_default() + .parse() + .map_err(|_| std::io::Error::other("--location-id must be a UUID"))?; + let ca = text(namespace, "login_ca").map(std::path::PathBuf::from); + let sql = read_node_sql( + ( + &text(namespace, "login_server").unwrap_or_default(), + ca.as_deref(), + ), + ( + location, + &text(namespace, "target").unwrap_or_default(), + &text(namespace, "person_node").unwrap_or_default(), + ), + )?; + serde_json::to_string(&sql).map_err(std::io::Error::other) + })(); + Some(match result { + Ok(json) => { + let lost = write_output(&[(Stream::Stdout, &format!("{json}\n"))]); + if lost.is_some() { + drop(write_output(&[( + Stream::Stderr, + "SQL inspection failed: could not write JSON to stdout\n", + )])); + 1 + } else { + drop(write_output(&[( + Stream::Stderr, + "Published SQL read successfully\n", + )])); + 0 + } + } + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("SQL inspection failed: {error}\n"), + )])); + 1 + } + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/mod.rs b/crates/sqlbuild-cli/src/native_commands/mod.rs index d99ca200f2..5ac20489d5 100644 --- a/crates/sqlbuild-cli/src/native_commands/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod constants; pub mod errors; pub mod main; pub(crate) mod models; +pub(crate) mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 93db875a33..61f4d42fd2 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -2,6 +2,57 @@ use std::path::PathBuf; +use chrono::{DateTime, Utc}; +use sqlbuild_cache::project_snapshot::models::{PathStamp, SnapshotRules}; +use sqlbuild_orchestration::credentials::models::CredentialScope; +use sqlbuild_orchestration::declarations::models::TriggerChange; +use sqlbuild_orchestration::platform::models::LocationWorkRequest; +use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest}; +use sqlbuild_orchestration::warehouse_metadata::models::{ + MetadataBatch, StatisticsRead, WarehouseHistoryKind, WarehouseStateLocation, +}; +use uuid::Uuid; + +use crate::argv::models::Namespace; +use crate::native_commands::types::LocationPublication; + +/// Privileged local operations require direct ownership of the control database. +pub(crate) enum ControlAdmin { + CreateLocation(String), + CreatePrincipal(String), + BootstrapUser(String), + IssueToken(CredentialScope, DateTime), + RevokeToken(Uuid), + RotateToken(Uuid), + SetPassword(Uuid, String), + GrantAdmin(Uuid), + RequestJobRun(JobRunRequest), + Rerun(RerunRequest), + RequestLocationWork(LocationWorkRequest), + LocationWork(Uuid), + CancelRun { + event_id: Uuid, + run_id: Uuid, + reason: String, + }, + ChangeTrigger { + event_id: Uuid, + trigger_id: Uuid, + change: TriggerChange, + reason: String, + }, +} + +/// Parsing can finish an invocation before package discovery or any Python handoff. +#[derive(Debug)] +pub(crate) enum NativeParse { + Exit(i32), + Invocation { + namespace: Namespace, + use_color: bool, + }, +} + /// What `sqb skills` installs. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct SkillsOptions { @@ -88,6 +139,101 @@ pub(crate) struct ProjectLocation { pub(crate) resolved: PathBuf, } +/// A long-running location owns no control-store credentials or executable user objects. +#[derive(Debug)] +pub(crate) struct LocationServe { + pub(crate) project_dir: PathBuf, + pub(crate) server_url: String, + pub(crate) location_id: uuid::Uuid, + pub(crate) target: String, + pub(crate) image_id: Option, + pub(crate) metadata_reconcile_seconds: u32, +} + +/// Retained source observations survive compilation; warehouse exclusions never replace them. +#[derive(Debug)] +pub(crate) struct LocationWatch { + pub(crate) rules: SnapshotRules, + pub(crate) local_files: Vec, + pub(crate) observed: (Vec, Option), + pub(crate) attempted: Option<(Vec, Option)>, + pub(crate) changed_at: tokio::time::Instant, + pub(crate) retry_at: Option, +} + +/// A bounded supervisor request; dropping the reply cancels and reaps an active compiler. +#[derive(Debug)] +pub(crate) struct LocationCompileRequest { + pub(crate) generation: i64, + pub(crate) python_sources: Vec, + pub(crate) reply: tokio::sync::oneshot::Sender, String>>, +} + +/// Only this task owns the isolated interpreter and its private stdio protocol. +#[derive(Debug)] +pub(crate) struct LocationCompiler { + pub(crate) child: tokio::process::Child, + pub(crate) stdin: tokio::process::ChildStdin, + pub(crate) stdout: tokio::io::BufReader, + pub(crate) python_sources: Vec, +} + +/// A delivered source page remains frozen until the API acknowledges it. +#[derive(Debug)] +pub(crate) struct LocationMetadataPage { + pub(crate) batch: MetadataBatch, + pub(crate) next_source_after: Option>>, +} + +/// Only local retry and navigation state; the control store keeps no scan progress. +#[derive(Debug)] +pub(crate) struct LocationMetadataSync { + pub(crate) locations: Vec, + pub(crate) generation: i64, + pub(crate) index: usize, + pub(crate) scan_id: Uuid, + pub(crate) source_after: Option>>, + pub(crate) observed_since: Option>, + pub(crate) watermark_loaded: bool, + pub(crate) statistics: Option, + pub(crate) reconcile: bool, + pub(crate) reconcile_at: tokio::time::Instant, + pub(crate) reconcile_seconds: u32, + pub(crate) limit: usize, + pub(crate) pending: Option, + pub(crate) next_at: tokio::time::Instant, +} + +impl LocationMetadataSync { + /// Failed samples revisit catalog before retrying; no frozen delivery is discarded. + pub(crate) fn retry_read(&mut self) { + self.limit = (self.limit / 2).max(1); + self.statistics = None; + if self.kind() == WarehouseHistoryKind::Statistics { + self.index -= 1; + self.source_after = None; + self.scan_id = Uuid::now_v7(); + self.observed_since = None; + self.watermark_loaded = false; + } + self.next_at = tokio::time::Instant::now() + + std::time::Duration::from_secs(super::constants::METADATA_POLL_SECONDS); + } + + pub(crate) fn reconciling(&self) -> bool { + self.kind() != WarehouseHistoryKind::Statistics + && (self.reconcile || self.kind() == WarehouseHistoryKind::Catalog) + } + + pub(crate) fn kind(&self) -> WarehouseHistoryKind { + WarehouseHistoryKind::ALL[self.index % WarehouseHistoryKind::ALL.len()] + } + + pub(crate) fn location(&self) -> &WarehouseStateLocation { + &self.locations[self.index / WarehouseHistoryKind::ALL.len()] + } +} + /// What Python's buffered stdout does with output its reader no longer takes. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum LostStdout { diff --git a/crates/sqlbuild-cli/src/native_commands/types.rs b/crates/sqlbuild-cli/src/native_commands/types.rs new file mode 100644 index 0000000000..862c4aa10d --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/types.rs @@ -0,0 +1,12 @@ +//! Private interpreter protocol shapes owned by the location supervisor. + +use chrono::{DateTime, Utc}; +use sqlbuild_orchestration::snapshots::models::SnapshotRecord; +use sqlbuild_orchestration::warehouse_metadata::models::WarehouseStateLocation; + +/// Publication metadata and the earliest calendar refresh deadline. +pub(crate) type LocationPublication = ( + SnapshotRecord, + Option>, + Vec, +); diff --git a/crates/sqlbuild-cli/src/person_login/_helpers/credentials.rs b/crates/sqlbuild-cli/src/person_login/_helpers/credentials.rs new file mode 100644 index 0000000000..de13ef5101 --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/_helpers/credentials.rs @@ -0,0 +1,173 @@ +//! Prefer the OS credential store; the headless Linux fallback is one owner-only 0600 file. + +use crate::person_login::constants::KEYRING_SERVICE; +use crate::person_login::models::PersonCredential; + +fn headless_linux() -> bool { + cfg!(target_os = "linux") + && std::env::var_os("DBUS_SESSION_BUS_ADDRESS").is_none() + && std::env::var_os("XDG_RUNTIME_DIR").is_none() +} + +pub(crate) fn load(origin: &str) -> Result, std::io::Error> { + let encoded = match keyring::Entry::new(KEYRING_SERVICE, origin) + .and_then(|entry| entry.get_password()) + { + Ok(value) => Some(value), + Err(keyring::Error::NoEntry) => private_file(origin, None)?, + Err(keyring::Error::PlatformFailure(_)) if headless_linux() => private_file(origin, None)?, + Err(_) => { + return Err(std::io::Error::other( + "OS credential store could not be read; unlock it before retrying", + )); + } + }; + encoded + .map(|value| { + serde_json::from_str(&value).map_err(|_| { + std::io::Error::other("stored CLI credential is invalid; run sqb logout") + }) + }) + .transpose() +} + +pub(crate) fn save(origin: &str, credential: &PersonCredential) -> Result { + let encoded = serde_json::to_string(credential) + .map_err(|_| std::io::Error::other("could not encode the CLI credential"))?; + match keyring::Entry::new(KEYRING_SERVICE, origin) + .and_then(|entry| entry.set_password(&encoded)) + { + Ok(()) => { + remove_file(origin)?; + Ok(true) + } + Err(keyring::Error::PlatformFailure(_)) if headless_linux() => { + private_file(origin, Some(&encoded))?; + Ok(false) + } + Err(_) => Err(std::io::Error::other( + "OS credential store rejected persistence; unlock it before retrying", + )), + } +} + +pub(crate) fn remove(origin: &str) -> Result<(), std::io::Error> { + let removed = match keyring::Entry::new(KEYRING_SERVICE, origin) + .and_then(|entry| entry.delete_credential()) + { + Ok(()) | Err(keyring::Error::NoEntry) => Ok(()), + Err(keyring::Error::PlatformFailure(_)) if headless_linux() => Ok(()), + Err(_) => Err(std::io::Error::other( + "OS credential store cleanup could not be confirmed", + )), + }; + remove_file(origin)?; + removed +} + +#[cfg(target_os = "linux")] +fn path(origin: &str, create: bool) -> Result, std::io::Error> { + use std::os::unix::fs::DirBuilderExt; + let Some(home) = std::env::var_os("HOME").filter(|value| !value.is_empty()) else { + return if create { + Err(std::io::Error::other( + "a user home directory is required for credential storage", + )) + } else { + Ok(None) + }; + }; + let directory = std::path::PathBuf::from(home).join(".config/sqlbuild/credentials"); + if !directory.is_absolute() { + return Err(std::io::Error::other( + "the credential home directory must be absolute", + )); + } + if create { + std::fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(&directory)?; + } + Ok(Some(directory.join(format!( + "{}.json", + blake3::hash(origin.as_bytes()).to_hex() + )))) +} + +#[cfg(target_os = "linux")] +fn private_file(origin: &str, encoded: Option<&str>) -> Result, std::io::Error> { + use std::io::{Read, Write}; + use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; + let Some(path) = path(origin, encoded.is_some())? else { + return Ok(None); + }; + if let Some(encoded) = encoded { + let mut bytes = [0_u8; 16]; + getrandom::fill(&mut bytes).map_err(std::io::Error::other)?; + let temporary = path.with_extension(format!("{}.tmp", uuid::Uuid::from_bytes(bytes))); + let result = (|| { + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&temporary)?; + file.write_all(encoded.as_bytes())?; + file.sync_all()?; + std::fs::rename(&temporary, &path) + })(); + if result.is_err() { + let _ = std::fs::remove_file(&temporary); + } + result?; + return Ok(None); + } + let mut file = match std::fs::OpenOptions::new() + .read(true) + .custom_flags(rustix::fs::OFlags::NOFOLLOW.bits() as i32) + .open(&path) + { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + let metadata = file.metadata()?; + if !metadata.is_file() + || metadata.uid() != rustix::process::geteuid().as_raw() + || metadata.mode() & 0o077 != 0 + { + return Err(std::io::Error::other( + "CLI credential file must be owned by you and accessible only to you", + )); + } + let mut encoded = String::new(); + file.read_to_string(&mut encoded)?; + Ok(Some(encoded)) +} + +#[cfg(target_os = "linux")] +fn remove_file(origin: &str) -> Result { + let Some(path) = path(origin, false)? else { + return Ok(false); + }; + match std::fs::remove_file(path) { + Ok(()) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + +#[cfg(not(target_os = "linux"))] +fn private_file(_origin: &str, encoded: Option<&str>) -> Result, std::io::Error> { + match encoded { + None => Ok(None), + Some(_) => Err(std::io::Error::other( + "an available OS credential store is required on this platform", + )), + } +} + +#[cfg(not(target_os = "linux"))] +fn remove_file(_origin: &str) -> Result { + Ok(false) +} diff --git a/crates/sqlbuild-cli/src/person_login/_helpers/flow.rs b/crates/sqlbuild-cli/src/person_login/_helpers/flow.rs new file mode 100644 index 0000000000..3c14e27cb9 --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/_helpers/flow.rs @@ -0,0 +1,117 @@ +//! A CLI-only random secret can be exchanged only after an explicit browser decision. + +use std::fmt::Write; +use std::time::{Duration, Instant}; + +use chrono::Utc; +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::person_login::constants::{LOGIN_SECONDS, POLL_SECONDS, SECRET_BYTES}; +use crate::person_login::models::{LoginFlow, PersonClient, PersonCredential}; + +pub(crate) async fn login( + client: &PersonClient, + no_browser: bool, + report: &impl Fn(&str), +) -> Result { + let mut bytes = [0_u8; SECRET_BYTES]; + getrandom::fill(&mut bytes) + .map_err(|_| std::io::Error::other("could not generate a secure CLI exchange secret"))?; + let mut secret = String::new(); + for byte in bytes { + write!(secret, "{byte:02x}").map_err(std::io::Error::other)?; + } + let response = client + .client + .post(format!("{}/api/auth/cli/start", client.origin)) + .json(&json!({"client_hash":sha256_hex(secret.as_bytes())})) + .send() + .await + .map_err(|_| std::io::Error::other("could not start browser sign-in"))?; + if !response.status().is_success() { + return Err(std::io::Error::other( + "server rejected CLI sign-in; check its authentication configuration or wait before retrying", + )); + } + let flow: LoginFlow = response + .json() + .await + .map_err(|_| std::io::Error::other("server returned an invalid CLI handoff"))?; + let (id, code) = flow + .browser_code + .split_once('.') + .ok_or_else(|| std::io::Error::other("server returned an invalid CLI handoff"))?; + let id = uuid::Uuid::parse_str(id) + .map_err(|_| std::io::Error::other("server returned an invalid CLI handoff"))?; + if code.len() != SECRET_BYTES * 2 || !code.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(std::io::Error::other( + "server returned an invalid CLI handoff", + )); + } + let url = format!("{}/auth/cli#{}", client.origin, flow.browser_code); + report(&format!( + "Open {url}\nVerification code: {}\nWaiting for browser authorization…\n", + code[..8].to_ascii_uppercase() + )); + if !no_browser && !open_browser(&url) { + report("Could not open a browser automatically. Open the URL above manually.\n"); + } + let started = Instant::now(); + while started.elapsed() < Duration::from_secs(LOGIN_SECONDS) && Utc::now() < flow.expires_at { + let response = client + .client + .post(format!("{}/api/auth/cli/claim", client.origin)) + .json(&json!({"flow_id":id,"client_secret":secret})) + .send() + .await + .map_err(|_| { + std::io::Error::other("could not complete CLI sign-in; start sqb login again") + })?; + match response.status() { + reqwest::StatusCode::ACCEPTED => { + tokio::time::sleep(Duration::from_secs(POLL_SECONDS)).await + } + reqwest::StatusCode::OK => { + return response.json().await.map_err(|_| { + std::io::Error::other("server returned an invalid CLI credential") + }); + } + _ => { + return Err(std::io::Error::other( + "CLI sign-in was cancelled, expired or is unavailable; start sqb login again", + )); + } + } + } + Err(std::io::Error::other( + "CLI sign-in expired; start sqb login again", + )) +} + +fn open_browser(url: &str) -> bool { + #[cfg(target_os = "macos")] + let mut command = std::process::Command::new("open"); + #[cfg(all(unix, not(target_os = "macos")))] + let mut command = std::process::Command::new("xdg-open"); + #[cfg(windows)] + let mut command = { + let mut command = std::process::Command::new("rundll32"); + command.arg("url.dll,FileProtocolHandler"); + command + }; + command + .arg(url) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()); + match command.spawn() { + Ok(mut child) => { + std::thread::spawn(move || { + let _ = child.wait(); + }); + true + } + Err(_) => false, + } +} diff --git a/crates/sqlbuild-cli/src/person_login/_helpers/mod.rs b/crates/sqlbuild-cli/src/person_login/_helpers/mod.rs new file mode 100644 index 0000000000..11f145944a --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(super) mod credentials; +pub(super) mod flow; +pub(super) mod transport; diff --git a/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs b/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs new file mode 100644 index 0000000000..f44ef2f0ce --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/_helpers/transport.rs @@ -0,0 +1,153 @@ +//! HTTPS identity is verified; redirects cannot forward person credentials to another origin. + +use std::path::Path; +use std::time::Duration; + +use reqwest::header::{AUTHORIZATION, HeaderValue}; +use sqlbuild_orchestration::api::models::ApiErrorBody; +use sqlbuild_orchestration::credentials::models::SessionIdentity; +use sqlbuild_orchestration::snapshots::models::NodeSql; +use uuid::Uuid; + +use crate::person_login::constants::REQUEST_SECONDS; +use crate::person_login::models::{PersonClient, PersonCredential}; + +pub(crate) fn connect(origin: &str, ca: Option<&Path>) -> Result { + let mut builder = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(REQUEST_SECONDS)) + .default_headers( + [( + reqwest::header::ORIGIN, + HeaderValue::from_str(origin).map_err(std::io::Error::other)?, + )] + .into_iter() + .collect(), + ); + if let Some(ca) = ca { + builder = builder.add_root_certificate( + reqwest::Certificate::from_pem(&std::fs::read(ca)?) + .map_err(|_| std::io::Error::other("could not read the trusted CA certificate"))?, + ); + } + Ok(PersonClient { + client: builder + .build() + .map_err(|_| std::io::Error::other("could not initialize the HTTPS client"))?, + origin: origin.into(), + }) +} + +pub(crate) fn authorized( + client: &PersonClient, + method: reqwest::Method, + path: &str, + credential: &PersonCredential, +) -> Result { + let mut authorization = HeaderValue::from_str(&format!("Session {}", credential.bearer)) + .map_err(|_| std::io::Error::other("stored CLI credential is invalid"))?; + authorization.set_sensitive(true); + Ok(client + .client + .request(method, format!("{}{path}", client.origin)) + .header(AUTHORIZATION, authorization)) +} + +pub(crate) async fn current( + client: &PersonClient, + credential: &PersonCredential, +) -> Result { + let response = authorized( + client, + reqwest::Method::GET, + "/api/auth/session", + credential, + )? + .send() + .await + .map_err(|_| std::io::Error::other("could not verify the CLI session with the server"))?; + if !response.status().is_success() { + return Err(std::io::Error::other( + "CLI session is unavailable or revoked; run sqb login again", + )); + } + response + .json() + .await + .map_err(|_| std::io::Error::other("server returned an invalid person identity")) +} + +pub(crate) async fn revoke( + client: &PersonClient, + credential: &PersonCredential, +) -> Result<(), std::io::Error> { + let response = authorized( + client, + reqwest::Method::POST, + "/api/auth/logout", + credential, + )? + .send() + .await + .map_err(|_| std::io::Error::other("remote session revocation could not be confirmed"))?; + if response.status().is_success() || response.status() == reqwest::StatusCode::UNAUTHORIZED { + Ok(()) + } else { + Err(std::io::Error::other( + "remote session revocation could not be confirmed", + )) + } +} + +pub(crate) async fn node_sql( + client: &PersonClient, + credential: &PersonCredential, + (location, target, node): (Uuid, &str, &str), +) -> Result, std::io::Error> { + let mut url = reqwest::Url::parse(&client.origin).map_err(std::io::Error::other)?; + url.path_segments_mut() + .map_err(|()| std::io::Error::other("invalid server origin"))? + .pop_if_empty() + .extend([ + "api", + "control", + "locations", + &location.to_string(), + "targets", + target, + "nodes", + node, + "sql", + ]); + let response = authorized(client, reqwest::Method::GET, url.path(), credential)? + .send() + .await + .map_err(|_| std::io::Error::other("could not read published SQL from the server"))?; + let status = response.status(); + if !status.is_success() { + if status == reqwest::StatusCode::UNAUTHORIZED { + return Err(std::io::Error::other( + "CLI session is unavailable or revoked; run sqb login again", + )); + } + let decision = response + .json::() + .await + .map_err(|_| { + std::io::Error::other(format!( + "SQL inspection rejected (HTTP {}); invalid error response", + status.as_u16() + )) + })? + .decision_id; + return Err(std::io::Error::other(format!( + "SQL inspection rejected (HTTP {}{})", + status.as_u16(), + decision.map_or_else(String::new, |id| format!("; decision {id}")) + ))); + } + response + .json() + .await + .map_err(|_| std::io::Error::other("server returned an invalid SQL inspection result")) +} diff --git a/crates/sqlbuild-cli/src/person_login/constants.rs b/crates/sqlbuild-cli/src/person_login/constants.rs new file mode 100644 index 0000000000..d525dc760e --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/constants.rs @@ -0,0 +1,7 @@ +//! Person login bounds and user-owned credential storage names. + +pub(super) const SECRET_BYTES: usize = 32; +pub(super) const POLL_SECONDS: u64 = 1; +pub(super) const LOGIN_SECONDS: u64 = 300; +pub(super) const REQUEST_SECONDS: u64 = 15; +pub(super) const KEYRING_SERVICE: &str = "sqlbuild-cli"; diff --git a/crates/sqlbuild-cli/src/person_login/main/mod.rs b/crates/sqlbuild-cli/src/person_login/main/mod.rs new file mode 100644 index 0000000000..bf1d2af998 --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/main/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod read_node_sql; +pub(crate) mod run_login; diff --git a/crates/sqlbuild-cli/src/person_login/main/read_node_sql.rs b/crates/sqlbuild-cli/src/person_login/main/read_node_sql.rs new file mode 100644 index 0000000000..99970998c7 --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/main/read_node_sql.rs @@ -0,0 +1,32 @@ +//! A saved person session can inspect published SQL without opening a project or warehouse. + +use std::path::Path; + +use sqlbuild_orchestration::api::main::parse_origin::parse_origin; +use sqlbuild_orchestration::snapshots::models::NodeSql; +use uuid::Uuid; + +use crate::person_login::_helpers::{credentials, transport}; + +pub(crate) fn read_node_sql( + (server, ca): (&str, Option<&Path>), + (location, target, node): (Uuid, &str, &str), +) -> Result, std::io::Error> { + let origin = parse_origin(server, "CLI").map_err(std::io::Error::other)?; + if target.trim().is_empty() || node.trim().is_empty() { + return Err(std::io::Error::other( + "target and exact node key must not be blank", + )); + } + let credential = credentials::load(&origin)? + .ok_or_else(|| std::io::Error::other("not signed in; run sqb login"))?; + let client = transport::connect(&origin, ca)?; + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; + runtime.block_on(transport::node_sql( + &client, + &credential, + (location, target, node), + )) +} diff --git a/crates/sqlbuild-cli/src/person_login/main/run_login.rs b/crates/sqlbuild-cli/src/person_login/main/run_login.rs new file mode 100644 index 0000000000..08fb898dc2 --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/main/run_login.rs @@ -0,0 +1,70 @@ +//! Person login commands run before project discovery and never write credentials into a project. + +use crate::person_login::_helpers::{credentials, flow, transport}; +use crate::person_login::models::{LoginAction, LoginOptions}; +use sqlbuild_orchestration::api::main::parse_origin::parse_origin; +use sqlbuild_orchestration::credentials::models::SessionIdentity; + +pub(crate) fn run_login( + options: LoginOptions, + report: impl Fn(&str), +) -> Result, std::io::Error> { + let origin = parse_origin(&options.server, "CLI").map_err(std::io::Error::other)?; + let (logout_credential, cleanup) = if options.action == LoginAction::Logout { + report("Signing out of SQLBuild CLI…\n"); + let credential = credentials::load(&origin); + let cleanup = credentials::remove(&origin); + match credential { + Ok(Some(credential)) => (Some(credential), cleanup), + Ok(None) => { + cleanup?; + report("SQLBuild CLI signed out successfully\n"); + return Ok(None); + } + Err(error) => { + cleanup?; + return Err(error); + } + } + } else { + (None, Ok(())) + }; + let client = transport::connect(&origin, options.ca.as_deref())?; + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; + runtime.block_on(async { + if let Some(credential) = logout_credential { + let remote = transport::revoke(&client, &credential).await; + cleanup?; + remote?; + report("SQLBuild CLI signed out successfully\n"); + return Ok(None); + } + if options.action == LoginAction::Status { + report("Checking SQLBuild CLI session…\n"); + let credential = credentials::load(&client.origin)? + .ok_or_else(|| std::io::Error::other("not signed in; run sqb login"))?; + let identity = transport::current(&client, &credential).await?; + report("SQLBuild CLI session verified successfully\n"); + return Ok(Some(identity)); + } + report("Starting SQLBuild CLI sign-in…\n"); + let credential = flow::login(&client, options.no_browser, &report).await?; + match credentials::save(&client.origin, &credential) { + Ok(true) => (), + Ok(false) => report("OS credential store unavailable; using an owner-only credential file in your user configuration directory.\n"), + Err(error) => { + let _ = transport::revoke(&client, &credential).await; + return Err(error); + } + } + if let Err(error) = transport::current(&client, &credential).await { + let _ = transport::revoke(&client, &credential).await; + credentials::remove(&client.origin)?; + return Err(error); + } + report("SQLBuild CLI signed in successfully\n"); + Ok::<_, std::io::Error>(None) + }) +} diff --git a/crates/sqlbuild-cli/src/person_login/mod.rs b/crates/sqlbuild-cli/src/person_login/mod.rs new file mode 100644 index 0000000000..1c79f915ed --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/mod.rs @@ -0,0 +1,6 @@ +//! Browser-authorized person sessions are distinct from location and worker tokens. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub(crate) mod main; +pub(crate) mod models; diff --git a/crates/sqlbuild-cli/src/person_login/models.rs b/crates/sqlbuild-cli/src/person_login/models.rs new file mode 100644 index 0000000000..ddcc46e8da --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/models.rs @@ -0,0 +1,57 @@ +//! Only public identity is printable; persisted session material is always redacted. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sqlbuild_orchestration::credentials::models::SessionIdentity; + +#[derive(Debug)] +pub(crate) struct LoginOptions { + pub(crate) action: LoginAction, + pub(crate) server: String, + pub(crate) ca: Option, + pub(crate) no_browser: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum LoginAction { + Start, + Status, + Logout, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct PersonCredential { + pub(super) identity: SessionIdentity, + pub(super) bearer: String, +} + +impl std::fmt::Debug for PersonCredential { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PersonCredential") + .field("identity", &self.identity) + .finish_non_exhaustive() + } +} + +#[derive(Deserialize)] +pub(super) struct LoginFlow { + pub(super) browser_code: String, + pub(super) expires_at: DateTime, +} + +impl std::fmt::Debug for LoginFlow { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("LoginFlow") + .field("expires_at", &self.expires_at) + .finish_non_exhaustive() + } +} + +#[derive(Debug)] +pub(super) struct PersonClient { + pub(super) client: reqwest::Client, + pub(super) origin: String, +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/child.rs b/crates/sqlbuild-cli/src/workers/_helpers/child.rs new file mode 100644 index 0000000000..c8c2643d4b --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/child.rs @@ -0,0 +1,64 @@ +//! Launchers share physical exit observation and termination, not durable lifecycle authority. + +use sqlbuild_orchestration::store::errors::ControlError; + +use crate::workers::_helpers::supervisor::exit_code; +use crate::workers::main::terminate_child::terminate_child; +use crate::workers::models::{WorkerChild, WorkerExit}; + +impl WorkerExit { + pub(in crate::workers) const fn code(self) -> Option { + match self { + Self::Code(code) => Some(code), + Self::Absent => None, + } + } +} + +impl WorkerChild { + pub(in crate::workers) async fn try_wait( + &mut self, + ) -> Result, ControlError> { + match self { + Self::Local(child) => Ok(child + .try_wait()? + .map(|exit| WorkerExit::Code(exit_code(exit)))), + #[cfg(target_os = "linux")] + Self::RecoveredLocal(child) => child.try_wait(), + Self::Docker(child) => child.try_wait().await, + } + } + + pub(in crate::workers) async fn wait(&mut self) -> Result { + if let Self::Local(child) = self { + return Ok(WorkerExit::Code(exit_code(child.wait().await?))); + } + loop { + if let Some(exit) = self.try_wait().await? { + return Ok(exit); + } + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } + } + + pub(in crate::workers) async fn terminate(&mut self) -> Result<(), ControlError> { + match self { + Self::Local(child) => terminate_child(child), + #[cfg(target_os = "linux")] + Self::RecoveredLocal(child) => child.signal(rustix::process::Signal::TERM), + Self::Docker(child) => child.signal("SIGTERM").await, + } + } + + pub(in crate::workers) async fn kill(&mut self) -> Result<(), ControlError> { + match self { + Self::Local(child) => { + child.start_kill()?; + Ok(()) + } + #[cfg(target_os = "linux")] + Self::RecoveredLocal(child) => child.signal(rustix::process::Signal::KILL), + Self::Docker(child) => child.signal("SIGKILL").await, + } + } +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs b/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs new file mode 100644 index 0000000000..0027b59f29 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs @@ -0,0 +1,91 @@ +//! Reject ambiguous or nonexistent installation-owned launcher scopes before claiming any run. + +use std::collections::BTreeSet; + +use sqlbuild_orchestration::store::errors::ControlError; +use uuid::Uuid; + +use crate::workers::models::{Launcher, LauncherScope, Launchers}; + +impl<'a> Launcher<'a> { + pub(in crate::workers) fn scope(self) -> LauncherScope<'a> { + match self { + Self::Local(location) => LauncherScope { + location_id: location.location_id, + target: &location.target, + server_url: &location.server_url, + max_workers: location.max_workers, + }, + Self::Docker(location) => LauncherScope { + location_id: location.location_id, + target: &location.target, + server_url: &location.server_url, + max_workers: location.max_workers, + }, + } + } +} + +impl Launchers { + pub(in crate::workers) fn locations(&self) -> impl Iterator> { + self.local + .iter() + .map(Launcher::Local) + .chain(self.docker.iter().map(Launcher::Docker)) + } + + pub(in crate::workers) fn dispatch_scopes(&self) -> BTreeSet<(Uuid, String)> { + self.locations() + .map(|location| { + let scope = location.scope(); + (scope.location_id, scope.target.to_owned()) + }) + .collect() + } +} + +pub(in crate::workers) fn read_configuration( + path: Option<&str>, + scheduler_enabled: bool, +) -> Result { + let configuration: Launchers = match path { + Some(path) => serde_json::from_slice(&std::fs::read(path)?) + .map_err(|error| ControlError::Invalid(format!("launcher configuration: {error}")))?, + None => Launchers { + local: Vec::new(), + docker: Vec::new(), + concurrency_limits: Default::default(), + }, + }; + if !scheduler_enabled && configuration.locations().next().is_some() { + return Err(ControlError::Invalid( + "launchers require the embedded scheduler".into(), + )); + } + let mut scopes: BTreeSet<(Uuid, &str)> = BTreeSet::new(); + for location in &configuration.local { + if !location.project_dir.is_absolute() + || !location.project_dir.is_dir() + || !location.executable.is_absolute() + || !location.executable.is_file() + { + return Err(ControlError::Invalid("local launchers require unique location/target scopes and existing absolute project/executable paths".into())); + } + } + for location in configuration.locations() { + let scope = location.scope(); + if scope.target.trim().is_empty() || !scopes.insert((scope.location_id, scope.target)) { + return Err(ControlError::Invalid( + "launchers require unique location/target scopes".into(), + )); + } + if let Launcher::Docker(location) = location + && (!location.socket.is_absolute() + || location.location_container.trim().is_empty() + || location.executable.trim().is_empty()) + { + return Err(ControlError::Invalid("Docker launchers require an absolute socket path, location container and executable".into())); + } + } + Ok(configuration) +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/docker.rs b/crates/sqlbuild-cli/src/workers/_helpers/docker.rs new file mode 100644 index 0000000000..746ddf0c8e --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/docker.rs @@ -0,0 +1,205 @@ +//! The privileged role talks to a local Docker daemon; workers never receive its socket. + +use std::path::Path; + +use reqwest::{Method, StatusCode}; +use serde_json::{Value, json}; +use sqlbuild_orchestration::runs::models::RunClaim; +use sqlbuild_orchestration::store::errors::ControlError; + +use crate::workers::_helpers::invocation::{arguments, environment, excluded}; +use crate::workers::constants::IMAGE_ID_HEX_LENGTH; +use crate::workers::models::{DockerLauncher, DockerWorker, Launcher, WorkerExit}; + +impl DockerWorker { + pub(in crate::workers) fn prepare( + location: &DockerLauncher, + claim: &RunClaim, + ) -> Result { + let image_id = claim.request.image_id.as_deref().unwrap_or_default(); + match image_id.strip_prefix("sha256:") { + Some(digest) + if digest.len() == IMAGE_ID_HEX_LENGTH + && digest.as_bytes().iter().all(u8::is_ascii_hexdigit) => {} + _ => { + return Err(ControlError::Invalid( + "Docker workers require a snapshot pinned to a full sha256 image ID".into(), + )); + } + } + Ok(Self { + client: docker_client(&location.socket)?, + name: format!( + "sqlbuild-{}-{}", + claim.request.location_id, claim.request.run_id + ), + claim_id: claim.claim_id, + image_id: image_id.into(), + }) + } + + pub(in crate::workers) async fn launch( + &self, + location: &DockerLauncher, + claim: &RunClaim, + bearer: &str, + control_variable: &str, + ) -> Result<(), ControlError> { + let parent: Value = self + .request( + Method::GET, + &format!("containers/{}/json", location.location_container), + None, + ) + .await? + .error_for_status() + .map_err(docker_error)? + .json() + .await + .map_err(docker_error)?; + let project = parent["Config"]["WorkingDir"] + .as_str() + .filter(|path| path.starts_with('/')) + .ok_or_else(|| { + ControlError::Invalid( + "location container requires an absolute working directory".into(), + ) + })?; + let mut command = arguments(&Launcher::Docker(location).scope(), claim); + command.push(project.into()); + command.extend(["--image-id".into(), self.image_id.clone()]); + let mut variables: Vec = parent["Config"]["Env"] + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .filter(|entry| !excluded(entry.split('=').next().unwrap_or(entry), control_variable)) + .map(str::to_owned) + .collect(); + variables.extend( + environment(claim, bearer) + .into_iter() + .map(|(key, value)| format!("{key}={value}")), + ); + let body = json!({ + "Image": self.image_id, "Entrypoint": [location.executable], "Cmd": command, + "WorkingDir": project, "User": parent["Config"]["User"], "Env": variables, + "Labels": {"io.sqlbuild.claim-id": self.claim_id.to_string()}, + "HostConfig": { + "NetworkMode": parent["HostConfig"]["NetworkMode"], + "VolumesFrom": [location.location_container], + "Memory": parent["HostConfig"]["Memory"], "MemorySwap": parent["HostConfig"]["MemorySwap"], + "NanoCpus": parent["HostConfig"]["NanoCpus"], "PidsLimit": parent["HostConfig"]["PidsLimit"], + "ReadonlyRootfs": parent["HostConfig"]["ReadonlyRootfs"], + }, + }); + self.request( + Method::POST, + &format!("containers/create?name={}", self.name), + Some(&body), + ) + .await? + .error_for_status() + .map_err(docker_error)?; + self.request( + Method::POST, + &format!("containers/{}/start", self.name), + None, + ) + .await? + .error_for_status() + .map_err(docker_error)?; + Ok(()) + } + + pub(in crate::workers) async fn try_wait(&self) -> Result, ControlError> { + let response = self + .request(Method::GET, &format!("containers/{}/json", self.name), None) + .await?; + if response.status() == StatusCode::NOT_FOUND { + return Ok(Some(WorkerExit::Absent)); + } + let container: Value = response + .error_for_status() + .map_err(docker_error)? + .json() + .await + .map_err(docker_error)?; + if container["Config"]["Labels"]["io.sqlbuild.claim-id"] != self.claim_id.to_string() + || container["Image"] != self.image_id + { + return Err(ControlError::Invalid( + "Docker container does not belong to the retained claim and image".into(), + )); + } + if container["State"]["Running"].as_bool() == Some(true) { + return Ok(None); + } + if container["State"]["Status"].as_str() == Some("created") { + return Ok(Some(WorkerExit::Absent)); + } + let code = container["State"]["ExitCode"] + .as_i64() + .ok_or_else(|| ControlError::Invalid("Docker container has no exit evidence".into()))?; + Ok(Some(WorkerExit::Code(code))) + } + + pub(in crate::workers) async fn signal(&self, signal: &str) -> Result<(), ControlError> { + if self.try_wait().await?.is_some() { + return Ok(()); + } + let response = self + .request( + Method::POST, + &format!("containers/{}/kill?signal={signal}", self.name), + None, + ) + .await?; + if !matches!( + response.status(), + StatusCode::NOT_FOUND | StatusCode::CONFLICT + ) { + response.error_for_status().map_err(docker_error)?; + } + Ok(()) + } + + async fn request( + &self, + method: Method, + path: &str, + body: Option<&Value>, + ) -> Result { + let request = self + .client + .request(method, format!("http://localhost/{path}")); + match body { + Some(body) => request.json(body), + None => request, + } + .send() + .await + .map_err(docker_error) + } +} + +fn docker_error(error: reqwest::Error) -> ControlError { + ControlError::Invalid(format!("Docker daemon request failed: {error}")) +} + +#[cfg(unix)] +fn docker_client(socket: &Path) -> Result { + reqwest::Client::builder() + .no_proxy() + .unix_socket(socket) + .timeout(std::time::Duration::from_secs(5)) + .build() + .map_err(docker_error) +} + +#[cfg(not(unix))] +fn docker_client(_socket: &Path) -> Result { + Err(ControlError::Invalid( + "Docker launchers require a Unix socket".into(), + )) +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs b/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs new file mode 100644 index 0000000000..f1721202a0 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs @@ -0,0 +1,51 @@ +//! Both launchers hand off the same scoped invocation and exclude privileged credentials. + +use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; +use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE; +use sqlbuild_orchestration::runs::models::RunClaim; +use sqlbuild_orchestration::webhooks::constants::SECURITY_AUDIT_WEBHOOK_VARIABLE; + +use crate::workers::models::LauncherScope; + +pub(in crate::workers) fn arguments(scope: &LauncherScope<'_>, claim: &RunClaim) -> Vec { + [ + "location".into(), + "execute".into(), + "--server".into(), + scope.server_url.to_owned(), + "--location-id".into(), + scope.location_id.to_string(), + "--target".into(), + scope.target.to_owned(), + "--run-id".into(), + claim.request.run_id.to_string(), + "--claim-id".into(), + claim.claim_id.to_string(), + "--project-dir".into(), + ] + .into() +} + +pub(in crate::workers) fn excluded(name: &str, control_variable: &str) -> bool { + name == control_variable + || name == PLATFORM_WEBHOOK_VARIABLE + || name == SECURITY_AUDIT_WEBHOOK_VARIABLE + || matches!( + name, + "SQLBUILD_CONTROL_DATABASE_URL" | "SQLBUILD_LOCATION_TOKEN" | "SQLBUILD_RUN_TOKEN" + ) + || name == INVOCATION_CONTEXT_ENV +} + +pub(in crate::workers) fn environment( + claim: &RunClaim, + bearer: &str, +) -> [(&'static str, String); 2] { + [ + ("SQLBUILD_RUN_TOKEN", bearer.into()), + (INVOCATION_CONTEXT_ENV, serde_json::json!({ + "run_id": claim.request.run_id, + "integration": {"name": "sqlbuild", "run_id": claim.request.run_id, "job_name": claim.request.job_key}, + }).to_string()), + ] +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/local.rs b/crates/sqlbuild-cli/src/workers/_helpers/local.rs new file mode 100644 index 0000000000..82a1c4fba2 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/local.rs @@ -0,0 +1,128 @@ +//! Linux dev workers inherit a claim lock before exec; recovery never guesses from a stale PID. + +use std::fs::OpenOptions; +use std::os::unix::fs::MetadataExt; + +use command_fds::{CommandFdExt, FdMapping}; +use rustix::fs::{FlockOperation, flock}; +use rustix::process::{Pid, PidfdFlags, Signal, pidfd_open, pidfd_send_signal}; +use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; +use sqlbuild_orchestration::runs::models::RunClaim; +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::process::Command; + +use crate::workers::constants::LOCAL_CLAIM_FD; +use crate::workers::models::{LocalLauncher, LocalWorker, WorkerExit}; + +impl LocalWorker { + pub(in crate::workers) fn open( + location: &LocalLauncher, + claim: &RunClaim, + ) -> Result { + let directory = location.project_dir.join(".sqlbuild/workers"); + std::fs::create_dir_all(&directory)?; + let lock = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(directory.join(format!("{}.lock", claim.claim_id)))?; + Ok(Self { + lock, + run_id: claim.request.run_id, + stopped: false, + }) + } + + /// Keep the acquired lock until terminal publication, so a delayed spawn cannot start afterward. + pub(in crate::workers) fn try_wait(&mut self) -> Result, ControlError> { + if !self.stopped { + match flock(&self.lock, FlockOperation::NonBlockingLockExclusive) { + Ok(()) => self.stopped = true, + Err(rustix::io::Errno::WOULDBLOCK) => return Ok(None), + Err(error) => return Err(std::io::Error::from(error).into()), + } + } + Ok(Some(WorkerExit::Absent)) + } + + /// Only a kernel handle whose current descriptor and invocation match this claim can be signalled. + pub(in crate::workers) fn signal(&self, signal: Signal) -> Result<(), ControlError> { + let expected = self.lock.metadata()?; + for entry in std::fs::read_dir("/proc")? { + let entry = entry?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + let raw = match name.parse::() { + Ok(raw) => raw, + Err(_) => continue, + }; + let Some(pid) = Pid::from_raw(raw) else { + continue; + }; + let descriptor = entry.path().join(format!("fd/{LOCAL_CLAIM_FD}")); + if !same_file(&descriptor, &expected) { + continue; + } + let handle = match pidfd_open(pid, PidfdFlags::empty()) { + Ok(handle) => handle, + Err(rustix::io::Errno::SRCH) => continue, + Err(error) => return Err(std::io::Error::from(error).into()), + }; + if !same_file(&descriptor, &expected) || !self.owns_invocation(&entry.path())? { + continue; + } + match pidfd_send_signal(handle, signal) { + Ok(()) | Err(rustix::io::Errno::SRCH) => {} + Err(error) => return Err(std::io::Error::from(error).into()), + } + } + Ok(()) + } + + fn owns_invocation(&self, process: &std::path::Path) -> Result { + let environment = match std::fs::read(process.join("environ")) { + Ok(environment) => environment, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error.into()), + }; + let prefix = format!("{INVOCATION_CONTEXT_ENV}="); + for variable in environment.split(|byte| *byte == 0) { + if let Some(context) = variable.strip_prefix(prefix.as_bytes()) { + let context: serde_json::Value = + serde_json::from_slice(context).map_err(|error| { + ControlError::Invalid(format!( + "invalid local worker invocation context: {error}" + )) + })?; + return Ok(context["run_id"].as_str() == Some(&self.run_id.to_string())); + } + } + Ok(false) + } +} + +fn same_file(path: &std::path::Path, expected: &std::fs::Metadata) -> bool { + std::fs::metadata(path) + .is_ok_and(|actual| actual.dev() == expected.dev() && actual.ino() == expected.ino()) +} + +/// The child inherits the lock even before Python starts or a worker handshake can succeed. +pub(in crate::workers) fn inherit_claim( + command: &mut Command, + location: &LocalLauncher, + claim: &RunClaim, +) -> Result<(), ControlError> { + let owner = LocalWorker::open(location, claim)?; + flock(&owner.lock, FlockOperation::NonBlockingLockExclusive).map_err(std::io::Error::from)?; + command + .as_std_mut() + .fd_mappings(vec![FdMapping { + parent_fd: owner.lock.into(), + child_fd: LOCAL_CLAIM_FD, + }]) + .map_err(|error| ControlError::Invalid(format!("local claim descriptor: {error}")))?; + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/mod.rs b/crates/sqlbuild-cli/src/workers/_helpers/mod.rs new file mode 100644 index 0000000000..a0ead33080 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/mod.rs @@ -0,0 +1,9 @@ +mod child; +pub(super) mod configuration; +mod docker; +mod invocation; +#[cfg(target_os = "linux")] +mod local; +mod recovery; +mod shutdown; +mod supervisor; diff --git a/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs b/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs new file mode 100644 index 0000000000..f439a58dd6 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs @@ -0,0 +1,86 @@ +//! A new leader resumes supervising pinned containers without relaunching their execution. + +use std::collections::BTreeSet; +use std::io::Write; + +use sqlbuild_orchestration::runs::main::list_run_claims::list_run_claims; +use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; +use sqlbuild_orchestration::scheduler::models::SchedulerLease; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::models::ControlStore; + +use crate::workers::models::{ClaimedWorker, DockerWorker, Launcher, WorkerChild, Workers}; + +impl Workers { + pub(in crate::workers) async fn recover( + &mut self, + store: &ControlStore, + lease: &SchedulerLease, + ) -> Result<(), ControlError> { + if self.recovered_epoch == Some(lease.epoch) { + return Ok(()); + } + let mut known: BTreeSet<_> = self + .active + .iter() + .map(|worker| worker.claim.claim_id) + .collect(); + for location in self.configuration.locations() { + #[cfg(not(target_os = "linux"))] + if matches!(location, Launcher::Local(_)) { + continue; + } + let scope = location.scope(); + let mut after = None; + loop { + renew_lease(store, lease).await?; + let claims = + list_run_claims(store, (scope.location_id, scope.target), after).await?; + let Some(last) = claims.last() else { + break; + }; + after = Some(last.request.run_id); + for claim in claims { + if !known.insert(claim.claim_id) { + continue; + } + let child = match location { + #[cfg(target_os = "linux")] + Launcher::Local(location) => Some(WorkerChild::RecoveredLocal( + crate::workers::models::LocalWorker::open(location, &claim)?, + )), + #[cfg(not(target_os = "linux"))] + Launcher::Local(_) => continue, + Launcher::Docker(location) => match DockerWorker::prepare(location, &claim) + { + Ok(child) => Some(WorkerChild::Docker(child)), + Err(ControlError::Invalid(_)) => None, + Err(error) => return Err(error), + }, + }; + drop(writeln!( + std::io::stderr().lock(), + "Worker supervision recovered for run {}", + claim.request.run_id + )); + let rejected = child.is_none(); + let index = self.active.len(); + self.active.push(ClaimedWorker { + claim, + child, + stopping: None, + exit: None, + launch_failure: rejected.then_some("docker_configuration_rejected"), + }); + if rejected { + self.active[index] + .publish_launch_failure(store, lease) + .await?; + } + } + } + } + self.recovered_epoch = Some(lease.epoch); + Ok(()) + } +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs b/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs new file mode 100644 index 0000000000..7eafd6f876 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs @@ -0,0 +1,189 @@ +//! Keep leadership alive while reaping children, then release only successfully reported claims. + +use std::io::Write; +use std::time::{Duration, Instant}; + +use sqlbuild_orchestration::runs::main::observe_run::observe_run; +use sqlbuild_orchestration::runs::models::{RunChange, RunOutcome}; +use sqlbuild_orchestration::scheduler::constants::RENEW_INTERVAL_SECONDS; +use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; +use sqlbuild_orchestration::scheduler::models::SchedulerLease; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::models::ControlStore; + +use crate::workers::_helpers::supervisor::{publish, terminal}; +use crate::workers::models::{ClaimedWorker, WorkerChild, WorkerExit, Workers}; + +impl ClaimedWorker { + async fn reap( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + let Some(child) = self.child.as_mut() else { + return Ok(()); + }; + if self.exit.is_some() { + return Ok(()); + } + let mut killed = false; + if self.stopping.is_none() { + if child.terminate().await.is_err() { + child.kill().await?; + killed = true; + } + self.stopping = Some(Instant::now()); + } + let deadline = self.stopping.unwrap_or_else(Instant::now) + Duration::from_secs(60); + let mut renew = tokio::time::interval(Duration::from_secs(RENEW_INTERVAL_SECONDS)); + loop { + tokio::select! { + exit = child.wait() => { + self.exit = Some(exit?); + return Ok(()); + } + _ = tokio::time::sleep_until(deadline.into()), if !killed => { + child.kill().await?; + killed = true; + } + _ = renew.tick(), if lease.is_some() => { + if let Some(lease) = lease + && renew_lease(store, lease).await.is_err() + { + drop(writeln!(std::io::stderr().lock(), "Scheduler lease renewal failed during worker shutdown")); + } + } + } + } + } + + async fn finish_shutdown( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + self.reap(store, lease).await?; + let state = observe_run(store, self.claim.request.run_id).await?; + if !terminal(state.status) { + let lease = lease.ok_or(ControlError::Fenced)?; + let reason = if self.child.is_none() { + "worker launch preparation failed before a child started" + } else { + "scheduler confirmed its worker stopped or absent" + }; + publish( + store, + lease, + &self.claim, + ( + "local-shutdown", + RunChange::Finished { + outcome: if state.cancel_requested_at.is_some() { + RunOutcome::Cancelled + } else { + RunOutcome::Failed + }, + reason: reason.into(), + exit_code: self.exit.and_then(WorkerExit::code), + }, + ), + ) + .await?; + } + Ok(()) + } +} + +impl Workers { + pub(in crate::workers) async fn shutdown_workers( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + let mut retrying = false; + let mut renew = tokio::time::interval(Duration::from_secs(RENEW_INTERVAL_SECONDS)); + let mut renewed = Instant::now(); + while !self.active.is_empty() { + let mut failure = None; + for index in (0..self.active.len()).rev() { + if renewed.elapsed() >= Duration::from_secs(RENEW_INTERVAL_SECONDS) + && let Some(lease) = lease + { + if let Err(error) = renew_lease(store, lease).await { + failure = Some(error); + } + renewed = Instant::now(); + } + if self.active[index].launch_failure.is_some() { + let reported = match lease { + Some(lease) => { + self.active[index] + .publish_launch_failure(store, lease) + .await + } + None => Err(ControlError::Fenced), + }; + if let Err(error) = reported { + if matches!(error, ControlError::Fenced) { + failure = Some(error); + } else { + failure.get_or_insert(error); + } + continue; + } + } + if let Some(WorkerChild::Docker(child)) = &self.active[index].child { + match child.try_wait().await { + Ok(None) => { + self.active.swap_remove(index); + continue; + } + Ok(exit) => self.active[index].exit = exit, + Err(error) => { + failure.get_or_insert(error); + continue; + } + } + } + match self.active[index].finish_shutdown(store, lease).await { + Ok(()) => { + self.active.swap_remove(index); + } + Err(ControlError::Fenced) => failure = Some(ControlError::Fenced), + Err(error) => { + failure.get_or_insert(error); + } + } + } + if matches!(failure, Some(ControlError::Fenced)) { + return Err(ControlError::Fenced); + } + if failure.is_some() { + if !retrying { + drop(writeln!( + std::io::stderr().lock(), + "Retrying worker shutdown publication or physical observation; claims remain owned" + )); + retrying = true; + } + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(1)) => {}, + _ = renew.tick(), if lease.is_some() => { + if let Some(lease) = lease + && matches!(renew_lease(store, lease).await, Err(ControlError::Fenced)) + { + return Err(ControlError::Fenced); + } + } + } + } + } + if retrying { + drop(writeln!( + std::io::stderr().lock(), + "Worker shutdown recovered; stopped claims released and live containers remain recoverable" + )); + } + Ok(()) + } +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs new file mode 100644 index 0000000000..7c7dd5635f --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs @@ -0,0 +1,392 @@ +//! One privileged owner supervises both launchers; workers receive only claim-scoped HTTP. + +use std::io::Write; +use std::process::{ExitStatus, Stdio}; +use std::time::{Duration, Instant}; + +use chrono::{TimeDelta, Utc}; +use sqlbuild_orchestration::credentials::main::tokens::issue_token::issue_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::runs::main::advance_groups::advance_groups; +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::observe_run::observe_run; +use sqlbuild_orchestration::runs::models::{ + RunAuthority, RunChange, RunChangeRequest, RunClaim, RunDispatch, RunOutcome, RunStatus, +}; +use sqlbuild_orchestration::scheduler::constants::RENEW_INTERVAL_SECONDS; +use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; +use sqlbuild_orchestration::scheduler::models::SchedulerLease; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::models::ControlStore; +use tokio::process::Command; +use uuid::Uuid; + +use crate::workers::_helpers::invocation::{arguments, environment, excluded}; +use crate::workers::models::{ + ClaimedWorker, DockerWorker, Launcher, LocalLauncher, WorkerChild, WorkerExit, Workers, +}; + +impl sqlbuild_orchestration::scheduler::types::RunLauncher for Workers { + async fn poll( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + if let Some(lease) = lease { + let eligible = self.configuration.dispatch_scopes(); + for location in self.configuration.locations() { + let scope = location.scope(); + advance_groups( + store, + lease, + ( + &self.configuration.concurrency_limits, + RunDispatch { + scope: Some((scope.location_id, scope.target)), + eligible: Some(&eligible), + }, + ), + ) + .await?; + } + self.recover(store, lease).await?; + } + let mut renewed = Instant::now(); + for index in (0..self.active.len()).rev() { + if renewed.elapsed() >= Duration::from_secs(RENEW_INTERVAL_SECONDS) + && let Some(lease) = lease + { + renew_lease(store, lease).await?; + renewed = Instant::now(); + } + let worker = &mut self.active[index]; + let mut state = observe_run(store, worker.claim.request.run_id).await?; + if let Some(child) = worker.child.as_mut() { + worker.exit = worker.exit.or(child.try_wait().await?); + } + if worker.exit.is_none() + && let Some(child) = worker.child.as_mut() + { + let now = state.observed_at; + let not_started = matches!(state.status, RunStatus::Queued | RunStatus::Launched) + && state + .claimed_at + .is_some_and(|claimed| now - claimed > TimeDelta::minutes(2)); + let silent = state + .last_heartbeat_at + .is_some_and(|last| now - last > TimeDelta::minutes(10)); + if (state.cancel_requested_at.is_some() || not_started || silent) + && worker.stopping.is_none() + { + child.terminate().await?; + worker.stopping = Some(Instant::now()); + } + if worker + .stopping + .is_some_and(|at| at.elapsed() >= Duration::from_secs(60)) + { + child.kill().await?; + worker.exit = child.try_wait().await?; + state = observe_run(store, worker.claim.request.run_id).await?; + } + } + if worker.exit.is_some() || worker.child.is_none() { + if !terminal(state.status) { + let Some(lease) = lease else { + continue; + }; + let outcome = if state.cancel_requested_at.is_some() { + RunOutcome::Cancelled + } else { + RunOutcome::Failed + }; + publish( + store, + lease, + &worker.claim, + ( + "local-exit", + RunChange::Finished { + outcome, + reason: "worker stopped; physical exit or absence confirmed".into(), + exit_code: worker.exit.and_then(WorkerExit::code), + }, + ), + ) + .await?; + } + self.active.swap_remove(index); + } + } + let Some(lease) = lease else { + return Ok(()); + }; + self.launch_available(store, lease).await + } + + /// Never release claims until children have physically stopped, even if shutdown loses its fence. + async fn shutdown( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + self.shutdown_workers(store, lease).await + } +} + +impl Workers { + async fn launch_available( + &mut self, + store: &ControlStore, + lease: &SchedulerLease, + ) -> Result<(), ControlError> { + let mut renewed = Instant::now(); + let eligible = self.configuration.dispatch_scopes(); + for location in self.configuration.locations() { + let scope = location.scope(); + let running = self + .active + .iter() + .filter(|worker| { + worker.claim.request.location_id == scope.location_id + && worker.claim.request.target == scope.target + }) + .count(); + for _ in running..scope.max_workers.get() { + if renewed.elapsed() >= Duration::from_secs(RENEW_INTERVAL_SECONDS) { + renew_lease(store, lease).await?; + renewed = Instant::now(); + } + let Some(claim) = claim_next_run( + store, + lease, + &self.configuration.concurrency_limits, + RunDispatch { + scope: Some((scope.location_id, scope.target)), + eligible: Some(&eligible), + }, + ) + .await? + else { + break; + }; + let index = self.active.len(); + self.active.push(ClaimedWorker { + claim, + child: None, + stopping: None, + exit: None, + launch_failure: None, + }); + let worker = &mut self.active[index]; + let claim = &worker.claim; + if matches!(location, Launcher::Local(_)) && claim.request.image_id.is_some() { + worker.launch_failure = Some("local_image_rejected"); + worker.publish_launch_failure(store, lease).await?; + let claim = &worker.claim; + publish(store, lease, claim, ("local-image-rejected", RunChange::Finished { outcome: RunOutcome::Failed, reason: "image-pinned runs require a Docker launcher, not a local interpreter".into(), exit_code: None })).await?; + self.active.swap_remove(index); + continue; + } + let container_id = if let Launcher::Docker(location) = location { + match DockerWorker::prepare(location, claim) { + Ok(child) => { + let name = child.name.clone(); + worker.child = Some(WorkerChild::Docker(child)); + Some(name) + } + Err(_) => { + worker.launch_failure = Some("docker_configuration_rejected"); + worker.publish_launch_failure(store, lease).await?; + let claim = &worker.claim; + publish(store, lease, claim, ("docker-image-rejected", RunChange::Finished { + outcome: RunOutcome::Failed, reason: "Docker worker requires a pinned image and supported launcher configuration".into(), exit_code: None, + })).await?; + self.active.swap_remove(index); + continue; + } + } + } else { + None + }; + let context = AuditContext { + actor: format!("scheduler/{}", lease.holder), + granting_role: "scheduler".into(), + request_id: claim.claim_id, + ip_address: None, + }; + let credential = issue_token( + store, + &context, + CredentialScope::Run { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + Utc::now() + TimeDelta::days(7), + ) + .await?; + let local_command = match location { + Launcher::Local(location) => match prepare_spawn( + location, + claim, + &credential.bearer, + &self.control_variable, + ) { + Ok(command) => Some(command), + Err(error) => { + worker.launch_failure = Some("local_preparation_failed"); + worker.publish_launch_failure(store, lease).await?; + return Err(error); + } + }, + Launcher::Docker(_) => None, + }; + publish( + store, + lease, + claim, + ("local-launched", RunChange::Launched { container_id }), + ) + .await?; + let launched = match local_command { + Some(mut command) => command + .spawn() + .map(|child| { + worker.child = Some(WorkerChild::Local(child)); + }) + .map_err(ControlError::from), + None => { + if let (Launcher::Docker(location), Some(WorkerChild::Docker(child))) = + (location, &worker.child) + && let Err(error) = child + .launch(location, claim, &credential.bearer, &self.control_variable) + .await + { + worker.launch_failure = Some("docker_launch_failed"); + worker.publish_launch_failure(store, lease).await?; + return Err(error); + } + Ok(()) + } + }; + match launched { + Ok(()) => { + drop(writeln!( + std::io::stderr().lock(), + "Worker launched for run {}", + claim.request.run_id + )); + } + Err(_) => { + worker.launch_failure = Some("local_spawn_failed"); + worker.publish_launch_failure(store, lease).await?; + let claim = &worker.claim; + publish( + store, + lease, + claim, + ( + "local-spawn-failed", + RunChange::Finished { + outcome: RunOutcome::Failed, + reason: "local worker process could not start".into(), + exit_code: None, + }, + ), + ) + .await?; + self.active.swap_remove(index); + } + } + } + } + Ok(()) + } +} + +fn prepare_spawn( + location: &LocalLauncher, + claim: &RunClaim, + bearer: &str, + control_variable: &str, +) -> Result { + let mut command = Command::new(&location.executable); + command + .args(arguments(&Launcher::Local(location).scope(), claim)) + .arg(&location.project_dir) + .current_dir(&location.project_dir) + .env_clear() + .envs(std::env::vars_os().filter(|(key, _)| { + key.to_str() + .is_none_or(|key| !excluded(key, control_variable)) + })) + .envs(environment(claim, bearer)) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .kill_on_drop(true); + #[cfg(target_os = "linux")] + crate::workers::_helpers::local::inherit_claim(&mut command, location, claim)?; + Ok(command) +} + +impl ClaimedWorker { + pub(in crate::workers) async fn publish_launch_failure( + &mut self, + store: &ControlStore, + lease: &SchedulerLease, + ) -> Result<(), ControlError> { + if let Some(code) = self.launch_failure { + publish( + store, + lease, + &self.claim, + (code, RunChange::LaunchFailed { code: code.into() }), + ) + .await?; + self.launch_failure = None; + } + Ok(()) + } +} + +pub(in crate::workers) async fn publish( + store: &ControlStore, + lease: &SchedulerLease, + claim: &RunClaim, + (event, change): (&str, RunChange), +) -> Result<(), ControlError> { + change_run( + store, + RunAuthority::Scheduler(lease), + &RunChangeRequest { + event_id: Uuid::new_v5(&claim.claim_id, event.as_bytes()), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await?; + Ok(()) +} + +pub(in crate::workers) fn terminal(status: RunStatus) -> bool { + !matches!( + status, + RunStatus::Queued | RunStatus::Launched | RunStatus::Running | RunStatus::Unresponsive + ) +} + +pub(in crate::workers) fn exit_code(exit: ExitStatus) -> i64 { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + i64::from( + exit.code() + .unwrap_or_else(|| 128 + exit.signal().unwrap_or(0)), + ) + } + #[cfg(not(unix))] + i64::from(exit.code().unwrap_or(1)) +} diff --git a/crates/sqlbuild-cli/src/workers/constants.rs b/crates/sqlbuild-cli/src/workers/constants.rs new file mode 100644 index 0000000000..06a889dbfb --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/constants.rs @@ -0,0 +1,7 @@ +//! Immutable Docker image identities use the full SHA-256 hexadecimal digest. + +pub(super) const IMAGE_ID_HEX_LENGTH: usize = 64; + +/// Keep inherited claim ownership away from standard IO and the runtime's usual low descriptors. +#[cfg(target_os = "linux")] +pub(super) const LOCAL_CLAIM_FD: i32 = 198; diff --git a/crates/sqlbuild-cli/src/workers/main/docker_image_sources.rs b/crates/sqlbuild-cli/src/workers/main/docker_image_sources.rs new file mode 100644 index 0000000000..09bba564fc --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/main/docker_image_sources.rs @@ -0,0 +1,19 @@ +//! Derive publication image sources from the same privileged Docker launcher configuration. + +use sqlbuild_orchestration::api::models::DockerImageSource; + +use crate::workers::models::Workers; + +pub(crate) fn docker_image_sources(workers: &Workers) -> Vec { + workers + .configuration + .docker + .iter() + .map(|location| DockerImageSource { + location_id: location.location_id, + target: location.target.clone(), + socket: location.socket.clone(), + container: location.location_container.clone(), + }) + .collect() +} diff --git a/crates/sqlbuild-cli/src/workers/main/mod.rs b/crates/sqlbuild-cli/src/workers/main/mod.rs new file mode 100644 index 0000000000..576cfbff61 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/main/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod docker_image_sources; +pub(crate) mod open_development_workers; +pub(crate) mod open_workers; +pub(crate) mod terminate_child; diff --git a/crates/sqlbuild-cli/src/workers/main/open_development_workers.rs b/crates/sqlbuild-cli/src/workers/main/open_development_workers.rs new file mode 100644 index 0000000000..846eadcc68 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/main/open_development_workers.rs @@ -0,0 +1,33 @@ +//! Development uses the ordinary local launcher without writing derived launcher configuration. + +use std::path::PathBuf; + +use sqlbuild_orchestration::store::errors::ControlError; +use uuid::Uuid; + +use crate::workers::models::{Launchers, LocalLauncher, Workers}; + +pub(crate) fn open_development_workers( + (location_id, target): (Uuid, String), + project_dir: PathBuf, + executable: PathBuf, + server_url: String, +) -> Result { + Ok(Workers { + configuration: Launchers { + local: vec![LocalLauncher { + location_id, + target, + server_url, + max_workers: std::thread::available_parallelism()?, + project_dir, + executable, + }], + docker: Vec::new(), + concurrency_limits: Default::default(), + }, + active: Vec::new(), + control_variable: "SQLBUILD_CONTROL_DATABASE_URL".into(), + recovered_epoch: None, + }) +} diff --git a/crates/sqlbuild-cli/src/workers/main/open_workers.rs b/crates/sqlbuild-cli/src/workers/main/open_workers.rs new file mode 100644 index 0000000000..544dae4fc5 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/main/open_workers.rs @@ -0,0 +1,19 @@ +//! Open the installation's process owner before accepting scheduler work. + +use sqlbuild_orchestration::store::errors::ControlError; + +use crate::workers::_helpers::configuration::read_configuration; +use crate::workers::models::Workers; + +pub(crate) fn open_workers( + configuration_path: Option<&str>, + control_variable: String, + scheduler_enabled: bool, +) -> Result { + Ok(Workers { + configuration: read_configuration(configuration_path, scheduler_enabled)?, + active: Vec::new(), + control_variable, + recovered_epoch: None, + }) +} diff --git a/crates/sqlbuild-cli/src/workers/main/terminate_child.rs b/crates/sqlbuild-cli/src/workers/main/terminate_child.rs new file mode 100644 index 0000000000..1c2c371306 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/main/terminate_child.rs @@ -0,0 +1,21 @@ +//! Signal only a child whose process handle is still owned by the caller. + +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::process::Child; + +pub(crate) fn terminate_child(child: &mut Child) -> Result<(), ControlError> { + #[cfg(unix)] + if let Some(id) = child.id() { + let raw = i32::try_from(id) + .map_err(|error| ControlError::Invalid(format!("invalid child process ID: {error}")))?; + let pid = rustix::process::Pid::from_raw(raw) + .ok_or_else(|| ControlError::Invalid("invalid child process ID".into()))?; + match rustix::process::kill_process(pid, rustix::process::Signal::TERM) { + Ok(()) | Err(rustix::io::Errno::SRCH) => {} + Err(error) => return Err(std::io::Error::from(error).into()), + } + } + #[cfg(not(unix))] + let _ = child; + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/workers/mod.rs b/crates/sqlbuild-cli/src/workers/mod.rs new file mode 100644 index 0000000000..59bf71d77a --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/mod.rs @@ -0,0 +1,6 @@ +//! Native process owners launch claim-scoped workers without transferring control-store access. + +mod _helpers; +mod constants; +pub(crate) mod main; +pub(crate) mod models; diff --git a/crates/sqlbuild-cli/src/workers/models.rs b/crates/sqlbuild-cli/src/workers/models.rs new file mode 100644 index 0000000000..35639fe18e --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/models.rs @@ -0,0 +1,111 @@ +//! Installation launch configuration and the physical children retained until reaping. + +use std::collections::BTreeMap; +use std::num::{NonZeroU32, NonZeroUsize}; +use std::path::PathBuf; +use std::time::Instant; + +use sqlbuild_orchestration::runs::models::RunClaim; +use tokio::process::Child; +use uuid::Uuid; + +/// Physical ownership and stop evidence are independent of the scheduler's durable run state. +#[derive(Debug)] +pub(super) enum WorkerChild { + Local(Child), + #[cfg(target_os = "linux")] + RecoveredLocal(LocalWorker), + Docker(DockerWorker), +} + +/// The inherited lock proves physical ownership without relying on a remembered process ID. +#[cfg(target_os = "linux")] +#[derive(Debug)] +pub(super) struct LocalWorker { + pub(super) lock: std::fs::File, + pub(super) run_id: Uuid, + pub(super) stopped: bool, +} + +#[derive(Clone, Copy, Debug)] +pub(super) enum WorkerExit { + Code(i64), + Absent, +} + +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Launchers { + #[serde(default)] + pub(super) local: Vec, + #[serde(default)] + pub(super) docker: Vec, + #[serde(default)] + pub(super) concurrency_limits: BTreeMap, +} + +#[derive(Clone, Copy, Debug)] +pub(super) struct LauncherScope<'a> { + pub(super) location_id: Uuid, + pub(super) target: &'a str, + pub(super) server_url: &'a str, + pub(super) max_workers: NonZeroUsize, +} + +/// No executable, project path or API address is supplied by an untrusted run request. +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct LocalLauncher { + pub(super) location_id: Uuid, + pub(super) target: String, + pub(super) server_url: String, + pub(super) max_workers: NonZeroUsize, + pub(super) project_dir: PathBuf, + pub(super) executable: PathBuf, +} + +/// The installation chooses the daemon and location container; the snapshot pins the worker image. +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct DockerLauncher { + pub(super) location_id: Uuid, + pub(super) target: String, + pub(super) server_url: String, + pub(super) max_workers: NonZeroUsize, + pub(super) socket: PathBuf, + pub(super) location_container: String, + pub(super) executable: String, +} + +#[derive(Clone, Copy, Debug)] +pub(super) enum Launcher<'a> { + Local(&'a LocalLauncher), + Docker(&'a DockerLauncher), +} + +/// Retain the deterministic identity before create/start, including uncertain HTTP responses. +#[derive(Debug)] +pub(super) struct DockerWorker { + pub(super) client: reqwest::Client, + pub(super) name: String, + pub(super) claim_id: Uuid, + pub(super) image_id: String, +} + +#[derive(Debug)] +pub(super) struct ClaimedWorker { + pub(super) claim: RunClaim, + pub(super) child: Option, + pub(super) stopping: Option, + pub(super) exit: Option, + pub(super) launch_failure: Option<&'static str>, +} + +/// Physical ownership survives leadership loss; only the current fence changes durable lifecycle. +#[derive(Debug)] +pub(crate) struct Workers { + pub(super) configuration: Launchers, + pub(super) active: Vec, + pub(super) control_variable: String, + pub(super) recovered_epoch: Option, +} diff --git a/crates/sqlbuild-cli/tests/native_commands.rs b/crates/sqlbuild-cli/tests/native_commands.rs new file mode 100644 index 0000000000..163c973a81 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands.rs @@ -0,0 +1,16 @@ +#[path = "native_commands/admin/helpers.rs"] +mod admin_helpers; +#[path = "native_commands/helpers.rs"] +mod helpers; +#[path = "native_commands/admin/test_admin.rs"] +mod test_admin; +#[path = "native_commands/test_control.rs"] +mod test_control; +#[path = "native_commands/admin/test_passwords.rs"] +mod test_passwords; +#[path = "native_commands/test_proxy.rs"] +mod test_proxy; +#[path = "native_commands/test_types.rs"] +mod test_types; +#[path = "native_commands/worker/test_worker.rs"] +mod worker; diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs new file mode 100644 index 0000000000..4820f96f28 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs @@ -0,0 +1,117 @@ +//! Run the isolated native executable; never include returned bearer bytes in assertion messages. + +use std::io::Write; +use std::process::{Command, Output, Stdio}; + +use serde_json::Value; +use sqlbuild_orchestration::declarations::models::TriggerRecord; +use uuid::Uuid; + +use crate::test_types::TestControl; + +pub(crate) fn administer(control: &TestControl, arguments: &[&str]) -> Value { + let output = invoke_admin(control, arguments); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stderr), + "Opening control store...\nControl store ready\nControl administration completed successfully\n" + ); + serde_json::from_slice(&output.stdout).expect("JSON-only admin stdout") +} + +pub(crate) fn invoke_admin(control: &TestControl, arguments: &[&str]) -> Output { + admin_command(control, arguments) + .output() + .expect("native administration process") +} + +fn admin_command(control: &TestControl, arguments: &[&str]) -> Command { + let mut command = Command::new( + control + .directory + .path() + .join(format!("sqb{}", std::env::consts::EXE_SUFFIX)), + ); + command + .arg("admin") + .args(arguments) + .current_dir(control.directory.path()) + .env("PATH", "") + .env_remove("SQLBUILD_CONTROL_DATABASE_URL"); + command +} + +pub(crate) fn invoke_password(control: &TestControl, principal: &str, input: &[u8]) -> Output { + let mut child = admin_command(control, &["set-password", principal]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("native password provisioning"); + let mut stdin = child.stdin.take().expect("password pipe"); + stdin + .write_all(input) + .expect("deliver password through stdin"); + drop(stdin); + child + .wait_with_output() + .expect("password command completed") +} + +pub(crate) async fn read_triggers( + url: &str, + bearer: &str, + (location, target): (Uuid, &str), +) -> Vec { + let mut endpoint = reqwest::Url::parse(url).expect("API address"); + endpoint + .path_segments_mut() + .expect("API path") + .pop_if_empty() + .extend([ + "api", + "locations", + &location.to_string(), + "targets", + target, + "triggers", + ]); + reqwest::Client::new() + .get(endpoint) + .bearer_auth(bearer) + .timeout(std::time::Duration::from_secs(5)) + .send() + .await + .expect("trigger response") + .error_for_status() + .expect("authorized trigger read") + .json() + .await + .expect("trigger records") +} + +pub(crate) fn audit_actions(control: &TestControl, bearer: &str) -> Vec { + let connection = + rusqlite::Connection::open(control.directory.path().join(".sqlbuild/control.sqlite3")) + .expect("inspect control audit"); + let mut query = connection.prepare("SELECT action, actor, granting_role, scope_json || before_json || after_json FROM sqb_security_audit ORDER BY occurred_at, event_id") + .expect("audit records"); + query + .query_map([], |row| { + let action: String = row.get(0)?; + let actor: String = row.get(1)?; + let role: String = row.get(2)?; + let payload: String = row.get(3)?; + assert_eq!(actor, "local-control-store-owner"); + assert_eq!(role, "control-store-owner"); + assert!(!payload.contains(bearer)); + Ok(action) + }) + .expect("read audit") + .collect::>() + .expect("decode audit") +} diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs new file mode 100644 index 0000000000..d5a8bcaf86 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs @@ -0,0 +1,240 @@ +//! Native bootstrap credentials authorize real publication and stop working after revocation. + +#[path = "test_types.rs"] +mod test_types; + +use std::collections::BTreeMap; + +use chrono::{Duration, Utc}; +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use uuid::Uuid; + +use crate::admin_helpers::{administer, audit_actions, invoke_admin, read_triggers}; +use crate::helpers::{api_ready, start_control}; +#[cfg(unix)] +use crate::test_admin::test_types::AdminDeliveryTestCase; +use crate::test_admin::test_types::{AdminTestCase, TriggerAdminTestCase}; + +#[test] +fn given_a_standalone_control_server_when_bootstrapped_by_an_operator_then_scoped_publication_and_revocation_are_audited() + { + let test_cases = [AdminTestCase { + description: "native operator bootstrap preserves labels and scopes credentials", + location_name: "orders / café", + expected_name: "orders / café", + expected_actions: [ + "location.created", + "location.created", + "token.issued", + "token.issued", + "token.revoked", + "token.revoked", + "token.rotated", + "trigger.changed", + "trigger.changed", + "trigger.changed", + "trigger.changed", + "trigger.changed", + ], + }]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(&["server", "--port", "0", "--no-scheduler"]); + let url = api_ready(&mut control).await; + let first = administer(&control, &["create-location", test_case.location_name]); + let second = administer(&control, &["create-location", "customers"]); + let location_id = + Uuid::parse_str(first["location_id"].as_str().expect("location UUID")) + .expect("UUID"); + let other_id = Uuid::parse_str(second["location_id"].as_str().expect("location UUID")) + .expect("UUID"); + assert_ne!(location_id, other_id); + assert_eq!( + first["name"], test_case.expected_name, + "{}", + test_case.description + ); + let expiry = (Utc::now() + Duration::hours(1)).to_rfc3339(); + let credential = administer( + &control, + &[ + "issue-location-token", + &location_id.to_string(), + "--expires-at", + &expiry, + ], + ); + let bearer = credential["bearer"].as_str().expect("one-time bearer"); + assert_eq!(credential["identity"]["scope"]["kind"], "location"); + assert_eq!( + credential["identity"]["scope"]["location_id"], + location_id.to_string() + ); + let mut snapshot = SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: None, + environment_hash: "orders-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::from([("model:orders".into(), SnapshotNode { + kind: "model".into(), + definition: serde_json::json!({"materialization": "table"}), + raw_sql: Some("select 1 as order_id".into()), + compiled_sql: Some("select 1 as order_id".into()), + python_source: None, + })]), + documents: BTreeMap::from([ + ("jobs".into(), serde_json::json!({"orders": {"description": "Refresh orders.", "selection": ["model:orders"]}})), + ("triggers".into(), serde_json::json!({"daily_orders": { + "description": "Daily orders.", "job_key": "orders", "default_status": "paused", + "schedule": {"kind": "cron", "expression": "0 2 * * *", "timezone": "UTC"}, + }})), + ]), + }; + let published = publish_remote_snapshot(&url, bearer, &snapshot) + .await + .expect("operator-issued credential publishes"); + assert_eq!(published.location_id, location_id); + let rotated = administer(&control, &["rotate-token", credential["identity"]["token_id"].as_str().expect("old token UUID")]); + assert_eq!(rotated["identity"]["scope"], credential["identity"]["scope"]); + assert_eq!(rotated["identity"]["expires_at"], credential["identity"]["expires_at"]); + assert_ne!(rotated["identity"]["token_id"], credential["identity"]["token_id"]); + assert_ne!(rotated["bearer"], credential["bearer"]); + assert!(matches!(publish_remote_snapshot(&url, bearer, &snapshot).await, + Err(ClientError::Rejected(reqwest::StatusCode::UNAUTHORIZED)))); + let repeated = invoke_admin(&control, &["rotate-token", credential["identity"]["token_id"].as_str().expect("old token UUID")]); + assert!(!repeated.status.success()); + assert!(repeated.stdout.is_empty()); + let credential = rotated; + let bearer = credential["bearer"].as_str().expect("rotated one-time bearer"); + let trigger_id = read_triggers(&url, bearer, (location_id, "development")).await[0].trigger_id.to_string(); + for case in [ + TriggerAdminTestCase { description: "enable overrides the paused code default", change: "enable", cursor_arguments: &[], expected_status: "running", expected_cursor: None }, + TriggerAdminTestCase { description: "pause overrides an enabled trigger", change: "pause", cursor_arguments: &[], expected_status: "paused", expected_cursor: None }, + TriggerAdminTestCase { description: "reset-default restores the paused code default", change: "reset-default", cursor_arguments: &[], expected_status: "paused", expected_cursor: None }, + TriggerAdminTestCase { description: "set-cursor normalizes the timestamp offset", change: "set-cursor", cursor_arguments: &["--cursor", "2026-07-03T01:17:23+05:30"], expected_status: "paused", expected_cursor: Some("2026-07-02T19:47:23+00:00") }, + TriggerAdminTestCase { description: "reset-cursor removes the durable cursor", change: "reset-cursor", cursor_arguments: &[], expected_status: "paused", expected_cursor: None }, + ] { + let event_id = Uuid::now_v7().to_string(); + let mut arguments = vec!["change-trigger", &trigger_id, case.change, "--event-id", &event_id, "--reason", "Order refresh maintenance"]; + arguments.extend_from_slice(case.cursor_arguments); + for expected in [true, false] { + let result = administer(&control, &arguments); + assert_eq!(result["applied"], expected, "{}", case.description); + assert_eq!(result["event_id"], event_id); + } + let mut conflict = arguments.clone(); + conflict[6] = "Different request content"; + assert!(!invoke_admin(&control, &conflict).status.success(), "{}", case.description); + let triggers = read_triggers(&url, bearer, (location_id, "development")).await; + assert_eq!(triggers[0].status.as_str(), case.expected_status, "{}", case.description); + assert_eq!(triggers[0].last_tick_for.map(|value| value.to_rfc3339()).as_deref(), case.expected_cursor, "{}", case.description); + } + snapshot.generation = 2; + snapshot.documents.get_mut("triggers").expect("trigger document")["daily_orders"]["default_status"] = serde_json::json!("running"); + publish_remote_snapshot(&url, bearer, &snapshot).await.expect("publish a new default"); + assert_eq!(read_triggers(&url, bearer, (location_id, "development")).await[0].status.as_str(), "running", "reset-default removed the operator override"); + snapshot.location_id = other_id; + assert!(matches!( + publish_remote_snapshot(&url, bearer, &snapshot).await, + Err(ClientError::Rejected(reqwest::StatusCode::FORBIDDEN)) + )); + snapshot.location_id = location_id; + snapshot.generation = 2; + let token_id = credential["identity"]["token_id"] + .as_str() + .expect("credential ID"); + for expected in [true, false] { + let revoked = administer(&control, &["revoke-token", token_id]); + assert_eq!(revoked["revoked"], expected); + assert_eq!(revoked["token_id"], token_id); + } + assert!(matches!( + publish_remote_snapshot(&url, bearer, &snapshot).await, + Err(ClientError::Rejected(reqwest::StatusCode::UNAUTHORIZED)) + )); + let mut actions = audit_actions(&control, bearer); + actions.sort(); + assert_eq!( + actions, test_case.expected_actions, + "{}", + test_case.description + ); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } +} + +#[test] +fn given_a_committed_admin_operation_when_json_output_is_lost_then_all_response_sizes_fail_without_claiming_rollback() + { + let test_cases = [ + AdminDeliveryTestCase { + description: "small response", + name_bytes: 8, + expected_exit: 1, + expected_rows: (1, 1), + }, + AdminDeliveryTestCase { + description: "Python compatibility drop window", + name_bytes: 5000, + expected_exit: 1, + expected_rows: (1, 1), + }, + AdminDeliveryTestCase { + description: "large response", + name_bytes: 9000, + expected_exit: 1, + expected_rows: (1, 1), + }, + ]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("isolated administration directory"); + let binary = crate::helpers::isolated_binary(directory.path()); + let output = std::process::Command::new(binary) + .args([ + "admin", + "create-location", + &"x".repeat(test_case.name_bytes), + ]) + .current_dir(directory.path()) + .env("PATH", "") + .env_remove("SQLBUILD_CONTROL_DATABASE_URL") + .stdout( + std::fs::OpenOptions::new() + .write(true) + .open("/dev/full") + .expect("failing output device"), + ) + .output() + .expect("native administration"); + assert_eq!( + output.status.code(), + Some(test_case.expected_exit), + "{}", + test_case.description + ); + assert_eq!( + String::from_utf8_lossy(&output.stderr), + "Opening control store...\nControl store ready\nControl administration JSON result delivery failed; changes may already be committed\n", + "{}", + test_case.description + ); + let connection = + rusqlite::Connection::open(directory.path().join(".sqlbuild/control.sqlite3")) + .expect("inspect committed operation"); + let rows: (i64, i64) = connection.query_row("SELECT (SELECT COUNT(*) FROM sqb_locations), (SELECT COUNT(*) FROM sqb_security_audit)", [], |row| Ok((row.get(0)?, row.get(1)?))).expect("location and audit remain committed"); + assert_eq!(rows, test_case.expected_rows, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_passwords.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_passwords.rs new file mode 100644 index 0000000000..ed64b7643c --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_passwords.rs @@ -0,0 +1,240 @@ +//! Standalone owner provisioning enables browser identity, never machine-route authority. + +use std::time::Duration; + +use reqwest::{StatusCode, header}; +use serde_json::{Value, json}; + +use crate::admin_helpers::{administer, invoke_admin, invoke_password}; +use crate::helpers::{api_ready, start_control, wait_for_exit}; +use crate::test_types::{AdministratorCliTestCase, PasswordCliTestCase, PasswordOriginTestCase}; + +#[test] +fn given_local_store_ownership_when_create_user_runs_then_only_the_first_process_initializes_an_admin() + { + let test_cases = [AdministratorCliTestCase { + description: "once-only owner bootstrap and authenticated administrator changes", + expected_administrators: 1, + }]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(&["server","--port","0","--no-scheduler","--password-origin","https://control.example.com"]); + let url = api_ready(&mut control).await; + let first = administer(&control,&["create-user","orders-admin"]); + let ordinary = administer(&control,&["create-principal","orders-reader"]); + assert_eq!(first["role"],"admin"); + let connection = rusqlite::Connection::open(control.directory.path().join(".sqlbuild/control.sqlite3")).expect("inspect initialization"); + let (count,id): (i64,String) = connection.query_row("SELECT COUNT(*),MIN(principal_id) FROM sqb_role_assignments WHERE role_key='admin'",[],|row| Ok((row.get(0)?,row.get(1)?))).expect("initial administrator membership"); + assert_eq!(count,test_case.expected_administrators,"{}",test_case.description); + assert_eq!(id,first["principal_id"].as_str().expect("initial administrator UUID")); + assert_ne!(id,ordinary["principal_id"].as_str().expect("ordinary person UUID")); + let client = reqwest::Client::builder().timeout(Duration::from_secs(5)).build().expect("HTTP client"); + let mut sessions = Vec::new(); + for person in [&first,&ordinary] { + let output = invoke_password(&control,person["principal_id"].as_str().expect("person UUID"),b"orders-admin-password-2026"); + assert!(output.status.success()); + let response = client.post(format!("{url}/api/auth/password/login")) + .header(header::HOST,"control.example.com").header(header::ORIGIN,"https://control.example.com") + .json(&json!({"username":person["name"],"password":"orders-admin-password-2026"})).send().await.expect("password login"); + assert_eq!(response.status(),StatusCode::OK); + let cookie = response.headers().get_all(header::SET_COOKIE).iter().filter_map(|value| value.to_str().ok()) + .find(|value| value.starts_with("__Host-sqb-session=")).expect("session cookie").split(';').next().expect("cookie pair").to_owned(); + let body: Value = response.json().await.expect("session body"); + sessions.push((cookie,body["csrf_token"].as_str().expect("CSRF token").to_owned())); + } + let ordinary_id = ordinary["principal_id"].as_str().expect("ordinary person UUID"); + for (actor,target,action,expected) in [ + (1,ordinary_id,"grant",StatusCode::FORBIDDEN), + (0,id.as_str(),"revoke",StatusCode::FORBIDDEN), + (0,ordinary_id,"grant",StatusCode::OK), + (1,id.as_str(),"revoke",StatusCode::OK), + (0,id.as_str(),"grant",StatusCode::FORBIDDEN), + (1,ordinary_id,"disable",StatusCode::FORBIDDEN), + ] { + let response = client.post(format!("{url}/api/admin/people/{target}")) + .header(header::HOST,"control.example.com").header(header::ORIGIN,"https://control.example.com") + .header(header::COOKIE,&sessions[actor].0).header("x-sqb-csrf",&sessions[actor].1) + .json(&action).send().await.expect("administrator mutation response"); + assert_eq!(response.status(),expected,"{action} by actor {actor}"); + let body: Value = response.json().await.expect("mutation result"); + assert_eq!(body["applied"].as_bool(), (expected == StatusCode::OK).then_some(true)); + assert_eq!(body["error"].as_str().is_some_and(|message| message.contains("decision")),expected == StatusCode::FORBIDDEN); + } + for disabled_at in [None,Some("2026-01-01T00:00:00Z")] { + connection.execute("UPDATE sqb_principals SET disabled_at=?1",[disabled_at]).expect("simulate out-of-band administrator loss"); + let rejected = invoke_admin(&control,&["create-user","replacement-admin"]); + assert!(!rejected.status.success()); + assert!(rejected.stdout.is_empty()); + assert!(String::from_utf8_lossy(&rejected.stderr).contains("bootstrap is closed")); + } + let initialized: i64 = connection.query_row("SELECT COUNT(*) FROM sqb_security_audit WHERE action='admin.initialized' AND actor='local-control-store-owner'",[],|row| row.get(0)).expect("bootstrap audit"); + assert_eq!(initialized,1); + let recovery = invoke_admin(&control, &["grant-admin", id.as_str()]); + assert!(recovery.status.success(), "{}", String::from_utf8_lossy(&recovery.stderr)); + assert_eq!(serde_json::from_slice::(&recovery.stdout).expect("JSON-only grant"), json!({"principal_id":id,"role":"admin"})); + let audits: i64 = connection.query_row("SELECT COUNT(*) FROM sqb_security_audit WHERE action='admin.granted'",[],|row| row.get(0)).expect("grant audit"); + assert_eq!(audits,1); + let previous = client.get(format!("{url}/api/auth/session")).header(header::HOST,"control.example.com") + .header(header::COOKIE,&sessions[0].0).send().await.expect("old session after grant-admin"); + assert_eq!(previous.status(),StatusCode::UNAUTHORIZED); + let response = client.post(format!("{url}/api/auth/password/login")) + .header(header::HOST,"control.example.com").header(header::ORIGIN,"https://control.example.com") + .json(&json!({"username":first["name"],"password":"orders-admin-password-2026"})).send().await.expect("recovered admin login"); + assert_eq!(response.status(),StatusCode::OK); + let cookie = response.headers().get_all(header::SET_COOKIE).iter().filter_map(|value| value.to_str().ok()) + .find(|value| value.starts_with("__Host-sqb-session=")).expect("new session cookie").split(';').next().expect("cookie pair").to_owned(); + let body: Value = response.json().await.expect("recovered session"); + let denied = client.post(format!("{url}/api/admin/people/{id}")) + .header(header::HOST,"control.example.com").header(header::ORIGIN,"https://control.example.com") + .header(header::COOKIE,cookie).header("x-sqb-csrf",body["csrf_token"].as_str().expect("new CSRF")) + .json(&"revoke").send().await.expect("final-admin protection after recovery"); + assert_eq!(denied.status(),StatusCode::FORBIDDEN); + let rejected = invoke_admin(&control,&["create-user","replacement-admin"]); + assert!(!rejected.status.success()); + assert!(String::from_utf8_lossy(&rejected.stderr).contains("bootstrap is closed")); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } +} + +#[test] +fn given_owner_provisioned_passwords_when_a_standalone_server_authenticates_then_resets_invalidate_sessions_without_granting_authority() + { + let test_cases = [PasswordCliTestCase { + description: "owner-provisioned identity remains separate from location authority", + password: " orders-password-café ", + replacement: "customers-password-2026", + expected_password_rows: 1, + expected_audits: 2, + }]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(&["server", "--port", "0", "--no-scheduler", "--password-origin", "https://control.example.com"]); + let url = api_ready(&mut control).await; + let person = administer(&control, &["create-principal", "orders-reader"]); + let principal = person["principal_id"].as_str().expect("person identity"); + let password = test_case.password; + let invalid_inputs = [Vec::new(), b"short\n".to_vec(), vec![b'x'; 5000], vec![0xff, b'\n']]; + for input in invalid_inputs { + let output = invoke_password(&control, principal, &input); + assert!(!output.status.success(), "invalid stdin must not set a password"); + assert!(output.stdout.is_empty()); + } + + let client = reqwest::Client::builder().timeout(Duration::from_secs(5)).build().expect("HTTP client"); + let login = |password: &str| client.post(format!("{url}/api/auth/password/login")) + .header(header::HOST, "control.example.com") + .header(header::ORIGIN, "https://control.example.com") + .json(&json!({"username": "orders-reader", "password": password})).send(); + let output = invoke_password(&control, principal, format!("{password}\r\n").as_bytes()); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + let result: Value = serde_json::from_slice(&output.stdout).expect("JSON-only provisioning result"); + assert_eq!(result, json!({"principal_id": principal, "password_set": true})); + assert!(!String::from_utf8_lossy(&output.stderr).contains(password)); + let response = login(password).await.expect("provisioned password login"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); + let cookie = response.headers().get_all(header::SET_COOKIE).iter() + .filter_map(|header| header.to_str().ok()) + .find(|value| value.starts_with("__Host-sqb-session=")) + .expect("host-only session cookie").split(';').next().expect("cookie pair").to_owned(); + let body: Value = response.json().await.expect("session identity"); + assert_eq!(body["identity"]["principal_id"], principal); + let session = client.get(format!("{url}/api/auth/session")) + .header(header::HOST, "control.example.com").header(header::COOKIE, &cookie) + .send().await.expect("current session"); + assert_eq!(session.status(), StatusCode::OK); + let machine = client.get(format!("{url}/api/locations/{}/targets/development/triggers", uuid::Uuid::now_v7())) + .header(header::HOST, "control.example.com").header(header::COOKIE, &cookie) + .send().await.expect("browser cookie has no location authority"); + assert_eq!(machine.status(), StatusCode::UNAUTHORIZED); + let reset = invoke_password(&control, principal, test_case.replacement.as_bytes()); + assert!(reset.status.success(), "{}", String::from_utf8_lossy(&reset.stderr)); + assert_eq!(serde_json::from_slice::(&reset.stdout).expect("reset JSON"), json!({"principal_id": principal, "password_set": true})); + assert!(!String::from_utf8_lossy(&reset.stderr).contains(test_case.replacement)); + let previous = client.get(format!("{url}/api/auth/session")) + .header(header::HOST, "control.example.com").header(header::COOKIE, cookie) + .send().await.expect("previous session after child-process reset"); + assert_eq!(previous.status(), StatusCode::UNAUTHORIZED); + assert_eq!(login(password).await.expect("old password rejected").status(), StatusCode::UNAUTHORIZED); + assert_eq!(login(test_case.replacement).await.expect("replacement password accepted").status(), StatusCode::OK); + let connection = rusqlite::Connection::open(control.directory.path().join(".sqlbuild/control.sqlite3")).expect("inspect durable provisioning"); + let (count, hash): (i64, String) = connection.query_row( + "SELECT COUNT(*), MIN(password_hash) FROM sqb_password_credentials", [], + |row| Ok((row.get(0)?, row.get(1)?))).expect("password storage"); + assert_eq!(count, test_case.expected_password_rows, "{}", test_case.description); + assert!(hash.starts_with("$argon2id$")); + let audits: i64 = connection.query_row( + "SELECT COUNT(*) FROM sqb_security_audit WHERE action='password.changed' AND actor='local-control-store-owner' AND granting_role='control-store-owner'", [], + |row| row.get(0)).expect("owner authority audit"); + assert_eq!(audits, test_case.expected_audits, "{}", test_case.description); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } +} + +#[test] +fn given_invalid_password_origins_when_a_standalone_server_starts_then_it_exits_with_a_configuration_error() + { + let test_cases = [ + PasswordOriginTestCase { + description: "HTTP is not an external HTTPS origin", + origin: "http://control.example.com", + expected_error: "password authentication requires an HTTPS origin", + }, + PasswordOriginTestCase { + description: "origin cannot contain a path", + origin: "https://control.example.com/path", + expected_error: "password authentication requires an HTTPS origin", + }, + PasswordOriginTestCase { + description: "origin cannot contain credentials", + origin: "https://user:password@control.example.com", + expected_error: "password authentication requires an HTTPS origin", + }, + PasswordOriginTestCase { + description: "empty origin is not a request to disable authentication", + origin: "", + expected_error: "password authentication requires an HTTPS origin", + }, + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + for test_case in test_cases { + runtime.block_on(async { + let origin = test_case.origin; + let mut control = start_control(&[ + "server", + "--port", + "0", + "--no-scheduler", + "--password-origin", + origin, + ]); + let status = wait_for_exit(&mut control).await; + assert!(!status.success()); + let log = std::fs::read_to_string(&control.stderr).expect("startup error"); + assert!( + log.contains(test_case.expected_error), + "{}", + test_case.description + ); + assert!( + !log.contains(origin) || origin.is_empty(), + "invalid configuration is not echoed" + ); + }); + } +} diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs new file mode 100644 index 0000000000..e34ffece25 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs @@ -0,0 +1,22 @@ +pub(crate) struct AdminTestCase { + pub(crate) description: &'static str, + pub(crate) location_name: &'static str, + pub(crate) expected_name: &'static str, + pub(crate) expected_actions: [&'static str; 12], +} + +pub(crate) struct TriggerAdminTestCase { + pub(crate) description: &'static str, + pub(crate) change: &'static str, + pub(crate) cursor_arguments: &'static [&'static str], + pub(crate) expected_status: &'static str, + pub(crate) expected_cursor: Option<&'static str>, +} + +#[cfg(unix)] +pub(crate) struct AdminDeliveryTestCase { + pub(crate) description: &'static str, + pub(crate) name_bytes: usize, + pub(crate) expected_exit: i32, + pub(crate) expected_rows: (i64, i64), +} diff --git a/crates/sqlbuild-cli/tests/native_commands/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/helpers.rs new file mode 100644 index 0000000000..7d3fe3ba71 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/helpers.rs @@ -0,0 +1,159 @@ +use std::path::Path; +use std::process::{Command, ExitStatus, Stdio}; +use std::time::Duration; + +use crate::test_types::{ServiceTestCase, TestControl}; + +pub(crate) fn isolated_binary(directory: &Path) -> std::path::PathBuf { + let binary = directory.join(format!("sqb{}", std::env::consts::EXE_SUFFIX)); + // Do not open the executable for writing: concurrent process launches can inherit + // the copy's writable descriptor until exec, making this launch fail with ETXTBSY. + std::fs::hard_link(env!("CARGO_BIN_EXE_sqb"), &binary) + .expect("link standalone binary without Python CLI"); + binary +} + +pub(crate) fn start_control(arguments: &[&str]) -> TestControl { + let directory = tempfile::tempdir().expect("isolated native control project"); + start_control_in(directory, arguments) +} + +pub(crate) fn start_control_in(directory: tempfile::TempDir, arguments: &[&str]) -> TestControl { + isolated_binary(directory.path()); + let stderr = directory.path().join("control.log"); + let child = spawn_control((directory.path(), &stderr), arguments); + TestControl { + child, + directory, + stderr, + } +} + +pub(crate) fn spawn_control( + (directory, stderr): (&Path, &Path), + arguments: &[&str], +) -> std::process::Child { + Command::new(directory.join(format!("sqb{}", std::env::consts::EXE_SUFFIX))) + .args(arguments) + .current_dir(directory) + .env("PATH", "") + .env_remove("SQLBUILD_CONTROL_DATABASE_URL") + .stdout(Stdio::null()) + .stderr(std::fs::File::create(stderr).expect("control stderr")) + .spawn() + .expect("start native control process") +} + +pub(crate) async fn api_ready(control: &mut TestControl) -> String { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(1)) + .build() + .expect("HTTP client"); + tokio::time::timeout(Duration::from_secs(10), async { + for _ in 0..500 { + assert!( + control.child.try_wait().expect("child status").is_none(), + "control process exited: {}", + std::fs::read_to_string(&control.stderr).unwrap_or_default() + ); + tokio::time::sleep(Duration::from_millis(20)).await; + let log = std::fs::read_to_string(&control.stderr).expect("read control log"); + let Some(address) = log + .lines() + .find_map(|line| line.strip_prefix("Control API ready: ")) + else { + continue; + }; + let url = format!("http://{address}"); + let Ok(response) = client + .get(format!("{url}/api/openapi.json")) + .send() + .await + .and_then(reqwest::Response::error_for_status) + else { + continue; + }; + drop(response); + return url; + } + panic!("native API exhausted readiness attempts"); + }) + .await + .expect("native API becomes ready") +} + +pub(crate) fn health_counts(control: &TestControl) -> Option<(i64, i64)> { + let connection = rusqlite::Connection::open_with_flags( + control.directory.path().join(".sqlbuild/control.sqlite3"), + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .ok()?; + connection + .query_row( + "SELECT (SELECT COUNT(*) FROM sqb_scheduler_lease), \ + (SELECT COUNT(*) FROM recall_events WHERE event_type='platform_heartbeat')", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .ok() +} + +pub(crate) async fn wait_for_health( + control: &mut TestControl, + case: &ServiceTestCase, +) -> (i64, i64) { + tokio::time::timeout(Duration::from_secs(5), async { + for _ in 0..250 { + assert!( + control.child.try_wait().expect("service status").is_none(), + "service exited: {}", + std::fs::read_to_string(&control.stderr).unwrap_or_default() + ); + tokio::time::sleep(Duration::from_millis(20)).await; + let Some(counts) = health_counts(control).filter(|(leaders, heartbeats)| { + *leaders == case.expected_leaders && *heartbeats >= case.expected_heartbeats + }) else { + continue; + }; + return counts; + } + panic!("control service exhausted health readiness attempts"); + }) + .await + .expect("native control service publishes its health state") +} + +#[cfg(unix)] +pub(crate) async fn terminate(control: &mut TestControl) { + let status = Command::new("kill") + .args(["-TERM", &control.child.id().to_string()]) + .status() + .expect("send SIGTERM"); + assert!(status.success()); + let status = wait_for_exit(control).await; + assert!( + status.success(), + "shutdown failed: {}", + std::fs::read_to_string(&control.stderr).unwrap_or_default() + ); + assert!( + std::fs::read_to_string(&control.stderr) + .expect("shutdown log") + .ends_with("Control service stopped successfully\n") + ); +} + +pub(crate) async fn wait_for_exit(control: &mut TestControl) -> ExitStatus { + tokio::time::timeout(Duration::from_secs(5), async { + for _ in 0..250 { + let Some(exit) = control.child.try_wait().expect("control process exit") else { + tokio::time::sleep(Duration::from_millis(20)).await; + continue; + }; + return exit; + } + panic!("control process exhausted shutdown attempts"); + }) + .await + .expect("control process exits promptly") +} diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs new file mode 100644 index 0000000000..59bea6e574 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -0,0 +1,461 @@ +use std::process::Command; + +use crate::helpers::{api_ready, isolated_binary, start_control, wait_for_health}; +use crate::test_types::{CommandTestCase, OpenApiTestCase, ServiceTestCase, TriggerCursorTestCase}; + +#[test] +fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_python_and_database_access_are_not_required() + { + let test_cases = [ + CommandTestCase { + description: "native server help", + arguments: &["server", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb ", + expected_stderr: "", + }, + CommandTestCase { + description: "native scheduler help", + arguments: &["scheduler", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb ", + expected_stderr: "", + }, + CommandTestCase { + description: "native privileged administration help", + arguments: &["admin", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin ", + expected_stderr: "", + }, + CommandTestCase { + description: "native password provisioning help", + arguments: &["admin", "set-password", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin set-password ", + expected_stderr: "", + }, + CommandTestCase { + description: "native trigger administration help", + arguments: &["admin", "change-trigger", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin change-trigger ", + expected_stderr: "", + }, + CommandTestCase { + description: "native cancellation help", + arguments: &["admin", "cancel-run", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin cancel-run ", + expected_stderr: "", + }, + CommandTestCase { + description: "native manual job request help", + arguments: &["admin", "request-run", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin request-run ", + expected_stderr: "", + }, + CommandTestCase { + description: "native location work help", + arguments: &["admin", "check-location", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin check-location ", + expected_stderr: "", + }, + CommandTestCase { + description: "blank location work reason rejected before store initialization", + arguments: &[ + "admin", + "check-location", + "00000000-0000-0000-0000-000000000001", + "--target", + "development", + "--request-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + " ", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: target and reason must not be blank\n", + }, + CommandTestCase { + description: "blank job rejected before store initialization", + arguments: &[ + "admin", + "request-run", + "00000000-0000-0000-0000-000000000001", + " ", + "--target", + "development", + "--run-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + "Refresh orders", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: target and job must not be blank\n", + }, + CommandTestCase { + description: "oversized manual queue priority rejected before store initialization", + arguments: &[ + "admin", + "request-run", + "00000000-0000-0000-0000-000000000001", + "daily_orders", + "--target", + "development", + "--run-id", + "00000000-0000-0000-0000-000000000002", + "--priority", + "9223372036854775808", + "--reason", + "Refresh orders", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: --priority must fit a signed 64-bit integer\n", + }, + CommandTestCase { + description: "blank cancellation reason rejected before store initialization", + arguments: &[ + "admin", + "cancel-run", + "00000000-0000-0000-0000-000000000001", + "--event-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + " ", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: --reason must not be blank\n", + }, + CommandTestCase { + description: "blank trigger reason rejected before store initialization", + arguments: &[ + "admin", + "change-trigger", + "00000000-0000-0000-0000-000000000001", + "pause", + "--event-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + " ", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: --reason must not be blank\n", + }, + CommandTestCase { + description: "set-cursor requires a timestamp before store initialization", + arguments: &[ + "admin", + "change-trigger", + "00000000-0000-0000-0000-000000000001", + "set-cursor", + "--event-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + "Refresh orders", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: trigger change: missing field `time`\n", + }, + CommandTestCase { + description: "blank location rejected before store initialization", + arguments: &["admin", "create-location", " "], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: location name is required\n", + }, + CommandTestCase { + description: "blank principal rejected before store initialization", + arguments: &["admin", "create-principal", " "], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: principal name is required\n", + }, + CommandTestCase { + description: "invalid credential ID rejected before store initialization", + arguments: &["admin", "revoke-token", "not-a-uuid"], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: a valid subject UUID is required\n", + }, + CommandTestCase { + description: "invalid password principal rejected before input or store initialization", + arguments: &["admin", "set-password", "not-a-uuid"], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: a valid subject UUID is required\n", + }, + CommandTestCase { + description: "invalid expiry rejected before store initialization", + arguments: &[ + "admin", + "issue-location-token", + "00000000-0000-0000-0000-000000000001", + "--expires-at", + "tomorrow", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: --expires-at must be an RFC3339 timestamp\n", + }, + CommandTestCase { + description: "expired credential rejected before store initialization", + arguments: &[ + "admin", + "issue-location-token", + "00000000-0000-0000-0000-000000000001", + "--expires-at", + "2000-01-01T00:00:00Z", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: token expiry must be in the future\n", + }, + CommandTestCase { + description: "invalid port is rejected before store initialization", + arguments: &["server", "--port", "65536"], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control service failed: invalid control-plane data: --port must be between 0 and 65535\n", + }, + ]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("isolated CLI directory"); + let binary = isolated_binary(directory.path()); + let output = Command::new(&binary) + .args(test_case.arguments) + .current_dir(directory.path()) + .env("PATH", "") + .env("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://not-opened") + .output() + .expect("standalone native command"); + assert_eq!( + output.status.success(), + test_case.expected_success, + "{}", + test_case.description + ); + assert_eq!( + String::from_utf8_lossy(&output.stderr), + test_case.expected_stderr, + "{}", + test_case.description + ); + assert!( + String::from_utf8_lossy(&output.stdout).starts_with(test_case.expected_stdout_prefix), + "{}", + test_case.description + ); + assert!( + !directory.path().join(".sqlbuild").exists(), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_cursor_free_trigger_change_when_a_cursor_is_supplied_then_it_fails_before_opening_the_store() + { + let test_cases = [ + TriggerCursorTestCase { + description: "enable rejects an extra cursor", + change: "enable", + expected_success: false, + }, + TriggerCursorTestCase { + description: "pause rejects an extra cursor", + change: "pause", + expected_success: false, + }, + TriggerCursorTestCase { + description: "reset-default rejects an extra cursor", + change: "reset-default", + expected_success: false, + }, + TriggerCursorTestCase { + description: "reset-cursor rejects an extra cursor", + change: "reset-cursor", + expected_success: false, + }, + ]; + let directory = tempfile::tempdir().expect("isolated CLI directory"); + let binary = isolated_binary(directory.path()); + for test_case in test_cases { + for cursor in ["2026-01-01T00:00:00Z", "nonsense"] { + let output = Command::new(&binary) + .args([ + "admin", + "change-trigger", + "00000000-0000-0000-0000-000000000001", + test_case.change, + "--event-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + "Refresh orders", + "--cursor", + cursor, + ]) + .current_dir(directory.path()) + .env("PATH", "") + .env("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://not-opened") + .output() + .expect("standalone native command"); + assert_eq!( + output.status.success(), + test_case.expected_success, + "{}: {cursor}", + test_case.description + ); + assert_eq!( + String::from_utf8_lossy(&output.stderr), + "Control administration failed: invalid control-plane data: --cursor is only valid for set-cursor\n", + "{}: {cursor}", + test_case.description + ); + assert!( + output.stdout.is_empty(), + "{}: {cursor}", + test_case.description + ); + assert!(!directory.path().join(".sqlbuild").exists()); + } + } +} + +#[test] +fn given_a_standalone_server_when_exporting_openapi_then_the_worker_contract_is_machine_readable_without_python_or_a_database() + { + let test_cases = [OpenApiTestCase { + description: "worker lifecycle endpoint", + expected_path: "/api/runs/{run_id}/events", + }]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("isolated CLI directory"); + let output = Command::new(isolated_binary(directory.path())) + .args(["server", "--openapi"]) + .current_dir(directory.path()) + .env("PATH", "") + .env("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://not-opened") + .output() + .expect("standalone OpenAPI"); + assert!(output.status.success(), "{}", test_case.description); + assert!(output.stderr.is_empty(), "{}", test_case.description); + let document: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("machine-readable OpenAPI"); + assert!( + document["paths"][test_case.expected_path].is_object(), + "{}", + test_case.description + ); + assert!( + !directory.path().join(".sqlbuild").exists(), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_standalone_binary_when_serving_with_or_without_a_scheduler_then_migrations_auth_and_shutdown_are_native() + { + let test_cases = [ + ServiceTestCase { + description: "server without scheduling still observes platform health", + arguments: &["server", "--port", "0", "--no-scheduler"], + expected_leaders: 0, + expected_heartbeats: 1, + }, + ServiceTestCase { + description: "server with embedded scheduling", + arguments: &["server", "--port", "0"], + expected_leaders: 1, + expected_heartbeats: 1, + }, + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(test_case.arguments); + let url = api_ready(&mut control).await; + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(5)) + .build() + .expect("bounded HTTP requests"); + let response = client + .post(format!("{url}/api/runs/{}/heartbeat", uuid::Uuid::now_v7())) + .send() + .await + .expect("protected native API"); + assert_eq!( + response.status(), + reqwest::StatusCode::UNAUTHORIZED, + "{}", + test_case.description + ); + let password_session = client + .get(format!("{url}/api/auth/session")) + .send() + .await + .expect("password routes are opt-in"); + assert_eq!(password_session.status(), reqwest::StatusCode::NOT_FOUND); + let (leaders, heartbeats) = wait_for_health(&mut control, &test_case).await; + assert_eq!( + leaders, test_case.expected_leaders, + "{}", + test_case.description + ); + assert!( + heartbeats >= test_case.expected_heartbeats, + "{}", + test_case.description + ); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } +} + +#[test] +fn given_a_standalone_scheduler_when_started_without_python_then_it_migrates_and_takes_the_durable_lease() + { + let test_cases = [ServiceTestCase { + description: "standalone scheduler takes a lease without starting a server watchdog", + arguments: &["scheduler"], + expected_leaders: 1, + expected_heartbeats: 0, + }]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(test_case.arguments); + let (leaders, heartbeats) = wait_for_health(&mut control, &test_case).await; + assert_eq!( + leaders, test_case.expected_leaders, + "{}", + test_case.description + ); + assert_eq!( + heartbeats, test_case.expected_heartbeats, + "{}", + test_case.description + ); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } +} diff --git a/crates/sqlbuild-cli/tests/native_commands/test_proxy.rs b/crates/sqlbuild-cli/tests/native_commands/test_proxy.rs new file mode 100644 index 0000000000..db54e35bb1 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/test_proxy.rs @@ -0,0 +1,197 @@ +//! A standalone server authenticates each request from explicit proxy bindings and rejects ambiguous modes. + +use reqwest::{Client, StatusCode, header}; +use serde_json::{Value, json}; + +use crate::helpers::{api_ready, start_control_in, wait_for_exit}; +use crate::test_types::{AuthenticationConfigurationTestCase, ProxyCliTestCase}; + +#[test] +fn given_an_owner_configured_proxy_when_requests_carry_a_subject_then_only_explicit_people_resolve_without_sessions_or_machine_authority() + { + let test_cases = [ProxyCliTestCase { + description: "explicit upstream subject differs from the local username", + subject: "external-orders-reader", + expected_statuses: [ + StatusCode::FORBIDDEN, + StatusCode::UNAUTHORIZED, + StatusCode::UNAUTHORIZED, + ], + }]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + for test_case in test_cases { + runtime.block_on(async { + let directory = tempfile::tempdir().expect("prepared proxy installation"); + let provisioned = std::process::Command::new(env!("CARGO_BIN_EXE_sqb")) + .args(["admin", "create-principal", "orders-reader"]) + .current_dir(directory.path()).env("PATH", "") + .env_remove("SQLBUILD_CONTROL_DATABASE_URL").output().expect("native owner provisioning"); + assert!(provisioned.status.success()); + let person: Value = serde_json::from_slice(&provisioned.stdout).expect("person JSON"); + let principal = person["principal_id"].as_str().expect("person UUID"); + let configuration = json!({"origin": "https://control.example.com", "trusted_peers": ["127.0.0.1"], "identities": {test_case.subject: principal}}); + std::fs::write(directory.path().join("proxy.json"), configuration.to_string()).expect("server-owned identity bindings"); + let mut control = start_control_in(directory, &["server", "--port", "0", "--no-scheduler", "--trusted-proxy-config", "proxy.json"]); + let url = api_ready(&mut control).await; + let client = Client::builder().timeout(std::time::Duration::from_secs(5)).build().expect("HTTP client"); + let runs = format!("{url}/api/control/locations/{}/targets/development/runs", uuid::Uuid::now_v7()); + let read = || client.get(&runs).header(header::HOST, "control.example.com"); + let statuses = [ + read().header("x-sqb-user", test_case.subject).send().await.expect("bound subject").status(), + read().header("x-sqb-user", "orders-reader").send().await.expect("local name is not a subject").status(), + read().send().await.expect("missing subject").status(), + ]; + assert_eq!(statuses, test_case.expected_statuses, "{}", test_case.description); + let session = client.get(format!("{url}/api/auth/session")) + .header(header::HOST, "control.example.com").header("x-sqb-user", test_case.subject) + .send().await.expect("no session is minted"); + assert_eq!(session.status(), StatusCode::UNAUTHORIZED); + let machine = client.get(format!("{url}/api/locations/{}/health", uuid::Uuid::now_v7())) + .header("x-sqb-user", test_case.subject).send().await.expect("machine boundary"); + assert_eq!(machine.status(), StatusCode::UNAUTHORIZED); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } +} + +#[test] +fn given_invalid_or_conflicting_identity_configuration_when_a_standalone_server_starts_then_it_fails_closed() + { + let test_cases = [ + AuthenticationConfigurationTestCase { + description: "malformed JSON", + option: "--trusted-proxy-config", + configuration: "{", + arguments: &[], + expected_error: "trusted-proxy configuration requires valid JSON", + }, + AuthenticationConfigurationTestCase { + description: "unknown configuration fields", + option: "--trusted-proxy-config", + configuration: r#"{"origin":"https://control.example.com","trusted_peers":["127.0.0.1"],"identities":{},"unknown":"synthetic-unrecognized-value"}"#, + arguments: &[], + expected_error: "trusted-proxy configuration requires valid JSON", + }, + AuthenticationConfigurationTestCase { + description: "no trusted peer boundary", + option: "--trusted-proxy-config", + configuration: r#"{"origin":"https://control.example.com","trusted_peers":[],"identities":{}}"#, + arguments: &[], + expected_error: "requires explicit trusted peer IP addresses", + }, + AuthenticationConfigurationTestCase { + description: "non-HTTPS external origin", + option: "--trusted-proxy-config", + configuration: r#"{"origin":"http://control.example.com","trusted_peers":["127.0.0.1"],"identities":{}}"#, + arguments: &[], + expected_error: "trusted-proxy authentication requires an HTTPS origin", + }, + AuthenticationConfigurationTestCase { + description: "subject cannot be represented by the header transport", + option: "--trusted-proxy-config", + configuration: r#"{"origin":"https://control.example.com","trusted_peers":["127.0.0.1"],"identities":{"external-café":"00000000-0000-0000-0000-000000000001"}}"#, + arguments: &[], + expected_error: "subjects must be nonempty, unpadded ASCII header values", + }, + AuthenticationConfigurationTestCase { + description: "ambiguous password/proxy modes", + option: "--trusted-proxy-config", + configuration: r#"{"origin":"https://control.example.com","trusted_peers":["127.0.0.1"],"identities":{}}"#, + arguments: &["--password-origin", "https://control.example.com"], + expected_error: "trusted-proxy authentication cannot be combined", + }, + AuthenticationConfigurationTestCase { + description: "OIDC malformed configuration", + option: "--oidc-config", + configuration: "{", + arguments: &[], + expected_error: "OIDC configuration requires valid JSON", + }, + AuthenticationConfigurationTestCase { + description: "OIDC unknown configuration fields", + option: "--oidc-config", + configuration: r#"{"origin":"https://control.example.com","issuer":"https://identity.example.com","client_id":"orders-control","identities":{},"unknown":"synthetic-unrecognized-value"}"#, + arguments: &[], + expected_error: "OIDC configuration requires valid JSON", + }, + AuthenticationConfigurationTestCase { + description: "OIDC rejects plaintext issuer", + option: "--oidc-config", + configuration: r#"{"origin":"https://control.example.com","issuer":"http://identity.example.com","client_id":"orders-control","identities":{}}"#, + arguments: &[], + expected_error: "OIDC endpoints require HTTPS URLs without credentials", + }, + AuthenticationConfigurationTestCase { + description: "OIDC rejects plaintext browser origin", + option: "--oidc-config", + configuration: r#"{"origin":"http://control.example.com","issuer":"https://identity.example.com","client_id":"orders-control","identities":{}}"#, + arguments: &[], + expected_error: "OIDC authentication requires an HTTPS origin", + }, + AuthenticationConfigurationTestCase { + description: "OIDC rejects missing client secret environment variable", + option: "--oidc-config", + configuration: r#"{"origin":"https://control.example.com","issuer":"https://identity.example.com","client_id":"orders-control","identities":{},"client_secret_env":"SQLBUILD_SYNTHETIC_OIDC_MISSING"}"#, + arguments: &[], + expected_error: "OIDC client-secret environment variable is unavailable", + }, + AuthenticationConfigurationTestCase { + description: "OIDC rejects missing certificate authority", + option: "--oidc-config", + configuration: r#"{"origin":"https://control.example.com","issuer":"https://identity.example.com","client_id":"orders-control","identities":{},"certificate_authority":"absent-orders-ca.pem"}"#, + arguments: &[], + expected_error: "could not read OIDC certificate authority", + }, + AuthenticationConfigurationTestCase { + description: "OIDC conflicts with password authentication", + option: "--oidc-config", + configuration: r#"{"origin":"https://control.example.com","issuer":"https://identity.example.com","client_id":"orders-control","identities":{}}"#, + arguments: &["--password-origin", "https://control.example.com"], + expected_error: "OIDC authentication cannot be combined", + }, + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + for test_case in test_cases { + runtime.block_on(async { + let directory = tempfile::tempdir().expect("invalid identity installation"); + std::fs::write( + directory.path().join("identity.json"), + test_case.configuration, + ) + .expect("configuration input"); + let mut arguments = vec![ + "server", + "--port", + "0", + "--no-scheduler", + test_case.option, + "identity.json", + ]; + arguments.extend_from_slice(test_case.arguments); + let mut control = start_control_in(directory, &arguments); + let status = wait_for_exit(&mut control).await; + assert!(!status.success(), "{}", test_case.description); + let log = std::fs::read_to_string(&control.stderr).expect("startup failure"); + assert!( + log.contains(test_case.expected_error), + "{}", + test_case.description + ); + assert!( + !log.contains(test_case.configuration), + "configuration contents must not be echoed" + ); + assert!( + !log.contains("Control API ready:"), + "failed startup must not announce readiness" + ); + }); + } +} diff --git a/crates/sqlbuild-cli/tests/native_commands/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/test_types.rs new file mode 100644 index 0000000000..b0cefc8a6c --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/test_types.rs @@ -0,0 +1,74 @@ +use std::path::PathBuf; +use std::process::Child; + +pub(crate) struct CommandTestCase { + pub(crate) description: &'static str, + pub(crate) arguments: &'static [&'static str], + pub(crate) expected_success: bool, + pub(crate) expected_stdout_prefix: &'static str, + pub(crate) expected_stderr: &'static str, +} + +pub(crate) struct TriggerCursorTestCase { + pub(crate) description: &'static str, + pub(crate) change: &'static str, + pub(crate) expected_success: bool, +} + +pub(crate) struct OpenApiTestCase { + pub(crate) description: &'static str, + pub(crate) expected_path: &'static str, +} + +pub(crate) struct ServiceTestCase { + pub(crate) description: &'static str, + pub(crate) arguments: &'static [&'static str], + pub(crate) expected_leaders: i64, + pub(crate) expected_heartbeats: i64, +} + +pub(crate) struct PasswordCliTestCase { + pub(crate) description: &'static str, + pub(crate) password: &'static str, + pub(crate) replacement: &'static str, + pub(crate) expected_password_rows: i64, + pub(crate) expected_audits: i64, +} + +pub(crate) struct PasswordOriginTestCase { + pub(crate) description: &'static str, + pub(crate) origin: &'static str, + pub(crate) expected_error: &'static str, +} + +pub(crate) struct AdministratorCliTestCase { + pub(crate) description: &'static str, + pub(crate) expected_administrators: i64, +} + +pub(crate) struct ProxyCliTestCase { + pub(crate) description: &'static str, + pub(crate) subject: &'static str, + pub(crate) expected_statuses: [reqwest::StatusCode; 3], +} + +pub(crate) struct AuthenticationConfigurationTestCase { + pub(crate) description: &'static str, + pub(crate) option: &'static str, + pub(crate) configuration: &'static str, + pub(crate) arguments: &'static [&'static str], + pub(crate) expected_error: &'static str, +} + +pub(crate) struct TestControl { + pub(crate) child: Child, + pub(crate) directory: tempfile::TempDir, + pub(crate) stderr: PathBuf, +} + +impl Drop for TestControl { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs new file mode 100644 index 0000000000..31a14b3721 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs @@ -0,0 +1,351 @@ +//! Privileged test operators stay separate from the API-only worker session. + +use std::collections::BTreeMap; +use std::process::{Command, Stdio}; +use std::time::{Duration, SystemTime}; + +use sqlbuild_orchestration::credentials::main::locations::create_location::create_location; +use sqlbuild_orchestration::credentials::main::tokens::issue_token::issue_token; +use sqlbuild_orchestration::credentials::main::tokens::revoke_token::revoke_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope, IssuedToken}; +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; +use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; +use sqlbuild_orchestration::scheduler::models::{SchedulerLease, SchedulerState}; +use sqlbuild_orchestration::scheduler::types::RunLauncher; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::open_store::open_store; +use sqlbuild_orchestration::store::models::ControlStore; +use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; +use sqlbuild_orchestration::worker::main::start_worker::start_worker; +use sqlbuild_orchestration::worker::models::WorkerSession; +use uuid::Uuid; + +use crate::admin_helpers::administer; +use crate::helpers::isolated_binary; +use crate::test_types::TestControl; +use crate::worker::test_types::{LeaseCapture, WorkerControl}; + +impl RunLauncher for LeaseCapture { + async fn poll( + &mut self, + _store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + if let Some(lease) = lease { + self.lease + .take() + .expect("first scheduler lease") + .send(lease.clone()) + .expect("lease receiver"); + self.shutdown.send(true).expect("scheduler shutdown"); + } + Ok(()) + } + + async fn shutdown( + &mut self, + _store: &ControlStore, + _lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + Ok(()) + } +} + +pub(crate) async fn start_claimed_worker( + control: &TestControl, + url: &str, +) -> (WorkerSession, WorkerControl) { + let (worker, baseline, token) = claim_worker_run(control, None).await; + change_run( + &worker.store, + RunAuthority::Scheduler(&worker.lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: worker.identity.0, + claim_id: Some(worker.identity.1), + change: RunChange::Launched { container_id: None }, + }, + ) + .await + .expect("launched worker"); + let client = connect_worker(url, &token.bearer, worker.identity).expect("worker connection"); + let session = start_worker(&client, &baseline) + .await + .expect("verified worker"); + (session, worker) +} + +pub(crate) async fn claim_worker_run( + control: &TestControl, + image_id: Option<&str>, +) -> (WorkerControl, SnapshotInput, IssuedToken) { + let path = control.directory.path().join(".sqlbuild/control.sqlite3"); + let store = open_store(path.to_str().expect("test database path")) + .await + .expect("native control store"); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let location_id = create_location(&store, "orders", &context) + .await + .expect("location"); + let baseline = SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: image_id.map(str::to_owned), + environment_hash: "orders-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::from([( + "orders".into(), + SnapshotNode { + kind: "model".into(), + definition: serde_json::json!({"materialization": "table"}), + raw_sql: Some("select 1 as id".into()), + compiled_sql: Some("select 1 as id".into()), + python_source: None, + }, + )]), + documents: BTreeMap::from([( + "jobs".into(), + serde_json::json!({"daily_orders": { + "description": "Build orders", "selection": ["orders"], "settings": {"full_refresh": true} + }}), + )]), + }; + publish_snapshot(&store, &baseline).await.expect("snapshot"); + let run_id = Uuid::now_v7(); + let location = location_id.to_string(); + let run = run_id.to_string(); + let arguments = [ + "request-run", + &location, + "daily_orders", + "--target", + "development", + "--run-id", + &run, + "--priority", + "-3", + "--reason", + "Refresh orders", + ]; + let requested = administer(control, &arguments); + assert_eq!(requested["run_id"], run); + assert_eq!(requested["applied"], true); + assert_eq!(administer(control, &arguments)["applied"], false); + let connection = rusqlite::Connection::open(&path).expect("audit connection"); + let count: i64 = connection.query_row( + "SELECT COUNT(*) FROM sqb_security_audit WHERE action='job.run_requested' AND actor='local-control-store-owner' AND granting_role='control-store-owner' AND json_extract(scope_json,'$.run_id')=?1", + [&run], |row| row.get(0)).expect("trusted manual request audit"); + assert_eq!(count, 1); + drop(connection); + let (lease_sender, lease_receiver) = tokio::sync::oneshot::channel(); + let (shutdown, signal) = tokio::sync::watch::channel(false); + tokio::time::timeout( + Duration::from_secs(5), + run_scheduler( + &store, + SchedulerState::new(Uuid::now_v7()), + signal, + LeaseCapture { + lease: Some(lease_sender), + shutdown, + }, + ), + ) + .await + .expect("scheduler acquires leadership promptly") + .expect("scheduler shutdown"); + let lease = lease_receiver.await.expect("public scheduler leadership"); + let claim = claim_next_run( + &store, + &lease, + &BTreeMap::new(), + Some((location_id, "development")), + ) + .await + .expect("claim") + .expect("queued work"); + assert_eq!(claim.request.priority, -3); + assert_eq!( + claim.request.settings.get("full_refresh"), + Some(&serde_json::json!(true)) + ); + let identity = (run_id, claim.claim_id); + let token = issue_token( + &store, + &context, + CredentialScope::Run { + run_id, + claim_id: identity.1, + }, + (SystemTime::now() + Duration::from_secs(3600)).into(), + ) + .await + .expect("run-scoped token"); + ( + WorkerControl { + store, + context, + lease, + token_id: token.identity.token_id, + identity, + }, + baseline, + token, + ) +} + +pub(crate) fn cancel_worker(worker: WorkerControl, control: &mut TestControl) { + let event_id = Uuid::now_v7().to_string(); + let run_id = worker.identity.0.to_string(); + let arguments = [ + "cancel-run", + &run_id, + "--event-id", + &event_id, + "--reason", + "Stop the orders build", + ]; + let applied = administer(control, &arguments); + assert_eq!(applied["run_id"], run_id); + assert_eq!(applied["event_id"], event_id); + assert_eq!(applied["applied"], true); + assert_eq!(administer(control, &arguments)["applied"], false); + let connection = + rusqlite::Connection::open(control.directory.path().join(".sqlbuild/control.sqlite3")) + .expect("observe durable cancellation audit"); + connection + .query_row( + "SELECT actor, granting_role, scope_json, before_json, after_json FROM sqb_security_audit WHERE event_id=?1 AND action='run.cancel_requested'", + [&event_id], + |row| { + let actor: String = row.get(0)?; + let role: String = row.get(1)?; + let scope: String = row.get(2)?; + let before: String = row.get(3)?; + let after: String = row.get(4)?; + let scope: serde_json::Value = serde_json::from_str(&scope).expect("audit scope"); + let before: serde_json::Value = serde_json::from_str(&before).expect("before state"); + let after: serde_json::Value = serde_json::from_str(&after).expect("after state"); + assert_eq!(actor, "local-control-store-owner"); + assert_eq!(role, "control-store-owner"); + assert_eq!(scope["run_id"], run_id); + assert_eq!(scope["target"], "development"); + assert!(scope["location_id"].is_string()); + assert_eq!(before["status"], "running"); + assert!(before["cancel_requested_at"].is_null()); + assert_eq!(after["status"], "running"); + assert!(after["cancel_requested_at"].is_string()); + assert_eq!(after["reason"], "Stop the orders build"); + Ok(()) + }, + ) + .expect("same-transaction operator history"); + let claims: i64 = connection + .query_row( + "SELECT COUNT(*) FROM sqb_write_claims WHERE run_id=?1", + [&run_id], + |row| row.get(0), + ) + .expect("physical claims after cancellation request"); + assert_eq!( + claims, 1, + "cancellation is not proof of physical termination" + ); +} + +pub(crate) fn revoke_worker(worker: WorkerControl, _control: &mut TestControl) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("independent operator runtime"); + assert!( + runtime + .block_on(revoke_token( + &worker.store, + &worker.context, + worker.token_id + )) + .expect("revoke worker credential") + ); +} + +pub(crate) fn disconnect_worker(_worker: WorkerControl, control: &mut TestControl) { + control.child.kill().expect("stop API process"); + control.child.wait().expect("reap API process"); +} + +pub(crate) fn start_recovery_scheduler( + control: &TestControl, + url: &str, + location_id: Uuid, +) -> TestControl { + let directory = tempfile::tempdir().expect("recovery scheduler logs"); + let configuration = directory.path().join("launchers.json"); + let launcher = serde_json::json!({"docker": [{ + "location_id": location_id, "target": "development", "server_url": url, + "max_workers": 1, "socket": directory.path().join("docker.sock"), + "location_container": "orders-location", "executable": "sqb" + }]}); + std::fs::write(&configuration, launcher.to_string()).expect("Docker scope configuration"); + let stderr = directory.path().join("scheduler.log"); + let child = Command::new(isolated_binary(directory.path())) + .args([ + "scheduler", + "--database-env", + "SQLBUILD_RECOVERY_TEST_STORE", + "--launchers", + ]) + .arg(configuration) + .current_dir(directory.path()) + .env("PATH", "") + .env( + "SQLBUILD_RECOVERY_TEST_STORE", + control.directory.path().join(".sqlbuild/control.sqlite3"), + ) + .stdout(Stdio::null()) + .stderr(std::fs::File::create(&stderr).expect("scheduler stderr")) + .spawn() + .expect("replacement scheduler"); + TestControl { + child, + directory, + stderr, + } +} + +pub(crate) async fn wait_for_worker_log(control: &mut TestControl, expected: &str) { + tokio::time::timeout(Duration::from_secs(5), async { + for _ in 0..250 { + let log = std::fs::read_to_string(&control.stderr).expect("worker supervision log"); + assert!( + control + .child + .try_wait() + .expect("scheduler status") + .is_none(), + "{log}" + ); + let Some(_) = log.find(expected) else { + tokio::time::sleep(Duration::from_millis(20)).await; + continue; + }; + return; + } + panic!("worker supervision exhausted readiness attempts"); + }) + .await + .expect("worker supervision phase completes"); +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_heartbeat.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_heartbeat.rs new file mode 100644 index 0000000000..011e0e5490 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_heartbeat.rs @@ -0,0 +1,63 @@ +//! The standalone server remains runnable while the worker's runtime is deliberately blocked. + +use std::time::{Duration, Instant}; + +use crate::helpers::{api_ready, start_control}; +use crate::worker::helpers::{ + cancel_worker, disconnect_worker, revoke_worker, start_claimed_worker, +}; +use crate::worker::test_types::WorkerHeartbeatTestCase; + +#[test] +fn given_blocking_worker_work_when_control_changes_then_heartbeats_remain_independent() { + let test_cases = [ + WorkerHeartbeatTestCase { + description: "cancellation reaches a blocked worker runtime", + interruption: cancel_worker, + expected_cancelled: true, + }, + WorkerHeartbeatTestCase { + description: "credential revocation reaches a blocked worker runtime", + interruption: revoke_worker, + expected_cancelled: true, + }, + WorkerHeartbeatTestCase { + description: "temporary API unavailability does not cancel warehouse work", + interruption: disconnect_worker, + expected_cancelled: false, + }, + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("worker runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(&["server", "--port", "0", "--no-scheduler"]); + let url = api_ready(&mut control).await; + let (session, worker) = start_claimed_worker(&control, &url).await; + let operator = std::thread::spawn(move || { + std::thread::sleep(Duration::from_secs(2)); + (test_case.interruption)(worker, &mut control); + control + }); + // No Tokio IO can run on this caller until after the assertion. A heartbeat spawned + // on this runtime, or sharing its connection pool, cannot deliver cancellation. + std::thread::sleep(Duration::from_secs(12)); + assert_eq!( + session.cancel_requested(), + test_case.expected_cancelled, + "{}", + test_case.description + ); + let stopping = Instant::now(); + drop(session); + assert!( + stopping.elapsed() < Duration::from_secs(2), + "{}", + test_case.description + ); + drop(operator.join().expect("operator completed")); + }); + } +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs new file mode 100644 index 0000000000..aeb5ae95ec --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs @@ -0,0 +1,112 @@ +//! A committed but unlaunchable claim must not poison every replacement scheduler. + +use std::time::Duration; + +use crate::helpers::{api_ready, start_control, wait_for_exit}; +use crate::worker::helpers::{claim_worker_run, start_recovery_scheduler, wait_for_worker_log}; +use crate::worker::test_types::WorkerRecoveryTestCase; + +#[test] +fn given_an_unlaunchable_committed_claim_when_recovering_then_rejection_retries_keep_ownership() { + let test_cases = [ + WorkerRecoveryTestCase { + description: "snapshot without an image pin", + image_id: None, + failure_event: "run_terminated", + expected_pending_observations: 1, + expected_status: "failed", + }, + WorkerRecoveryTestCase { + description: "snapshot with a malformed image pin", + image_id: Some("sha256:invalid"), + failure_event: "run_terminated", + expected_pending_observations: 1, + expected_status: "failed", + }, + WorkerRecoveryTestCase { + description: "missing image pin retains a rolled-back launch observation", + image_id: None, + failure_event: "platform_launch_failed", + expected_pending_observations: 0, + expected_status: "failed", + }, + WorkerRecoveryTestCase { + description: "invalid image pin retains a rolled-back launch observation", + image_id: Some("sha256:invalid"), + failure_event: "platform_launch_failed", + expected_pending_observations: 0, + expected_status: "failed", + }, + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("recovery test runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(&["server", "--port", "0", "--no-scheduler"]); + let url = api_ready(&mut control).await; + let (worker, baseline, _) = claim_worker_run(&control, test_case.image_id).await; + let connection = + rusqlite::Connection::open(control.directory.path().join(".sqlbuild/control.sqlite3")) + .expect("observe durable ownership"); + connection + .busy_timeout(Duration::from_secs(5)) + .expect("database contention timeout"); + connection.execute_batch(&format!( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00Z'; + CREATE TRIGGER reject_recovery_publication BEFORE INSERT ON recall_events + WHEN NEW.event_type='{}' BEGIN SELECT RAISE(ABORT, 'synthetic unavailable publication'); END;", test_case.failure_event + )).expect("expire the departed leader and interrupt durable publication"); + let mut replacement = start_recovery_scheduler(&control, &url, baseline.location_id); + wait_for_worker_log(&mut replacement, "Retrying worker shutdown publication").await; + let held: (i64, i64) = connection.query_row( + "SELECT (SELECT COUNT(*) FROM sqb_write_claims), (SELECT COUNT(*) FROM sqb_runs WHERE worker_instance IS NOT NULL)", + [], |row| Ok((row.get(0)?, row.get(1)?)), + ).expect("retained claim while publication is unavailable"); + assert_eq!(held, (1, 1), "{}", test_case.description); + for event_type in ["run_launch_failed", "platform_launch_failed"] { + let pending: i64 = connection.query_row("SELECT COUNT(*) FROM recall_events WHERE event_type=?1", [event_type], |row| row.get(0)).expect("pending source/platform pair"); + assert_eq!(pending, test_case.expected_pending_observations, "{}: {event_type}", test_case.description); + } + connection + .execute_batch("DROP TRIGGER reject_recovery_publication") + .expect("restore terminal publication"); + let exit = wait_for_exit(&mut replacement).await; + assert!( + !exit.success(), + "the original publication failure remains visible" + ); + let finished: (String, i64, i64, i64) = connection.query_row( + "SELECT status, (SELECT COUNT(*) FROM sqb_write_claims), + (SELECT COUNT(*) FROM sqb_runs WHERE worker_instance IS NOT NULL), + (SELECT COUNT(*) FROM recall_events WHERE event_type='run_terminated') FROM sqb_runs WHERE run_id=?1", + [worker.identity.0.to_string()], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ).expect("terminal fact and released durable claims"); + assert_eq!( + finished, + (test_case.expected_status.into(), 0, 0, 1), + "{}", + test_case.description + ); + let (producer, payload, count): (String, String, i64) = connection.query_row( + "SELECT producer, payload, COUNT(*) OVER () FROM recall_events WHERE event_type='platform_launch_failed'", + [], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ).expect("durable launch observation from the real replacement scheduler"); + assert_eq!(producer, "platform"); + assert_eq!(count, 1, "{}", test_case.description); + let source_count: i64 = connection.query_row("SELECT COUNT(*) FROM recall_events WHERE event_type='run_launch_failed'", [], |row| row.get(0)).expect("retried source fact"); + assert_eq!(source_count, 1, "{}", test_case.description); + let event: serde_json::Value = serde_json::from_str(&payload).expect("typed event payload"); + assert_eq!(event["platform"]["kind"], "launch_failed"); + assert_eq!(event["platform"]["run_id"], worker.identity.0.to_string()); + assert_eq!(event["platform"]["claim_id"], worker.identity.1.to_string()); + assert_eq!(event["platform"]["code"], "docker_configuration_rejected"); + assert!( + std::fs::read_to_string(&replacement.stderr) + .expect("shutdown recovery log") + .contains("Worker shutdown recovered") + ); + }); + } +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs new file mode 100644 index 0000000000..0a6741cff7 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs @@ -0,0 +1,30 @@ +use crate::test_types::TestControl; + +pub(crate) struct LeaseCapture { + pub(crate) lease: Option< + tokio::sync::oneshot::Sender, + >, + pub(crate) shutdown: tokio::sync::watch::Sender, +} + +pub(crate) struct WorkerHeartbeatTestCase { + pub(crate) description: &'static str, + pub(crate) interruption: fn(WorkerControl, &mut TestControl), + pub(crate) expected_cancelled: bool, +} + +pub(crate) struct WorkerRecoveryTestCase { + pub(crate) description: &'static str, + pub(crate) image_id: Option<&'static str>, + pub(crate) failure_event: &'static str, + pub(crate) expected_pending_observations: i64, + pub(crate) expected_status: &'static str, +} + +pub(crate) struct WorkerControl { + pub(crate) store: sqlbuild_orchestration::store::models::ControlStore, + pub(crate) context: sqlbuild_orchestration::credentials::models::AuditContext, + pub(crate) lease: sqlbuild_orchestration::scheduler::models::SchedulerLease, + pub(crate) token_id: uuid::Uuid, + pub(crate) identity: (uuid::Uuid, uuid::Uuid), +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs new file mode 100644 index 0000000000..5bb5be2014 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs @@ -0,0 +1,8 @@ +#[path = "helpers.rs"] +mod helpers; +#[path = "test_heartbeat.rs"] +mod test_heartbeat; +#[path = "test_recovery.rs"] +mod test_recovery; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-config/src/command_connection/constants.rs b/crates/sqlbuild-config/src/command_connection/constants.rs index f5c5246f56..7a68e32d8a 100644 --- a/crates/sqlbuild-config/src/command_connection/constants.rs +++ b/crates/sqlbuild-config/src/command_connection/constants.rs @@ -24,7 +24,7 @@ pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; /// Two quotes around at least one character. pub(crate) const MIN_QUOTED_IDENTIFIER_LENGTH: usize = 3; /// The warehouse group each CLI command selects a target's default warehouse from. -pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 31] = [ +pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 32] = [ ("audit", Some("build")), ("build", Some("build")), ("check", Some("build")), @@ -50,6 +50,7 @@ pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 31] = [ ("format", None), ("init", None), ("lineage", None), + ("location", None), ("mv", None), ("playground", None), ("rename", None), diff --git a/crates/sqlbuild-config/src/effective/constants.rs b/crates/sqlbuild-config/src/effective/constants.rs index b7a0f2b172..f89d6956c6 100644 --- a/crates/sqlbuild-config/src/effective/constants.rs +++ b/crates/sqlbuild-config/src/effective/constants.rs @@ -217,4 +217,11 @@ pub(crate) const TARGET_FIELDS: &[TargetField] = &[ decided_by: &["concurrency"], }, }, + TargetField { + group: None, + name: "catalog_connection", + rule: MergeRule::Overlay { + decided_by: &["catalog_connection"], + }, + }, ]; diff --git a/crates/sqlbuild-config/src/project/_helpers/targets.rs b/crates/sqlbuild-config/src/project/_helpers/targets.rs index 0ba2fcebd1..1d0f4579c0 100644 --- a/crates/sqlbuild-config/src/project/_helpers/targets.rs +++ b/crates/sqlbuild-config/src/project/_helpers/targets.rs @@ -415,6 +415,10 @@ fn local_target(payload: &ConfigValue, target: TargetPlace<'_>) -> Loaded>()?; fields.push(("clone", object("LocalClonePolicy", clone_fields))); fields.push(("concurrency", target_concurrency(mapping, target)?)); + fields.push(( + "catalog_connection", + optional_text_field(mapping, "catalog_connection")?, + )); Ok(fields) } diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index 7b56d857d7..c3ae524611 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -172,8 +172,9 @@ pub(crate) const MODEL_HEADER_KEYS: [&str; 73] = [ "valid_from_column", "valid_to_column", ]; -pub(crate) const TARGET_KEYS: [&str; 20] = [ +pub(crate) const TARGET_KEYS: [&str; 21] = [ "connection", + "catalog_connection", "vars", "database", "schema", diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs index 2479ee5592..e556567d29 100644 --- a/crates/sqlbuild-config/src/project/tests/test_load_config.rs +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -1,3 +1,4 @@ +use crate::project::errors::ConfigFailure; use crate::project::main::load_git_base_ref::load_git_base_ref; use crate::project::main::load_review_config::load_review_config; use crate::project::models::ConfigFile; @@ -19,6 +20,22 @@ fn given_project_files_when_loading_then_rejections_match_python_loader() { local: "target = 'dev'\n[settings]\nmax_concurrency = 2\n", expected_failure: None, }, + ProjectFailureTestCase { + description: "a blank catalog connection is rejected", + project: "name = 'orders'\nadapter = 'duckdb'\n[targets.dev]\ncatalog_connection = ' '\n", + local: "", + expected_failure: Some(ConfigFailure::plain( + "Expected 'catalog_connection' to be a non-empty string when provided", + )), + }, + ProjectFailureTestCase { + description: "a catalog connection must be a name, not credential overrides", + project: "name = 'orders'\nadapter = 'duckdb'\n", + local: "[targets.dev]\ncatalog_connection = { user = 'catalog-reader' }\n", + expected_failure: Some(ConfigFailure::plain( + "Expected 'catalog_connection' to be a non-empty string when provided", + )), + }, ProjectFailureTestCase { description: "an unknown key suggests the nearest supported key", project: "name = 'orders'\nadapter = 'duckdb'\n[scopes]\nenforce_placment = true\n", diff --git a/crates/sqlbuild-core/Cargo.toml b/crates/sqlbuild-core/Cargo.toml index e70eb050a7..de423def04 100644 --- a/crates/sqlbuild-core/Cargo.toml +++ b/crates/sqlbuild-core/Cargo.toml @@ -9,6 +9,7 @@ rust-version.workspace = true [dependencies] regex.workspace = true +serde_json.workspace = true sha2.workspace = true [lints] diff --git a/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs b/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs index 04e8d3084f..9eb5e8408f 100644 --- a/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs +++ b/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs @@ -43,6 +43,17 @@ pub fn format_debug_json(report: &DebugReport) -> String { ), ("providers".to_owned(), lines_value(&report.providers)), ("connection".to_owned(), lines_value(&report.connection)), + ( + "local_files".to_owned(), + JsonValue::Array( + report + .local_files + .iter() + .cloned() + .map(JsonValue::String) + .collect(), + ), + ), ]); let options: StdlibJsonOptions = StdlibJsonOptions { sort_keys: true, diff --git a/crates/sqlbuild-core/src/debug_report/models.rs b/crates/sqlbuild-core/src/debug_report/models.rs index c0ec5075af..144079f7d9 100644 --- a/crates/sqlbuild-core/src/debug_report/models.rs +++ b/crates/sqlbuild-core/src/debug_report/models.rs @@ -38,6 +38,8 @@ pub struct DebugReport { pub configuration: Vec, pub providers: Vec, pub connection: Vec, + /// Exact local database paths, not the shortened display values in connection lines. + pub local_files: Vec, } impl DebugReport { diff --git a/crates/sqlbuild-core/src/debug_report/tests/helpers.rs b/crates/sqlbuild-core/src/debug_report/tests/helpers.rs index 623f516cd6..240efc55d5 100644 --- a/crates/sqlbuild-core/src/debug_report/tests/helpers.rs +++ b/crates/sqlbuild-core/src/debug_report/tests/helpers.rs @@ -2,6 +2,7 @@ use crate::debug_report::models::{ConnectionValue, DebugLine, DebugReport, Debug pub(super) fn sample_report() -> DebugReport { DebugReport { + local_files: vec!["/repo/warehouse.db".into()], runtime: vec![DebugLine::plain("sqlbuild version", "0.2.1")], configuration: vec![DebugLine::checked( "project file", diff --git a/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs b/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs index 7a2122f2d7..5fb74ee026 100644 --- a/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs +++ b/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs @@ -18,7 +18,7 @@ fn given_debug_report_when_formatting_then_matches_python_outputs() { description: "renders diagnostics sections and json sections", report: sample_report(), expected_text: "\nSQLBuild Diagnostics\n\nRuntime:\n sqlbuild version: 0.2.1\n\nConfiguration:\n project file: /repo/sqlbuild_project.toml [OK found and valid]\n\nProviders:\n providers: 1 [OK discovered]\n marker_provider: providers/marker.py:MarkerProvider [OK valid settings]\n\nConnection:\n token: ****\n connection test: [ERROR authentication failed]\n query test: [SKIP connection failed]\n\n", - expected_json: r#"{"configuration": [{"label": "project file", "message": "/repo/sqlbuild_project.toml", "status": "OK", "status_message": "found and valid"}], "connection": [{"label": "token", "message": "****"}, {"label": "connection test", "message": "", "status": "ERROR", "status_message": "authentication failed"}, {"label": "query test", "message": "", "status": "SKIP", "status_message": "connection failed"}], "providers": [{"label": "providers", "message": "1", "status": "OK", "status_message": "discovered"}, {"label": "marker_provider", "message": "providers/marker.py:MarkerProvider", "status": "OK", "status_message": "valid settings"}], "runtime": [{"label": "sqlbuild version", "message": "0.2.1"}], "success": false}"#, + expected_json: r#"{"configuration": [{"label": "project file", "message": "/repo/sqlbuild_project.toml", "status": "OK", "status_message": "found and valid"}], "connection": [{"label": "token", "message": "****"}, {"label": "connection test", "message": "", "status": "ERROR", "status_message": "authentication failed"}, {"label": "query test", "message": "", "status": "SKIP", "status_message": "connection failed"}], "local_files": ["/repo/warehouse.db"], "providers": [{"label": "providers", "message": "1", "status": "OK", "status_message": "discovered"}, {"label": "marker_provider", "message": "providers/marker.py:MarkerProvider", "status": "OK", "status_message": "valid settings"}], "runtime": [{"label": "sqlbuild version", "message": "0.2.1"}], "success": false}"#, expected_color_fragments: &[ "\x1b[34m\x1b[1mSQLBuild Diagnostics\x1b[0m", "\x1b[1mRuntime:\x1b[0m", diff --git a/crates/sqlbuild-core/src/fingerprint_identity/constants.rs b/crates/sqlbuild-core/src/fingerprint_identity/constants.rs new file mode 100644 index 0000000000..7b752a338a --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/constants.rs @@ -0,0 +1,6 @@ +//! Non-definition metadata excluded by the planner's version identity. + +pub const AUDIT_GATE_METADATA_KEY: &str = "audit_gate"; +pub const MIGRATION_FINGERPRINT_METADATA_KEY: &str = "migration_fingerprint"; +pub const DECLARED_COLUMNS_METADATA_KEY: &str = "declared_columns_hash"; +pub const LOCAL_FUNCTION_HASHES_METADATA_KEY: &str = "local_function_hashes"; diff --git a/crates/sqlbuild-core/src/fingerprint_identity/main/canonical_text.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/canonical_text.rs new file mode 100644 index 0000000000..4434934b12 --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/main/canonical_text.rs @@ -0,0 +1,33 @@ +//! Python-compatible canonical JSON for planner fingerprint metadata. + +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; +use serde_json::{Map, Value}; + +/// `json.dumps(object, sort_keys=True, separators=(",", ":"), default=str)`. +pub fn canonical_text(object: Map) -> String { + let dialect: JsonDialect = JsonDialect::Stdlib(StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None).with_separators(",", ":") + }); + dumps(&json_value(Value::Object(object)), &dialect).unwrap_or_default() +} + +fn json_value(value: Value) -> JsonValue { + match value { + Value::Null => JsonValue::Null, + Value::Bool(flag) => JsonValue::Bool(flag), + Value::Number(number) => match JsonInteger::parse(&number.to_string()) { + Some(integer) if !number.is_f64() => JsonValue::Integer(integer), + _ => JsonValue::Float(number.as_f64().unwrap_or_default()), + }, + Value::String(text) => JsonValue::String(text), + Value::Array(items) => JsonValue::Array(items.into_iter().map(json_value).collect()), + Value::Object(object) => JsonValue::Object( + object + .into_iter() + .map(|(key, value)| (key, json_value(value))) + .collect(), + ), + } +} diff --git a/crates/sqlbuild-core/src/fingerprint_identity/main/decoded_object.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/decoded_object.rs new file mode 100644 index 0000000000..ac1d41eb1b --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/main/decoded_object.rs @@ -0,0 +1,12 @@ +//! Decode fingerprint metadata as Python's JSON object reader does. + +use serde_json::{Map, Value}; + +pub fn decoded_object( + metadata_json: &str, +) -> Result>, serde_json::Error> { + Ok(match serde_json::from_str::(metadata_json)? { + Value::Object(object) => Some(object), + _ => None, + }) +} diff --git a/crates/sqlbuild-core/src/fingerprint_identity/main/mod.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/mod.rs new file mode 100644 index 0000000000..914d2d4cf4 --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/main/mod.rs @@ -0,0 +1,7 @@ +pub mod canonical_text; +pub mod decoded_object; +pub mod model_local_identity_hash; +pub mod normalize_version_identity_metadata; +pub mod python_equal; +pub mod recorded_model_local_hash; +pub mod stable_identity_hash; diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/model_local_identity_hash.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/model_local_identity_hash.rs similarity index 79% rename from crates/sqlbuild-analysis/src/planner_identity/main/model_local_identity_hash.rs rename to crates/sqlbuild-core/src/fingerprint_identity/main/model_local_identity_hash.rs index 11148c52dc..585c7ada95 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/main/model_local_identity_hash.rs +++ b/crates/sqlbuild-core/src/fingerprint_identity/main/model_local_identity_hash.rs @@ -1,6 +1,6 @@ //! A model's local identity hash from its query fingerprint and non-query metadata. -use crate::planner_identity::main::stable_identity_hash::stable_identity_hash; +use crate::fingerprint_identity::main::stable_identity_hash::stable_identity_hash; /// The hash of the query fingerprint and metadata JSON, one per line. pub fn model_local_identity_hash(query_fingerprint: &str, metadata_json: &str) -> String { diff --git a/crates/sqlbuild-core/src/fingerprint_identity/main/normalize_version_identity_metadata.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/normalize_version_identity_metadata.rs new file mode 100644 index 0000000000..f84f44064a --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/main/normalize_version_identity_metadata.rs @@ -0,0 +1,24 @@ +//! Normalize recorded metadata without changing the persisted fingerprint. + +use serde_json::{Map, Value}; + +use crate::fingerprint_identity::constants::{ + AUDIT_GATE_METADATA_KEY, DECLARED_COLUMNS_METADATA_KEY, LOCAL_FUNCTION_HASHES_METADATA_KEY, + MIGRATION_FINGERPRINT_METADATA_KEY, +}; + +pub fn normalize_version_identity_metadata( + object: &mut Map, + without_functions: bool, +) { + for key in [ + AUDIT_GATE_METADATA_KEY, + MIGRATION_FINGERPRINT_METADATA_KEY, + DECLARED_COLUMNS_METADATA_KEY, + ] { + object.shift_remove(key); + } + if without_functions { + object.shift_remove(LOCAL_FUNCTION_HASHES_METADATA_KEY); + } +} diff --git a/crates/sqlbuild-core/src/fingerprint_identity/main/python_equal.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/python_equal.rs new file mode 100644 index 0000000000..6feffc311d --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/main/python_equal.rs @@ -0,0 +1,40 @@ +//! Python equality of decoded fingerprint values, including numeric booleans. + +use serde_json::Value; + +pub fn python_equal(left: &Value, right: &Value) -> bool { + match (left, right) { + (Value::Array(left), Value::Array(right)) => { + left.len() == right.len() + && left + .iter() + .zip(right) + .all(|(left, right)| python_equal(left, right)) + } + (Value::Object(left), Value::Object(right)) => { + left.len() == right.len() + && left.iter().all(|(key, value)| { + right + .get(key) + .is_some_and(|other| python_equal(value, other)) + }) + } + (Value::Number(left_number), Value::Number(right_number)) + if !left_number.is_f64() && !right_number.is_f64() => + { + left_number == right_number + } + (Value::Number(_) | Value::Bool(_), Value::Number(_) | Value::Bool(_)) => { + numeric(left) == numeric(right) + } + _ => left == right, + } +} + +fn numeric(value: &Value) -> Option { + match value { + Value::Bool(flag) => Some(f64::from(u8::from(*flag))), + Value::Number(number) => number.as_f64(), + _ => None, + } +} diff --git a/crates/sqlbuild-core/src/fingerprint_identity/main/recorded_model_local_hash.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/recorded_model_local_hash.rs new file mode 100644 index 0000000000..bb91cdbb3a --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/main/recorded_model_local_hash.rs @@ -0,0 +1,27 @@ +//! Recover model-local identity only from a complete recorded planner payload. + +use crate::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; +use crate::fingerprint_identity::main::normalize_version_identity_metadata::normalize_version_identity_metadata; +use crate::fingerprint_identity::main::{ + canonical_text::canonical_text, decoded_object::decoded_object, +}; + +pub fn recorded_model_local_hash(query_hash: &str, metadata_json: &str) -> Option { + let mut object = match decoded_object(metadata_json) { + Ok(Some(object)) => object, + Ok(None) | Err(_) => return None, + }; + if query_hash.is_empty() + || !object.get("model_name")?.is_string() + || !object.get("config")?.is_object() + || !object.get("execution_signature")?.is_object() + || !object.get("local_function_hashes")?.is_object() + { + return None; + } + normalize_version_identity_metadata(&mut object, false); + Some(model_local_identity_hash( + query_hash, + &canonical_text(object), + )) +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/stable_identity_hash.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/stable_identity_hash.rs similarity index 100% rename from crates/sqlbuild-analysis/src/planner_identity/main/stable_identity_hash.rs rename to crates/sqlbuild-core/src/fingerprint_identity/main/stable_identity_hash.rs diff --git a/crates/sqlbuild-core/src/fingerprint_identity/mod.rs b/crates/sqlbuild-core/src/fingerprint_identity/mod.rs new file mode 100644 index 0000000000..919648bfd1 --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/mod.rs @@ -0,0 +1,4 @@ +//! Pure planner fingerprint identity, shared by planning and stored-evidence reads. + +pub mod constants; +pub mod main; diff --git a/crates/sqlbuild-core/src/identifier_text/main/mod.rs b/crates/sqlbuild-core/src/identifier_text/main/mod.rs index a29dc793b1..b9d09a0ef5 100644 --- a/crates/sqlbuild-core/src/identifier_text/main/mod.rs +++ b/crates/sqlbuild-core/src/identifier_text/main/mod.rs @@ -1 +1,2 @@ pub mod is_lowercase_identifier; +pub mod quote_identifier; diff --git a/crates/sqlbuild-core/src/identifier_text/main/quote_identifier.rs b/crates/sqlbuild-core/src/identifier_text/main/quote_identifier.rs new file mode 100644 index 0000000000..efd258cbc0 --- /dev/null +++ b/crates/sqlbuild-core/src/identifier_text/main/quote_identifier.rs @@ -0,0 +1,6 @@ +//! Standard SQL double-quoted identifiers, including embedded quote escaping. + +#[must_use] +pub fn quote_identifier(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 29ad9c99e3..0b6f781eea 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -28,6 +28,7 @@ pub mod diagnostic_routing; pub mod diagnostic_text; pub mod dlt_destination; pub mod event_export; +pub mod fingerprint_identity; pub mod format_command; pub mod freshness_text; pub mod git_revisions; diff --git a/crates/sqlbuild-core/src/progress/constants.rs b/crates/sqlbuild-core/src/progress/constants.rs index 36ee9d8e7e..bcdf9d0799 100644 --- a/crates/sqlbuild-core/src/progress/constants.rs +++ b/crates/sqlbuild-core/src/progress/constants.rs @@ -110,6 +110,7 @@ pub(crate) const SQL_PREVIEW_LENGTH: usize = 160; pub(crate) const EXECUTION_TITLE: &str = "Execution"; pub(crate) const LINEAGE_COMMAND: &str = "lineage"; pub(crate) const QUERY_COMMAND: &str = "query"; +pub(crate) const LOCATION_COMMAND: &str = "location"; pub(crate) const DBT_COMMAND: &str = "dbt"; pub(crate) const DBT_JSON_OPTION: &str = "--json"; pub(crate) const JSON_FORMAT: &str = "json"; diff --git a/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs b/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs index 7a6c3bc531..1337d46807 100644 --- a/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs +++ b/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs @@ -1,7 +1,7 @@ //! Whether a command's stdout carries machine output, so human progress must use stderr. use crate::progress::constants::{ - DBT_COMMAND, DBT_JSON_OPTION, JSON_FORMAT, LINEAGE_COMMAND, QUERY_COMMAND, + DBT_COMMAND, DBT_JSON_OPTION, JSON_FORMAT, LINEAGE_COMMAND, LOCATION_COMMAND, QUERY_COMMAND, }; use crate::progress::models::OutputModeFacts; @@ -11,6 +11,7 @@ pub fn machine_output_mode(facts: &OutputModeFacts) -> bool { let format = |value: &Option| value.as_deref().unwrap_or_default().to_owned(); match facts.command.as_deref() { _ if facts.json => true, + Some(LOCATION_COMMAND) => true, Some(DBT_COMMAND) => facts.dbt_args.iter().any(|arg| arg == DBT_JSON_OPTION), Some(LINEAGE_COMMAND) => format(&facts.lineage_format) == JSON_FORMAT, Some(QUERY_COMMAND) => format(&facts.query_format) == JSON_FORMAT, diff --git a/crates/sqlbuild-discovery/src/constants.rs b/crates/sqlbuild-discovery/src/constants.rs index 096580ebab..86a7a4dcd5 100644 --- a/crates/sqlbuild-discovery/src/constants.rs +++ b/crates/sqlbuild-discovery/src/constants.rs @@ -2,6 +2,8 @@ /// Root of SQL model files. pub const MODELS_DIRECTORY: &str = "models"; +/// Root of code-location orchestration declarations, never imported by the scheduler. +pub const ORCHESTRATION_DIRECTORY: &str = "orchestration"; /// Suffix of SQL files. pub const SQL_FILE_SUFFIX: &str = ".sql"; /// The folder below a resource tree that holds grouped scoped declarations. diff --git a/crates/sqlbuild-discovery/src/python_paths/constants.rs b/crates/sqlbuild-discovery/src/python_paths/constants.rs index eacdb349f9..4483a9f780 100644 --- a/crates/sqlbuild-discovery/src/python_paths/constants.rs +++ b/crates/sqlbuild-discovery/src/python_paths/constants.rs @@ -1,7 +1,9 @@ //! The documented roots project Python may live under, and the directories the walk skips. +use crate::constants::ORCHESTRATION_DIRECTORY; + /// Top-level directories whose Python files are always supported. -pub(crate) const DIRECT_PYTHON_ROOTS: [&str; 10] = [ +pub(crate) const DIRECT_PYTHON_ROOTS: [&str; 11] = [ "adapters", "materializations", "providers", @@ -12,6 +14,7 @@ pub(crate) const DIRECT_PYTHON_ROOTS: [&str; 10] = [ "enums", "constants", "event_exporters", + ORCHESTRATION_DIRECTORY, ]; /// `(root, language)` directory pairs whose Python files are supported. pub(crate) const LANGUAGE_PYTHON_ROOTS: [(&str, &str); 2] = diff --git a/crates/sqlbuild-discovery/src/python_paths/tests/test_unsupported_python_paths.rs b/crates/sqlbuild-discovery/src/python_paths/tests/test_unsupported_python_paths.rs index 64ecd45aa6..9a4843ec44 100644 --- a/crates/sqlbuild-discovery/src/python_paths/tests/test_unsupported_python_paths.rs +++ b/crates/sqlbuild-discovery/src/python_paths/tests/test_unsupported_python_paths.rs @@ -11,6 +11,8 @@ fn given_project_python_when_listing_unsupported_paths_then_only_undocumented_ro files: &[ "adapter.py", "python/orders.py", + "orchestration/jobs.py", + "orchestration/nested/calendar.py", "functions/python/f.py", "macros/m.py", "models/_macros/m.py", diff --git a/crates/sqlbuild-host/src/entries/_helpers/user_code.rs b/crates/sqlbuild-host/src/entries/_helpers/user_code.rs index 7deafb99c8..51d567c4e1 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/user_code.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/user_code.rs @@ -64,6 +64,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Function, edge: HostEdge::UserCode, }, + HostEntry { + name: "load_orchestration_module", + kind: HostKind::Function, + edge: HostEdge::UserCode, + }, HostEntry { name: "module_functions", kind: HostKind::Function, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 5c8482411a..057a1d4203 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -140,6 +140,7 @@ pub enum UserCodeHost { LoadProviderModule, LoadPythonNodeModule, LoadSinkModule, + LoadOrchestrationModule, ModuleFunctions, ModuleProviderClasses, ParameterAnnotations, diff --git a/crates/sqlbuild-orchestration/Cargo.toml b/crates/sqlbuild-orchestration/Cargo.toml new file mode 100644 index 0000000000..94170f1655 --- /dev/null +++ b/crates/sqlbuild-orchestration/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "sqlbuild-orchestration" +version.workspace = true +description = "SQLBuild control-plane storage, snapshots, scheduling and run coordination." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +argon2.workspace = true +axum.workspace = true +chrono.workspace = true +chrono-tz.workspace = true +croner.workspace = true +flate2.workspace = true +futures-util.workspace = true +getrandom.workspace = true +openidconnect.workspace = true +reqwest.workspace = true +rusqlite.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlbuild-core.workspace = true +sqlbuild-recall.workspace = true +sqlbuild-state.workspace = true +sqlx-core.workspace = true +sqlx-postgres.workspace = true +subtle.workspace = true +thiserror.workspace = true +tokio.workspace = true +utoipa.workspace = true +uuid.workspace = true + +[dev-dependencies] +axum-server.workspace = true +proptest.workspace = true +rcgen.workspace = true +rusqlite = { workspace = true, features = ["functions"] } +tempfile.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/bearer.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/bearer.rs new file mode 100644 index 0000000000..014cc2ec71 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/bearer.rs @@ -0,0 +1,145 @@ +//! Authenticate machine bearers before buffering; scope checks stay in each owning endpoint. + +use std::net::SocketAddr; + +use axum::extract::{Request, State}; +use axum::http::{HeaderMap, HeaderValue, StatusCode, header}; +use axum::middleware::Next; +use axum::response::Response; +use uuid::Uuid; + +use crate::api::errors::ApiError; +use crate::api::models::{DevelopmentScope, ReadResource}; +use crate::credentials::main::tokens::authenticate_token::authenticate_token; +use crate::credentials::models::{CredentialScope, TokenIdentity}; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn authenticate( + State(store): State, + mut request: Request, + next: Next, +) -> Result { + let development = request.extensions().get::().cloned(); + let identity = match bearer(request.headers()) { + Ok(token) => authenticate_token(&store, token).await?, + Err(_) if development.is_some() => None, + Err(error) => return Err(error), + }; + let identity = identity + .or_else(|| { + development.map(|scope| TokenIdentity { + token_id: Uuid::nil(), + scope: CredentialScope::Location { + location_id: scope.location_id, + }, + expires_at: chrono::DateTime::::MAX_UTC, + }) + }) + .ok_or_else(unauthorized)?; + request.extensions_mut().insert(identity); + let mut response = next.run(request).await; + response + .headers_mut() + .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + Ok(response) +} + +pub(crate) fn bearer(headers: &HeaderMap) -> Result<&str, ApiError> { + headers + .get(header::AUTHORIZATION) + .ok_or_else(unauthorized)? + .to_str() + .map_err(|_| unauthorized())? + .strip_prefix("Bearer ") + .ok_or_else(unauthorized) +} + +fn unauthorized() -> ApiError { + ApiError { + status: StatusCode::UNAUTHORIZED, + message: "valid bearer credential required".into(), + decision_id: None, + } +} + +/// Reject browser cross-origin and DNS-rebinding requests on the disabled-auth listener. +pub(crate) async fn local_request( + State(address): State, + request: Request, + next: Next, +) -> Result { + let authorities = [address.to_string(), format!("localhost:{}", address.port())]; + let valid_host = request.headers().get(header::HOST).is_some_and(|value| { + value + .to_str() + .is_ok_and(|host| authorities.iter().any(|allowed| host == allowed)) + }); + let valid_origin = request.headers().get(header::ORIGIN).is_none_or(|value| { + value.to_str().is_ok_and(|origin| { + authorities + .iter() + .any(|host| origin == format!("http://{host}")) + }) + }); + if !valid_host || !valid_origin { + return Err(ApiError { + status: StatusCode::FORBIDDEN, + message: "development API requires a local authority and origin".into(), + decision_id: None, + }); + } + Ok(next.run(request).await) +} + +pub(crate) fn require_location( + identity: &TokenIdentity, + location_id: Uuid, +) -> Result<(), ApiError> { + if identity.scope != (CredentialScope::Location { location_id }) { + return Err(ApiError { + status: StatusCode::FORBIDDEN, + message: "credential does not authorize this location".into(), + decision_id: None, + }); + } + Ok(()) +} + +/// A guessed resource ID never widens the already authenticated location and target. +pub(crate) async fn require_resource( + store: &ControlStore, + (location_id, target): (Uuid, &str), + (resource, id): (ReadResource, Uuid), +) -> Result<(), ApiError> { + let (statement, missing) = match resource { + ReadResource::Run => ( + "SELECT run_id FROM sqb_runs WHERE run_id=$1 AND location_id=$2 AND target=$3", + "run not found", + ), + ReadResource::Trigger => ( + "SELECT trigger_id FROM sqb_triggers WHERE trigger_id=$1 AND location_id=$2 AND target=$3", + "trigger not found", + ), + }; + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + statement, + vec![ + Parameter::Id(id), + Parameter::Id(location_id), + Parameter::Text(target.into()), + ], + ) + .await?; + commit(transaction).await?; + if rows.is_empty() { + return Err(ApiError { + status: StatusCode::NOT_FOUND, + message: missing.into(), + decision_id: None, + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.html b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.html new file mode 100644 index 0000000000..3b6202d970 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.html @@ -0,0 +1,47 @@ + + + + + +SQLBuild CLI sign-in + + + + +
+

Sign in to SQLBuild CLI

+

Only continue if you started sqb login yourself. Compare this code with your terminal:

+— +

Checking your sign-in…

+ + + + +
+ + diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.js b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.js new file mode 100644 index 0000000000..4234410fb7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.js @@ -0,0 +1,74 @@ +"use strict"; + +window.addEventListener("hashchange", () => location.reload()); +const element = (id) => document.getElementById(id); +const storageKey = "sqb.cli.handoff"; +const code = location.hash.slice(1) || sessionStorage.getItem(storageKey) || ""; +history.replaceState(null, "", location.pathname); +const valid = /^[0-9a-f-]{36}\.[0-9a-f]{64}$/i.test(code); +if (valid) sessionStorage.setItem(storageKey, code); +const message = (text) => { element("message").textContent = text; }; +const csrf = () => document.cookie.split(";").map((part) => part.trim()).find((part) => part.startsWith("__Host-sqb-csrf="))?.split("=")[1] || ""; + +async function request(path, body) { + return fetch(path, { method: "POST", credentials: "same-origin", headers: { "Content-Type": "application/json", "x-sqb-csrf": csrf() }, body: JSON.stringify(body) }); +} + +async function current() { + if (document.body.dataset.method === "proxy") { + element("account").textContent = "Signed in through your organization"; + element("confirmation").hidden = false; + message("Check that the verification code matches your terminal before authorizing."); + return; + } + const response = await fetch("/api/auth/session", { credentials: "same-origin" }); + if (!response.ok) { + message("Sign in before authorizing your CLI."); + element(document.body.dataset.method === "password" ? "password" : "sso").hidden = false; + return; + } + const identity = await response.json(); + for (const id of ["password", "sso"]) element(id).hidden = true; + element("account").textContent = `Signed in as person ${identity.principal_id}`; + element("confirmation").hidden = false; + message("Check that the verification code matches your terminal before authorizing."); +} + +element("password").addEventListener("submit", async (event) => { + event.preventDefault(); + const button = element("password").querySelector("button"); + button.disabled = true; + try { + const response = await request("/api/auth/password/login", { username: element("username").value, password: element("password-value").value }); + if (!response.ok) { message("Sign-in was rejected. Check your credentials, or wait before trying again."); return; } + await current(); + } catch { message("Could not reach SQLBuild. Try signing in again."); } + finally { element("password-value").value = ""; button.disabled = false; } +}); + +element("sso").addEventListener("click", () => { + document.cookie = "__Host-sqb-cli-return=1; Secure; SameSite=Lax; Path=/; Max-Age=300"; + location.assign("/api/auth/oidc/login"); +}); + +async function decide(approve) { + element("approve").disabled = true; + element("cancel").disabled = true; + try { + const response = await request("/api/auth/cli/approve", { browser_code: code, approve }); + if (!response.ok) { message("This sign-in is no longer available. Start sqb login again."); return; } + sessionStorage.removeItem(storageKey); + element("confirmation").hidden = true; + message(approve ? "CLI authorized. Return to your terminal to finish signing in." : "CLI sign-in cancelled. No CLI credential was issued."); + } catch { message("Could not confirm the decision. Start sqb login again."); } +} +element("approve").addEventListener("click", () => decide(true)); +element("cancel").addEventListener("click", () => decide(false)); + +if (!valid) { + sessionStorage.removeItem(storageKey); + message("Start sqb login in your terminal to open a valid sign-in request."); +} else { + element("code").textContent = code.split(".")[1].slice(0, 8).toUpperCase(); + current().catch(() => message("Could not reach SQLBuild. Start sqb login again.")); +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.rs new file mode 100644 index 0000000000..58f1a88533 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/cli_login.rs @@ -0,0 +1,122 @@ +//! Only the exchange secret can receive a CLI credential; a browser person approves it. + +use axum::extract::{DefaultBodyLimit, State}; +use axum::http::{HeaderMap, StatusCode, header}; +use axum::response::{Html, IntoResponse, Response}; +use axum::routing::{get, post}; +use axum::{Extension, Json, Router}; + +use crate::api::_helpers::authentication::password_transport::{deny, session_header}; +use crate::api::constants::PASSWORD_BODY_BYTES; +use crate::api::errors::ApiError; +use crate::api::models::{ + AuthenticationDenial, BrowserLoginMethod, CliLoginApprovalRequest, CliLoginClaimRequest, + CliLoginSessionResponse, CliLoginStartRequest, +}; +use crate::credentials::main::audit::reserve_authentication_attempt::reserve_authentication_attempt; +use crate::credentials::main::cli_login::{ + approve_login::approve_login, claim_login::claim_login, start_login::start_login, +}; +use crate::credentials::models::{ + AuditContext, AuthenticatedPerson, AuthenticationAttempt, ClaimedLogin, StartedLogin, +}; +use crate::store::models::ControlStore; + +pub(crate) fn router(method: BrowserLoginMethod) -> Router { + Router::new() + .route("/auth/cli", get(page)) + .route("/auth/cli.js", get(script)) + .route("/api/auth/cli/start", post(start)) + .route("/api/auth/cli/claim", post(claim)) + .layer(Extension(method)) + .layer(DefaultBodyLimit::max(PASSWORD_BODY_BYTES)) +} + +async fn page(Extension(method): Extension) -> Response { + let method = match method { + BrowserLoginMethod::Password => "password", + BrowserLoginMethod::Oidc => "oidc", + BrowserLoginMethod::Proxy => "proxy", + }; + let mut response = + Html(include_str!("cli_login.html").replace("__LOGIN_METHOD__", method)).into_response(); + response.headers_mut().insert(header::CONTENT_SECURITY_POLICY, + header::HeaderValue::from_static("default-src 'none'; script-src 'self'; style-src 'unsafe-inline'; connect-src 'self'; frame-ancestors 'none'; form-action 'self'; base-uri 'none'")); + response.headers_mut().insert( + header::REFERRER_POLICY, + header::HeaderValue::from_static("no-referrer"), + ); + response +} + +async fn script() -> impl IntoResponse { + ( + [(header::CONTENT_TYPE, "text/javascript; charset=utf-8")], + include_str!("cli_login.js"), + ) +} + +#[utoipa::path(post, path = "/api/auth/cli/start", operation_id = "startCliLogin", + request_body = CliLoginStartRequest, responses((status = 200, body = StartedLogin), + (status = 429, body = AuthenticationDenial)))] +pub(crate) async fn start( + State(store): State, + Extension(context): Extension, + Json(request): Json, +) -> Result { + let (_, allowed) = + reserve_authentication_attempt(&store, &context, AuthenticationAttempt::IdentityProvider) + .await?; + if !allowed { + let mut response = deny(&context)?; + *response.status_mut() = StatusCode::TOO_MANY_REQUESTS; + return Ok(response); + } + Ok(Json(start_login(&store, &context, &request.client_hash).await?).into_response()) +} + +#[utoipa::path(post, path = "/api/auth/cli/approve", operation_id = "approveCliLogin", + request_body = CliLoginApprovalRequest, security(("session" = []), ("proxy" = [])), + responses((status = 204), (status = 401, body = AuthenticationDenial)))] +pub(crate) async fn approve( + State(store): State, + (Extension(context), Extension(person)): ( + Extension, + Extension, + ), + headers: HeaderMap, + Json(request): Json, +) -> Result { + if !session_header(&headers) + && approve_login( + &store, + &context, + person, + (&request.browser_code, request.approve), + ) + .await? + { + Ok(StatusCode::NO_CONTENT.into_response()) + } else { + deny(&context) + } +} + +#[utoipa::path(post, path = "/api/auth/cli/claim", operation_id = "claimCliLogin", + request_body = CliLoginClaimRequest, responses((status = 200, body = CliLoginSessionResponse), + (status = 202), (status = 401, body = AuthenticationDenial)))] +pub(crate) async fn claim( + State(store): State, + Extension(context): Extension, + Json(request): Json, +) -> Result { + match claim_login(&store, &context, (request.flow_id, &request.client_secret)).await? { + ClaimedLogin::Pending => Ok(StatusCode::ACCEPTED.into_response()), + ClaimedLogin::Unavailable => deny(&context), + ClaimedLogin::Authenticated(session) => Ok(Json(CliLoginSessionResponse { + identity: session.identity, + bearer: session.bearer, + }) + .into_response()), + } +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs new file mode 100644 index 0000000000..0c94251bf5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/configure.rs @@ -0,0 +1,35 @@ +//! Build the configured sign-in routes before migrations or HTTP startup. + +use std::net::SocketAddr; + +use axum::{Extension, Router}; + +use crate::api::_helpers::authentication::{oidc, password_sessions, trusted_proxy}; +use crate::api::models::{ApiAuthentication, DevelopmentScope, HumanLogin}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +pub(crate) async fn configure( + store: &ControlStore, + ApiAuthentication { + login, + default_role, + }: ApiAuthentication, + address: SocketAddr, +) -> Result<(Option, Router), ControlError> { + let (development, routes) = match login { + HumanLogin::None => (None, Router::new()), + HumanLogin::Development(scope) => { + if !address.ip().is_loopback() { + return Err(ControlError::Invalid( + "disabled authentication requires a loopback listener".into(), + )); + } + (Some(scope), Router::new()) + } + HumanLogin::Password(origin) => (None, password_sessions::router(store, origin)?), + HumanLogin::TrustedProxy(settings) => (None, trusted_proxy::router(store, settings)?), + HumanLogin::Oidc(settings) => (None, oidc::router(store, settings).await?), + }; + Ok((development, routes.layer(Extension(default_role)))) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/mod.rs new file mode 100644 index 0000000000..95a52ea8b5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod bearer; +pub(crate) mod cli_login; +pub(crate) mod configure; +pub(crate) mod oidc; +pub(crate) mod oidc_provider; +pub(crate) mod password_sessions; +pub(crate) mod password_transport; +pub(crate) mod trusted_proxy; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc.rs new file mode 100644 index 0000000000..cf63f9b8f4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc.rs @@ -0,0 +1,159 @@ +//! Browser-bound authorization-code/PKCE transport authenticates explicitly bound people only. + +use std::sync::Arc; + +use axum::extract::{Query, State}; +use axum::http::{HeaderMap, StatusCode, header}; +use axum::response::{IntoResponse, Redirect, Response}; +use axum::routing::get; +use axum::{Extension, Router, middleware}; +use openidconnect::core::CoreAuthenticationFlow; +use openidconnect::{CsrfToken, Nonce, PkceCodeChallenge}; + +use crate::api::_helpers::authentication::password_transport::{ + browser_cookie, deny, external_session, guard, set_cookie, +}; +use crate::api::_helpers::authentication::{oidc_provider, password_sessions}; +use crate::api::constants::OIDC_FLOW_COOKIE; +use crate::api::errors::ApiError; +use crate::api::models::{ + AuthenticationDenial, BrowserLoginMethod, BrowserOrigin, ExternalIdentityResponse, + OidcAuthentication, OidcCallback, OidcRuntime, +}; +use crate::credentials::constants::OIDC_FLOW_SECONDS; +use crate::credentials::main::audit::reserve_authentication_attempt::reserve_authentication_attempt; +use crate::credentials::main::sessions::authenticate_bound_person::authenticate_bound_person; +use crate::credentials::main::sessions::{ + claim_oidc_flow::claim_oidc_flow, create_oidc_flow::create_oidc_flow, +}; +use crate::credentials::models::{AuditContext, AuthenticationAttempt, OidcFlowStart}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +pub(crate) async fn router( + store: &ControlStore, + settings: OidcAuthentication, +) -> Result, ControlError> { + let runtime = oidc_provider::configure(settings)?; + oidc_provider::client(&runtime).await?; + Ok(Router::new() + .route("/api/auth/oidc/login", get(login)) + .route("/api/auth/oidc/callback", get(callback)) + .merge(password_sessions::protected(store)) + .merge(super::cli_login::router(BrowserLoginMethod::Oidc)) + .layer(middleware::from_fn(guard)) + .layer(Extension(BrowserOrigin(runtime.settings.origin.clone()))) + .layer(Extension(Arc::new(runtime)))) +} + +#[utoipa::path(get, path = "/api/auth/oidc/login", operation_id = "loginOidc", + description = "Begins a browser-bound code flow with S256 PKCE; no account provisioning or grants.", + responses((status = 303), (status = 401, body = AuthenticationDenial), (status = 429, body = AuthenticationDenial)))] +pub(crate) async fn login( + State(store): State, + Extension(runtime): Extension>, + Extension(context): Extension, +) -> Result { + let (_, allowed) = + reserve_authentication_attempt(&store, &context, AuthenticationAttempt::IdentityProvider) + .await?; + if !allowed { + let mut response = deny(&context)?; + *response.status_mut() = StatusCode::TOO_MANY_REQUESTS; + return Ok(response); + } + let client = match oidc_provider::client(&runtime).await { + Ok(client) => client, + Err(_) => return deny(&context), + }; + let (challenge, pkce_verifier) = PkceCodeChallenge::new_random_sha256(); + let (url, state, nonce) = client + .authorize_url( + CoreAuthenticationFlow::AuthorizationCode, + CsrfToken::new_random, + Nonce::new_random, + ) + .set_pkce_challenge(challenge) + .url(); + let flow = OidcFlowStart { + state, + nonce, + pkce_verifier, + browser: CsrfToken::new_random(), + }; + create_oidc_flow(&store, &context, &runtime.configuration_id, &flow).await?; + let mut response = Redirect::to(url.as_str()).into_response(); + set_cookie( + &mut response, + (OIDC_FLOW_COOKIE, flow.browser.secret()), + OIDC_FLOW_SECONDS, + (true, "Lax"), + )?; + Ok(response) +} + +#[utoipa::path(get, path = "/api/auth/oidc/callback", operation_id = "oidcIdentity", + description = "Validates one browser-bound code and issues a bounded browser session for an explicitly bound person; no provisioning, personal API tokens or implicit grants.", + responses((status = 200, body = ExternalIdentityResponse), (status = 401, body = AuthenticationDenial)))] +pub(crate) async fn callback( + State(store): State, + Extension(runtime): Extension>, + Extension(context): Extension, + (headers, Query(request)): (HeaderMap, Query), +) -> Result { + let Some(state) = request.state else { + return deny(&context); + }; + let Some(flow) = claim_oidc_flow( + &store, + &context, + &runtime.configuration_id, + ( + state.secret(), + browser_cookie(&headers, OIDC_FLOW_COOKIE).unwrap_or_default(), + ), + ) + .await? + else { + return deny(&context); + }; + let subject = match request.code { + Some(code) => oidc_provider::verify_code(&runtime, code, flow).await, + None => Err(ControlError::Invalid( + "OIDC authorization was not completed".into(), + )), + }; + let mut response = match subject { + Ok(subject) => match runtime.settings.identities.get(&subject).copied() { + Some(principal) => { + if let Some(session) = + authenticate_bound_person(&store, &context, principal).await? + { + external_session(session)? + } else { + deny(&context)? + } + } + None => deny(&context)?, + }, + Err(_) => deny(&context)?, + }; + if browser_cookie(&headers, "__Host-sqb-cli-return") == Some("1") { + set_cookie( + &mut response, + ("__Host-sqb-cli-return", ""), + 0, + (false, "Lax"), + )?; + if response.status().is_success() { + *response.status_mut() = StatusCode::SEE_OTHER; + response.headers_mut().insert( + header::LOCATION, + header::HeaderValue::from_static("/auth/cli"), + ); + *response.body_mut() = axum::body::Body::empty(); + } + } + set_cookie(&mut response, (OIDC_FLOW_COOKIE, ""), 0, (true, "Lax"))?; + Ok(response) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc_provider.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc_provider.rs new file mode 100644 index 0000000000..4c71c7f501 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/oidc_provider.rs @@ -0,0 +1,183 @@ +//! OIDC uses verified HTTPS, bounded bodies and the library's signed issuer/audience/nonce checks. + +use std::time::Duration; + +use openidconnect::core::{CoreClient, CoreProviderMetadata}; +use openidconnect::{ + AccessTokenHash, AuthorizationCode, ClientId, ClientSecret, IssuerUrl, OAuth2TokenResponse, + RedirectUrl, TokenResponse, +}; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::api::constants::{HTTPS_SCHEME, OIDC_RESPONSE_BYTES}; +use crate::api::main::parse_origin::parse_origin; +use crate::api::models::{OidcAuthentication, OidcRuntime}; +use crate::api::types::OidcClient; +use crate::credentials::models::StoredOidcFlow; +use crate::store::errors::ControlError; + +pub(crate) fn configure(mut settings: OidcAuthentication) -> Result { + settings.origin = parse_origin(&settings.origin, "OIDC")?; + let issuer = https_url(&settings.issuer)?; + if issuer.query().is_some() || issuer.fragment().is_some() { + return Err(ControlError::Invalid( + "OIDC issuer cannot contain a query or fragment".into(), + )); + } + if settings.client_id.trim().is_empty() || settings.identities.keys().any(String::is_empty) { + return Err(ControlError::Invalid( + "OIDC requires a nonempty client ID and nonempty bound subjects".into(), + )); + } + let client_secret = settings + .client_secret_env + .as_ref() + .map(|name| { + let value = std::env::var(name).map_err(|_| { + ControlError::Invalid( + "OIDC client-secret environment variable is unavailable".into(), + ) + })?; + if value.is_empty() { + return Err(ControlError::Invalid( + "OIDC client secret must not be empty".into(), + )); + } + Ok(ClientSecret::new(value)) + }) + .transpose()?; + let mut http = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(10)); + if let Some(path) = &settings.certificate_authority { + let bytes = std::fs::read(path).map_err(|_| { + ControlError::Invalid("could not read OIDC certificate authority".into()) + })?; + let certificates = reqwest::Certificate::from_pem_bundle(&bytes) + .map_err(|_| ControlError::Invalid("invalid OIDC PEM certificate authority".into()))?; + if certificates.is_empty() { + return Err(ControlError::Invalid( + "OIDC certificate authority contains no certificates".into(), + )); + } + for certificate in certificates { + http = http.add_root_certificate(certificate); + } + } + let configuration_id = + sha256_hex(&serde_json::to_vec(&settings).map_err(|_| invalid_provider())?); + Ok(OidcRuntime { + settings, + client_secret, + http: http.build().map_err(|_| invalid_provider())?, + configuration_id, + }) +} + +pub(crate) async fn client(runtime: &OidcRuntime) -> Result { + let http = |request| request_provider(runtime.http.clone(), request); + let metadata = CoreProviderMetadata::discover_async( + IssuerUrl::new(runtime.settings.issuer.clone()).map_err(|_| invalid_provider())?, + &http, + ) + .await + .map_err(|_| invalid_provider())?; + https_url(metadata.authorization_endpoint().as_str())?; + https_url( + metadata + .token_endpoint() + .ok_or_else(invalid_provider)? + .as_str(), + )?; + Ok(CoreClient::from_provider_metadata( + metadata, + ClientId::new(runtime.settings.client_id.clone()), + runtime.client_secret.clone(), + ) + .set_redirect_uri( + RedirectUrl::new(format!( + "{}/api/auth/oidc/callback", + runtime.settings.origin + )) + .map_err(|_| invalid_provider())?, + )) +} + +pub(crate) async fn verify_code( + runtime: &OidcRuntime, + code: AuthorizationCode, + flow: StoredOidcFlow, +) -> Result { + let client = client(runtime).await?; + let http = |request| request_provider(runtime.http.clone(), request); + let tokens = client + .exchange_code(code) + .map_err(|_| invalid_provider())? + .set_pkce_verifier(flow.pkce_verifier) + .request_async(&http) + .await + .map_err(|_| invalid_provider())?; + let token = tokens.id_token().ok_or_else(invalid_provider)?; + let verifier = client.id_token_verifier(); + let claims = token + .claims(&verifier, &flow.nonce) + .map_err(|_| invalid_provider())?; + if let Some(expected) = claims.access_token_hash() { + let actual = AccessTokenHash::from_token( + tokens.access_token(), + token.signing_alg().map_err(|_| invalid_provider())?, + token + .signing_key(&verifier) + .map_err(|_| invalid_provider())?, + ) + .map_err(|_| invalid_provider())?; + if actual != *expected { + return Err(invalid_provider()); + } + } + Ok(claims.subject().as_str().to_owned()) +} + +async fn request_provider( + client: reqwest::Client, + request: openidconnect::HttpRequest, +) -> Result { + if request.uri().scheme_str() != Some("https") { + return Err(invalid_provider()); + } + let mut response = client + .execute(reqwest::Request::try_from(request).map_err(|_| invalid_provider())?) + .await + .map_err(|_| invalid_provider())?; + let status = response.status(); + let headers = response.headers().clone(); + let mut bytes: Vec = Vec::new(); + while let Some(chunk) = response.chunk().await.map_err(|_| invalid_provider())? { + if chunk.len() > OIDC_RESPONSE_BYTES.saturating_sub(bytes.len()) { + return Err(invalid_provider()); + } + bytes.extend_from_slice(&chunk); + } + let mut result = openidconnect::HttpResponse::new(bytes); + *result.status_mut() = status; + *result.headers_mut() = headers; + Ok(result) +} + +fn https_url(value: &str) -> Result { + let url = reqwest::Url::parse(value).map_err(|_| invalid_provider())?; + if url.scheme() != HTTPS_SCHEME + || url.host().is_none() + || !url.username().is_empty() + || url.password().is_some() + { + return Err(ControlError::Invalid( + "OIDC endpoints require HTTPS URLs without credentials".into(), + )); + } + Ok(url) +} + +fn invalid_provider() -> ControlError { + ControlError::Invalid("OIDC identity-provider validation failed".into()) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs new file mode 100644 index 0000000000..20fd69bcbd --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -0,0 +1,253 @@ +//! Password endpoints return an HttpOnly cookie, never a browser-readable session bearer. + +use axum::extract::{DefaultBodyLimit, State}; +use axum::http::{HeaderValue, StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use axum::{Extension, Json, Router, middleware}; + +use crate::api::_helpers::authentication::password_transport::{ + authenticate, deny, guard, session_cookies, set_cookie, +}; +use crate::api::constants::{PASSWORD_BODY_BYTES, PASSWORD_COOKIE, PASSWORD_CSRF_COOKIE}; +use crate::api::errors::ApiError; +use crate::api::main::parse_origin::parse_origin; +use crate::api::models::{ + AuthenticationDenial, BrowserLoginMethod, BrowserOrigin, PasswordChangeRequest, + PasswordLoginRequest, PasswordSessionResponse, +}; +use crate::credentials::constants::LOGIN_WINDOW_SECONDS; +use crate::credentials::main::{ + audit::reserve_authentication_attempt::reserve_authentication_attempt, + sessions::login_password::login_password, sessions::revoke_session::revoke_session, + sessions::set_password::set_password, sessions::verify_password::verify_password, +}; +use crate::credentials::models::{ + AuditContext, AuthenticationAttempt, PasswordAuthority, PasswordLogin, SessionIdentity, +}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +/// Session-authenticated person routes, plus routes acting on the caller's own session. +pub(crate) fn protected(store: &ControlStore) -> Router { + person_routes() + .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)) + .merge(session_routes(store)) +} + +/// Routes that act on the caller's own session. +pub(crate) fn session_routes(store: &ControlStore) -> Router { + Router::new() + .route("/api/auth/session", get(current)) + .route("/api/auth/logout", post(logout)) + .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)) + .layer(DefaultBodyLimit::max(PASSWORD_BODY_BYTES)) +} + +/// Person operations; the caller layers the authentication that identifies the person. +pub(crate) fn person_routes() -> Router { + Router::new() + .route("/api/auth/cli/approve", post(super::cli_login::approve)) + .route( + "/api/control/audit", + get(crate::api::_helpers::resources::audit::read), + ) + .route( + "/api/admin/people/{principal_id}", + post(crate::api::_helpers::resources::administrators::change), + ) + .route( + "/api/admin/roles", + post(crate::api::_helpers::resources::administrators::assign), + ) + .route( + "/api/control/runs", + post(crate::api::_helpers::resources::person_operations::request_job), + ) + .route( + "/api/control/runs/cancel", + post(crate::api::_helpers::resources::person_operations::cancel_run), + ) + .route( + "/api/control/runs/rerun", + post(crate::api::_helpers::resources::person_operations::rerun), + ) + .route( + "/api/control/triggers", + post(crate::api::_helpers::resources::person_operations::change_trigger), + ) + .route( + "/api/control/locations/{location_id}/targets/{target}/connection-tests", + post(crate::api::_helpers::resources::person_location_work::request), + ) + .route( + "/api/control/locations/{location_id}/targets/{target}/connection-tests/{request_id}", + get(crate::api::_helpers::resources::person_location_work::result), + ) + .route( + "/api/control/locations/{location_id}/targets/{target}/runs", + get(crate::api::_helpers::resources::person_operations::recent_runs), + ) + .route( + "/api/control/locations/{location_id}/targets/{target}/catalog", + get(crate::api::_helpers::resources::person_operations::catalog), + ) + .route( + "/api/control/locations/{location_id}/targets/{target}/statistics", + get(crate::api::_helpers::resources::person_operations::statistics), + ) + .route( + "/api/control/locations/{location_id}/targets/{target}/fingerprint", + get(crate::api::_helpers::resources::person_operations::fingerprint), + ) + .route( + "/api/control/locations/{location_id}/targets/{target}/node-status", + get(crate::api::_helpers::resources::person_operations::node_status), + ) + .route( + "/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/sql", + get(crate::api::_helpers::resources::person_node_sql::read), + ) + .layer(DefaultBodyLimit::max(PASSWORD_BODY_BYTES)) +} + +pub(crate) fn router( + store: &ControlStore, + origin: String, +) -> Result, ControlError> { + let origin = BrowserOrigin(parse_origin(&origin, "password")?); + let protected = protected(store).merge( + Router::new() + .route("/api/auth/password", post(change_password)) + .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)), + ); + Ok(protected + .route("/api/auth/password/login", post(login)) + .merge(super::cli_login::router(BrowserLoginMethod::Password)) + .layer(DefaultBodyLimit::max(PASSWORD_BODY_BYTES)) + .layer(middleware::from_fn(guard)) + .layer(Extension(origin))) +} + +#[utoipa::path(post, path = "/api/auth/password/login", operation_id = "loginPassword", + request_body = PasswordLoginRequest, + responses((status = 200, body = PasswordSessionResponse), (status = 401, body = AuthenticationDenial), + (status = 429, body = AuthenticationDenial)))] +pub(crate) async fn login( + State(store): State, + Extension(context): Extension, + Json(request): Json, +) -> Result { + match login_password(&store, &context, (&request.username, &request.password)).await? { + PasswordLogin::Authenticated(session) => { + let response = Json(PasswordSessionResponse { + identity: session.identity.clone(), + csrf_token: session.csrf.clone(), + }) + .into_response(); + session_cookies(response, &session) + } + PasswordLogin::Denied { throttled } => { + let mut response = ( + if throttled { + StatusCode::TOO_MANY_REQUESTS + } else { + StatusCode::UNAUTHORIZED + }, + Json(AuthenticationDenial { + error: "authentication rejected".into(), + request_id: context.request_id, + }), + ) + .into_response(); + if throttled { + response.headers_mut().insert( + header::RETRY_AFTER, + HeaderValue::from_str(&LOGIN_WINDOW_SECONDS.to_string()).map_err(|_| { + ControlError::Invalid("invalid login retry interval".into()) + })?, + ); + } + Ok(response) + } + } +} + +#[utoipa::path(get, path = "/api/auth/session", operation_id = "currentSession", + responses((status = 200, body = SessionIdentity), (status = 401, body = AuthenticationDenial)), + security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn current( + Extension(identity): Extension, +) -> Json { + Json(identity) +} + +#[utoipa::path(post, path = "/api/auth/logout", operation_id = "logoutSession", + params(("x-sqb-csrf" = String, Header, description = "Session CSRF token")), + responses((status = 204), (status = 401, body = AuthenticationDenial)), + security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn logout( + State(store): State, + Extension(context): Extension, + Extension(identity): Extension, +) -> Result { + let (transaction, _) = revoke_session( + begin(&store, TransactionMode::Write).await?, + &context, + (identity.principal_id, Some(identity.session_id)), + ) + .await?; + commit(transaction).await?; + clear_cookies() +} + +#[utoipa::path(post, path = "/api/auth/password", operation_id = "changePassword", + params(("x-sqb-csrf" = String, Header, description = "Session CSRF token")), + request_body = PasswordChangeRequest, + responses((status = 204), (status = 401, body = AuthenticationDenial)), + security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn change_password( + State(store): State, + Extension(context): Extension, + Extension(identity): Extension, + Json(request): Json, +) -> Result { + let (_, allowed) = reserve_authentication_attempt( + &store, + &context, + AuthenticationAttempt::Reauthenticate(identity.principal_id), + ) + .await?; + if !allowed { + return deny(&context); + } + let Some(proof) = + verify_password(&store, identity.principal_id, &request.current_password).await? + else { + return deny(&context); + }; + set_password( + &store, + &context, + PasswordAuthority::Reauthenticated { + proof, + session_id: identity.session_id, + }, + &request.new_password, + ) + .await?; + clear_cookies() +} + +fn clear_cookies() -> Result { + let mut response = StatusCode::NO_CONTENT.into_response(); + set_cookie(&mut response, (PASSWORD_COOKIE, ""), 0, (true, "Strict"))?; + set_cookie( + &mut response, + (PASSWORD_CSRF_COOKIE, ""), + 0, + (false, "Strict"), + )?; + Ok(response) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs new file mode 100644 index 0000000000..10b2c098d9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_transport.rs @@ -0,0 +1,210 @@ +//! Browser credentials use host-only cookies; peer identity never comes from forwarded headers. + +use std::net::SocketAddr; + +use axum::extract::{ConnectInfo, Request, State}; +use axum::http::{HeaderMap, HeaderValue, StatusCode, header}; +use axum::middleware::Next; +use axum::response::{IntoResponse, Response}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::constants::{PASSWORD_COOKIE, PASSWORD_CSRF_COOKIE, PASSWORD_CSRF_HEADER}; +use crate::api::errors::ApiError; +use crate::api::models::{AuthenticationDenial, BrowserOrigin, ExternalIdentityResponse}; +use crate::credentials::constants::PASSWORD_SESSION_SECONDS; +use crate::credentials::main::sessions::authenticate_session::authenticate_session; +use crate::credentials::models::{AuditContext, AuthenticatedPerson, IssuedSession, SessionAccess}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +pub(crate) async fn guard( + Extension(origin): Extension, + mut request: Request, + next: Next, +) -> Result { + let peer = request + .extensions() + .get::>() + .ok_or_else(|| ApiError { + status: StatusCode::INTERNAL_SERVER_ERROR, + message: "trusted request peer is unavailable".into(), + decision_id: None, + })? + .0; + let context = AuditContext { + actor: "anonymous".into(), + granting_role: "authentication".into(), + request_id: Uuid::now_v7(), + ip_address: Some(peer.ip().to_canonical()), + }; + let valid_host = single_header(request.headers(), header::HOST.as_str()) + == origin.0.strip_prefix("https://"); + let valid_origin = match single_header(request.headers(), header::ORIGIN.as_str()) { + Some(value) => value == origin.0, + None => request.method().is_safe() && !request.headers().contains_key(header::ORIGIN), + }; + let mut response = if !valid_host || !valid_origin { + deny(&context)? + } else { + request.extensions_mut().insert(context); + next.run(request).await + }; + response + .headers_mut() + .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + response.headers_mut().insert( + header::X_CONTENT_TYPE_OPTIONS, + HeaderValue::from_static("nosniff"), + ); + Ok(response) +} + +pub(crate) async fn authenticate( + State(store): State, + mut request: Request, + next: Next, +) -> Result { + let context = request + .extensions() + .get::() + .cloned() + .ok_or_else(|| ApiError { + status: StatusCode::INTERNAL_SERVER_ERROR, + message: "trusted authentication context is unavailable".into(), + decision_id: None, + })?; + let access = + if request.method().is_safe() || request.headers().contains_key(header::AUTHORIZATION) { + SessionAccess::Read + } else { + SessionAccess::Write { + csrf: single_header(request.headers(), PASSWORD_CSRF_HEADER).unwrap_or_default(), + } + }; + let identity = + authenticate_session(&store, session_credentials(request.headers()).0, access).await?; + let Some(identity) = identity else { + return deny(&context); + }; + let mut context = context; + context.actor = format!("principal:{}", identity.principal_id); + context.granting_role = "browser-session".into(); + request.extensions_mut().insert(context); + request.extensions_mut().insert(AuthenticatedPerson( + identity.principal_id, + Some(identity.session_id), + )); + request.extensions_mut().insert(identity); + Ok(next.run(request).await) +} + +pub(crate) fn single_header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> { + let mut values = headers.get_all(name).iter(); + let Ok(value) = values.next()?.to_str() else { + return None; + }; + values.next().is_none().then_some(value) +} + +/// A CLI session header; other `Authorization` schemes, such as a proxy's own, are not sessions. +pub(crate) fn session_header(headers: &HeaderMap) -> bool { + single_header(headers, header::AUTHORIZATION.as_str()) + .is_some_and(|value| value.starts_with("Session ")) +} + +pub(crate) fn session_credentials(headers: &HeaderMap) -> (&str, &str) { + let cookie = browser_cookie(headers, PASSWORD_COOKIE); + let authorization = if headers.contains_key(header::AUTHORIZATION) { + if headers.contains_key(header::COOKIE) { + return ("", ""); + } + single_header(headers, header::AUTHORIZATION.as_str()) + .and_then(|value| value.strip_prefix("Session ")) + } else { + cookie + }; + ( + authorization.unwrap_or_default(), + single_header(headers, PASSWORD_CSRF_HEADER).unwrap_or_default(), + ) +} + +pub(crate) fn browser_cookie<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> { + let mut session = None; + for header in headers.get_all(header::COOKIE) { + let Ok(value) = header.to_str() else { + return None; + }; + for cookie in value.split(';') { + let Some((cookie_name, value)) = cookie.trim().split_once('=') else { + continue; + }; + if cookie_name == name { + if session.is_some() { + return None; + } + session = Some(value); + } + } + } + session +} + +pub(crate) fn external_session(session: IssuedSession) -> Result { + let response = Json(ExternalIdentityResponse { + principal_id: session.identity.principal_id, + session_id: session.identity.session_id, + expires_at: session.identity.expires_at, + csrf_token: session.csrf.clone(), + }) + .into_response(); + session_cookies(response, &session) +} + +pub(crate) fn session_cookies( + mut response: Response, + session: &IssuedSession, +) -> Result { + set_cookie( + &mut response, + (PASSWORD_COOKIE, &session.bearer), + PASSWORD_SESSION_SECONDS, + (true, "Strict"), + )?; + set_cookie( + &mut response, + (PASSWORD_CSRF_COOKIE, &session.csrf), + PASSWORD_SESSION_SECONDS, + (false, "Strict"), + )?; + Ok(response) +} + +pub(crate) fn set_cookie( + response: &mut Response, + (name, value): (&str, &str), + max_age: i64, + (http_only, same_site): (bool, &str), +) -> Result<(), ApiError> { + let attributes = if http_only { "; HttpOnly" } else { "" }; + let mut cookie = HeaderValue::from_str(&format!( + "{name}={value}; Path=/; Secure; SameSite={same_site}; Max-Age={max_age}{attributes}" + )) + .map_err(|_| ControlError::Invalid("could not encode a session cookie".into()))?; + cookie.set_sensitive(true); + response.headers_mut().append(header::SET_COOKIE, cookie); + Ok(()) +} + +/// Anonymous denials are not durably audited; the response names the request. +pub(crate) fn deny(context: &AuditContext) -> Result { + Ok(( + StatusCode::UNAUTHORIZED, + Json(AuthenticationDenial { + error: "authentication rejected".into(), + request_id: context.request_id, + }), + ) + .into_response()) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/trusted_proxy.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/trusted_proxy.rs new file mode 100644 index 0000000000..65a24227bd --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/trusted_proxy.rs @@ -0,0 +1,105 @@ +//! Each request's proxy subject resolves through explicit bindings and the real socket peer. + +use std::sync::Arc; + +use axum::extract::{Request, State}; +use axum::http::HeaderValue; +use axum::middleware::Next; +use axum::response::Response; +use axum::{Extension, Router, middleware}; + +use crate::api::_helpers::authentication::password_sessions; +use crate::api::_helpers::authentication::password_transport::{self, deny, guard, single_header}; +use crate::api::constants::PROXY_USER_HEADER; +use crate::api::errors::ApiError; +use crate::api::main::parse_origin::parse_origin; +use crate::api::models::{BrowserLoginMethod, BrowserOrigin, TrustedProxyAuthentication}; +use crate::credentials::models::{AuditContext, AuthenticatedPerson}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) fn router( + store: &ControlStore, + mut settings: TrustedProxyAuthentication, +) -> Result, ControlError> { + let origin = BrowserOrigin(parse_origin(&settings.origin, "trusted-proxy")?); + if settings.trusted_peers.is_empty() { + return Err(ControlError::Invalid( + "trusted-proxy authentication requires explicit trusted peer IP addresses".into(), + )); + } + if settings.identities.keys().any(|subject| { + subject.trim().is_empty() + || subject.trim() != subject + || match HeaderValue::from_str(subject) { + Ok(value) => value.to_str().is_err(), + Err(_) => true, + } + }) { + return Err(ControlError::Invalid( + "trusted-proxy subjects must be nonempty, unpadded ASCII header values".into(), + )); + } + settings.trusted_peers = settings + .trusted_peers + .into_iter() + .map(|peer| peer.to_canonical()) + .collect(); + Ok(Router::new() + .merge( + password_sessions::person_routes() + .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)), + ) + .merge(password_sessions::session_routes(store)) + .merge(super::cli_login::router(BrowserLoginMethod::Proxy)) + .layer(middleware::from_fn(guard)) + .layer(Extension(origin)) + .layer(Extension(Arc::new(settings)))) +} + +/// A CLI session credential authenticates as a session; otherwise the bound proxy subject does. +async fn authenticate( + State(store): State, + Extension(settings): Extension>, + mut request: Request, + next: Next, +) -> Result { + if password_transport::session_header(request.headers()) { + return password_transport::authenticate(State(store), request, next).await; + } + let Some(mut context) = request.extensions().get::().cloned() else { + return Err( + ControlError::Invalid("trusted authentication context is unavailable".into()).into(), + ); + }; + if !context + .ip_address + .is_some_and(|peer| settings.trusted_peers.contains(&peer)) + { + return deny(&context); + } + let Some(principal_id) = single_header(request.headers(), PROXY_USER_HEADER) + .and_then(|subject| settings.identities.get(subject)) + .copied() + else { + return deny(&context); + }; + let (transaction, people) = query( + begin(&store, TransactionMode::Read).await?, + "SELECT principal_id FROM sqb_principals WHERE principal_id=$1 AND disabled_at IS NULL", + vec![Parameter::Id(principal_id)], + ) + .await?; + commit(transaction).await?; + if people.is_empty() { + return deny(&context); + } + context.actor = format!("principal:{principal_id}"); + context.granting_role = "trusted-proxy".into(); + request.extensions_mut().insert(context); + request + .extensions_mut() + .insert(AuthenticatedPerson(principal_id, None)); + Ok(next.run(request).await) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs new file mode 100644 index 0000000000..528c844b6e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod authentication; +pub(crate) mod resources; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs new file mode 100644 index 0000000000..89fbada6a9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs @@ -0,0 +1,44 @@ +//! Reauthenticate membership changes inside the serialized mutation, not only in middleware. + +use axum::extract::{Path, State}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::errors::ApiError; +use crate::api::models::MutationResponse; +use crate::authorization::main::assign_builtin_role::assign_builtin_role; +use crate::authorization::main::change_administrator::change_administrator; +use crate::credentials::models::AuthenticatedPerson; +use crate::credentials::models::{AdministratorChange, AuditContext, RoleAssignmentRequest}; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/admin/people/{principal_id}", operation_id = "changeAdministrator", + params(("principal_id" = Uuid, Path), ("x-sqb-csrf" = String, Header, description = "Session CSRF token")), + request_body = AdministratorChange, + responses((status = 200, body = MutationResponse), (status = 401, body = crate::api::models::AuthenticationDenial), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn change( + (State(store), Extension(context)): (State, Extension), + Path(principal_id): Path, + Extension(AuthenticatedPerson(principal, _)): Extension, + Json(change): Json, +) -> Result, ApiError> { + Ok(Json(MutationResponse { + applied: change_administrator(&store, (&context, principal), (principal_id, change)) + .await?, + })) +} + +#[utoipa::path(post, path = "/api/admin/roles", operation_id = "assignBuiltinRole", + params(("x-sqb-csrf" = String, Header)), request_body = RoleAssignmentRequest, + responses((status = 200, body = MutationResponse), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn assign( + (State(store), Extension(context)): (State, Extension), + Extension(AuthenticatedPerson(principal, _)): Extension, + Json(request): Json, +) -> Result, ApiError> { + Ok(Json(MutationResponse { + applied: assign_builtin_role(&store, (&context, principal), &request).await?, + })) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/audit.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/audit.rs new file mode 100644 index 0000000000..2ec377e95e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/audit.rs @@ -0,0 +1,31 @@ +//! Person sessions read bounded security facts without machine-token or ambient-viewer access. + +use axum::extract::{Query, State}; +use axum::{Extension, Json}; + +use crate::api::errors::ApiError; +use crate::api::models::ReadAuditQuery; +use crate::authorization::main::read_security_audit::read_security_audit; +use crate::credentials::models::AuthenticatedPerson; +use crate::credentials::models::{AuditContext, SecurityAuditEvent}; +use crate::store::models::ControlStore; + +#[utoipa::path(get, path = "/api/control/audit", operation_id = "readSecurityAudit", + params(("after" = Option, Query, description = "Exclusive storage-order cursor"), ("limit" = Option, Query)), + responses((status = 200, body = Vec), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Session rejected"), (status = 403, body = crate::api::models::ApiErrorBody)), + security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn read( + (State(store), Extension(context)): (State, Extension), + Query(options): Query, + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result>, ApiError> { + Ok(Json( + read_security_audit( + &store, + (&context, principal), + (options.after.unwrap_or(0), options.limit.unwrap_or(100)), + ) + .await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs new file mode 100644 index 0000000000..6771cd3ca5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs @@ -0,0 +1,166 @@ +//! Scoped invalidations, not a replayable event log or a second source of application data. + +use std::time::Duration; + +use axum::Extension; +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, HeaderValue}; +use axum::response::Sse; +use axum::response::sse::Event; +use futures_util::{Stream, stream}; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use tokio::sync::broadcast; +use uuid::Uuid; + +use crate::api::_helpers::authentication::bearer::{bearer, require_location}; +use crate::api::errors::ApiError; +use crate::api::models::{ChangeFeed, ChangeNotice, ChangeStream}; +use crate::credentials::main::tokens::authenticate_token::authenticate_token; +use crate::credentials::models::{CredentialScope, TokenIdentity}; +use crate::store::errors::ControlError; +use crate::store::main::listen_changes::listen_changes; +use crate::store::models::{ChangeMessage, ChangeTopic, ControlStore}; + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/changes", + params(("location_id" = Uuid, Path), ("target" = String, Path)), + responses((status = 200, body = ChangeNotice, content_type = "text/event-stream", + description = "changed events: all, runs, run:, ticks, snapshot:, node_status:. Refetch REST on all/reconnect; no durable cursor."), + (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized")), security(("bearer" = [])))] +pub(crate) async fn subscribe( + State(store): State, + (Extension(identity), Extension(feed)): (Extension, Extension), + Path((location_id, target)): Path<(Uuid, String)>, + headers: HeaderMap, +) -> Result>>, ApiError> { + require_location(&identity, location_id)?; + let authorization = if identity.token_id.is_nil() { + None + } else { + let mut authorization = HeaderValue::from_str(bearer(&headers)?) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + authorization.set_sensitive(true); + Some(authorization) + }; + let state = ChangeStream { + store, + authorization, + location_id, + target_hash: sha256_hex(target.as_bytes()), + target, + receiver: feed.sender.subscribe(), + stopped: feed.stopped, + keepalive: tokio::time::interval(Duration::from_secs(15)), + initial: true, + }; + Ok(Sse::new(stream::try_unfold(state, next_event))) +} + +/// Revalidate every emission, keeping unexpected storage failures as stream errors. +async fn next_event( + mut state: ChangeStream, +) -> Result, ControlError> { + loop { + if *state.stopped.borrow() { + return Ok(None); + } + let message = if std::mem::take(&mut state.initial) { + Some(ChangeMessage::Resync) + } else { + tokio::select! { + _ = state.stopped.changed() => return Ok(None), + _ = state.keepalive.tick() => None, + message = state.receiver.recv() => match message { + Ok(message) => Some(message), + Err(broadcast::error::RecvError::Lagged(_)) => Some(ChangeMessage::Resync), + Err(broadcast::error::RecvError::Closed) => return Ok(None), + }, + } + }; + let keepalive = message.is_none(); + let topic = match message { + Some(ChangeMessage::Changed { + location_id, + target_hash, + topic, + }) => { + if location_id != state.location_id || target_hash != state.target_hash { + continue; + } + Some(match topic { + ChangeTopic::Runs => "runs".into(), + ChangeTopic::Run(id) => format!("run:{id}"), + ChangeTopic::Ticks => "ticks".into(), + ChangeTopic::Snapshot => format!("snapshot:{}", state.location_id), + ChangeTopic::NodeStatus => format!("node_status:{}", state.target), + }) + } + Some(ChangeMessage::Resync) => Some("all".into()), + Some(ChangeMessage::CredentialsChanged) | None => None, + }; + if let Some(authorization) = &state.authorization { + let token = authorization + .to_str() + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let Some(identity) = authenticate_token(&state.store, token).await? else { + return Ok(None); + }; + if identity.scope + != (CredentialScope::Location { + location_id: state.location_id, + }) + { + return Ok(None); + } + } + if let Some(topic) = topic { + let event = Event::default() + .event("changed") + .json_data(ChangeNotice { topic }) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + return Ok(Some((event, state))); + } + if keepalive { + return Ok(Some((Event::default().comment("keep-alive"), state))); + } + } +} + +pub(crate) async fn relay( + store: ControlStore, + mut listener: Option, + sender: broadcast::Sender, +) -> Result<(), ControlError> { + let ControlStore::Postgres(pool) = store else { + return std::future::pending().await; + }; + loop { + if listener.is_none() { + tokio::time::sleep(Duration::from_secs(1)).await; + listener = match listen_changes(&pool).await { + Ok(listener) => { + let _ = sender.send(ChangeMessage::Resync); + Some(listener) + } + Err(ControlError::Database(error)) if ControlError::transient_database(&error) => { + continue; + } + Err(error) => return Err(error), + }; + } + let Some(active) = listener.as_mut() else { + continue; + }; + let message = match active.try_recv().await { + Ok(Some(notification)) => serde_json::from_str(notification.payload()) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + Ok(None) => ChangeMessage::Resync, + Err(error) if ControlError::transient_database(&error) => { + listener = None; + continue; + } + Err(error) => return Err(error.into()), + }; + let _ = sender.send(message); + } +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/client.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/client.rs new file mode 100644 index 0000000000..e2a86e2291 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/client.rs @@ -0,0 +1,38 @@ +//! Outbound clients own their pools; credentials and response bodies stay out of diagnostics. + +use std::time::Duration; + +use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue}; + +use crate::api::errors::ClientError; +use crate::api::models::ControlClient; + +pub(crate) fn http_client(authorization: HeaderValue) -> Result { + let mut headers = HeaderMap::new(); + headers.insert(AUTHORIZATION, authorization); + Ok(reqwest::Client::builder() + .default_headers(headers) + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(30)) + .connect_timeout(Duration::from_secs(10)) + .build()?) +} + +impl ControlClient { + pub(crate) fn independent(&self) -> Result { + Ok(Self { + client: http_client(self.authorization.clone())?, + authorization: self.authorization.clone(), + base_url: self.base_url.clone(), + }) + } + + pub(crate) fn endpoint(&self, segments: &[&str]) -> Result { + let mut url = self.base_url.clone(); + url.path_segments_mut() + .map_err(|_| ClientError::Configuration("server URL cannot contain an API path"))? + .pop_if_empty() + .extend(segments.iter().copied()); + Ok(url) + } +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/docker_image.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/docker_image.rs new file mode 100644 index 0000000000..f0e33e5df1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/docker_image.rs @@ -0,0 +1,76 @@ +//! Pin Docker publications to physical image evidence held by the privileged server. + +use axum::http::StatusCode; + +#[cfg(unix)] +use crate::api::constants::DOCKER_IMAGE_HEX_LENGTH; +use crate::api::errors::ApiError; +use crate::api::models::DockerImageSource; +use crate::snapshots::models::SnapshotInput; + +pub(crate) async fn pin( + sources: &[DockerImageSource], + mut input: SnapshotInput, +) -> Result { + let Some(source) = sources + .iter() + .find(|source| source.location_id == input.location_id && source.target == input.target) + else { + return Ok(input); + }; + let image = inspect(source).await?; + if input + .image_id + .as_ref() + .is_some_and(|claimed| *claimed != image) + { + return Err(ApiError { + status: StatusCode::CONFLICT, + message: "published image differs from the configured Docker location image".into(), + decision_id: None, + }); + } + input.image_id = Some(image); + Ok(input) +} + +#[cfg(unix)] +async fn inspect(source: &DockerImageSource) -> Result { + let client = reqwest::Client::builder() + .no_proxy() + .unix_socket(source.socket.as_path()) + .timeout(std::time::Duration::from_secs(5)) + .build() + .map_err(|_| unavailable())?; + let response: serde_json::Value = client + .get(format!( + "http://localhost/containers/{}/json", + source.container + )) + .send() + .await + .map_err(|_| unavailable())? + .error_for_status() + .map_err(|_| unavailable())? + .json() + .await + .map_err(|_| unavailable())?; + let image = response["Image"].as_str().ok_or_else(unavailable)?; + let digest = image.strip_prefix("sha256:").ok_or_else(unavailable)?; + (digest.len() == DOCKER_IMAGE_HEX_LENGTH && digest.bytes().all(|byte| byte.is_ascii_hexdigit())) + .then(|| image.to_owned()) + .ok_or_else(unavailable) +} + +#[cfg(not(unix))] +async fn inspect(_source: &DockerImageSource) -> Result { + Err(unavailable()) +} + +fn unavailable() -> ApiError { + ApiError { + status: StatusCode::SERVICE_UNAVAILABLE, + message: "cannot inspect the configured Docker location image".into(), + decision_id: None, + } +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs new file mode 100644 index 0000000000..24a318b3f9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs @@ -0,0 +1,45 @@ +//! Scope both observations and reads before exposing any location history. + +use axum::extract::{Path, State}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::bearer::require_location; +use crate::api::errors::ApiError; +use crate::api::models::MutationResponse; +use crate::credentials::models::TokenIdentity; +use crate::platform::main::location_health::location_health; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::models::{LocationHealthObservation, LocationHealthReport}; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/locations/{location_id}/health", operation_id = "report_location_health", + params(("location_id" = Uuid, Path)), request_body = LocationHealthReport, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn report( + State(store): State, + Extension(identity): Extension, + Path(location_id): Path, + Json(report): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(MutationResponse { + applied: report_location_health(&store, location_id, &report).await?, + })) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/health", operation_id = "read_location_health", + params(("location_id" = Uuid, Path)), + responses((status = 200, body = Vec), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn recent( + State(store): State, + Extension(identity): Extension, + Path(location_id): Path, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(location_health(&store, location_id).await?)) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_work.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_work.rs new file mode 100644 index 0000000000..f6bfb530c3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/location_work.rs @@ -0,0 +1,65 @@ +//! Locations may poll and complete only their own work; enqueueing stays a trusted-owner action. + +use axum::extract::{Path, State}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::bearer::require_location; +use crate::api::errors::ApiError; +use crate::api::models::MutationResponse; +use crate::credentials::models::TokenIdentity; +use crate::platform::main::claim_location_work::claim_location_work; +use crate::platform::main::complete_location_work::complete_location_work; +use crate::platform::main::location_work::location_work; +use crate::platform::models::{ + LocationWorkClaim, LocationWorkPoll, LocationWorkRecord, LocationWorkResult, +}; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/locations/{location_id}/work/poll", + params(("location_id" = Uuid, Path)), request_body = LocationWorkPoll, + responses((status = 200, body = Option), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn poll( + State(store): State, + Extension(identity): Extension, + Path(location_id): Path, + Json(poll): Json, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(claim_location_work(&store, location_id, &poll).await?)) +} + +#[utoipa::path(post, path = "/api/locations/{location_id}/work/{request_id}", + params(("location_id" = Uuid, Path), ("request_id" = Uuid, Path)), request_body = LocationWorkResult, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized"), + (status = 409, description = "Work claim expired or was replaced")), security(("bearer" = [])))] +pub(crate) async fn complete( + State(store): State, + Extension(identity): Extension, + Path((location_id, request_id)): Path<(Uuid, Uuid)>, + Json(result): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(MutationResponse { + applied: complete_location_work(&store, (location_id, request_id), &result).await?, + })) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/work/{request_id}", + params(("location_id" = Uuid, Path), ("request_id" = Uuid, Path)), + responses((status = 200, body = Option), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn result( + State(store): State, + Extension(identity): Extension, + Path((location_id, request_id)): Path<(Uuid, Uuid)>, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(location_work(&store, request_id).await?.filter( + |record| record.request_json.location_id == location_id, + ))) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs new file mode 100644 index 0000000000..a66f36cbc8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs @@ -0,0 +1,16 @@ +pub(crate) mod administrators; +pub(crate) mod audit; +pub(crate) mod changes; +pub(crate) mod client; +pub(crate) mod docker_image; +pub(crate) mod location_health; +pub(crate) mod location_work; +pub(crate) mod person_location_work; +pub(crate) mod person_node_sql; +pub(crate) mod person_operations; +pub(crate) mod router; +pub(crate) mod run_reads; +pub(crate) mod runs; +pub(crate) mod snapshots; +pub(crate) mod trigger_reads; +pub(crate) mod warehouse_metadata; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_location_work.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_location_work.rs new file mode 100644 index 0000000000..f3040a94db --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_location_work.rs @@ -0,0 +1,65 @@ +//! Person connection tests reuse the location's named outbound work, not arbitrary commands. + +use axum::extract::{Path, State}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::errors::ApiError; +use crate::api::models::MutationResponse; +use crate::credentials::models::AuditContext; +use crate::credentials::models::AuthenticatedPerson; +use crate::platform::main::read_person_connection_test::read_person_connection_test; +use crate::platform::main::request_location_work::request_location_work; +use crate::platform::models::{ + ConnectionTestRequest, LocationWork, LocationWorkRecord, LocationWorkRequest, +}; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/control/locations/{location_id}/targets/{target}/connection-tests", + operation_id = "requestConnectionTest", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("x-sqb-csrf" = String, Header)), + request_body = ConnectionTestRequest, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Session rejected"), (status = 403, body = crate::api::models::ApiErrorBody)), + security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn request( + (State(store), Extension(context)): (State, Extension), + Path((location_id, target)): Path<(Uuid, String)>, + Extension(AuthenticatedPerson(principal, _)): Extension, + Json(request): Json, +) -> Result, ApiError> { + Ok(Json(MutationResponse { + applied: request_location_work( + &store, + (&context, Some(principal)), + &LocationWorkRequest { + request_id: request.request_id, + location_id, + target, + reason: request.reason, + work: LocationWork::CheckConnection {}, + }, + ) + .await?, + })) +} + +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/connection-tests/{request_id}", + operation_id = "readConnectionTest", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("request_id" = Uuid, Path)), + responses((status = 200, body = Option), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn result( + (State(store), Extension(context)): (State, Extension), + Path((location_id, target, request_id)): Path<(Uuid, String, Uuid)>, + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result>, ApiError> { + Ok(Json( + read_person_connection_test( + &store, + (&context, principal), + (location_id, &target, request_id), + ) + .await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_node_sql.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_node_sql.rs new file mode 100644 index 0000000000..09af121295 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_node_sql.rs @@ -0,0 +1,31 @@ +//! SQL-only inspection uses live person grants, not the machine snapshot endpoint. + +use axum::extract::{Path, State}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::errors::ApiError; +use crate::credentials::models::AuditContext; +use crate::credentials::models::AuthenticatedPerson; +use crate::snapshots::main::read_person_node_sql::read_person_node_sql; +use crate::snapshots::models::NodeSql; +use crate::store::models::ControlStore; + +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/sql", operation_id = "readPersonNodeSql", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("node_key" = String, Path)), + responses((status = 200, body = Option), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn read( + (State(store), Extension(context)): (State, Extension), + Path((location_id, target, node_key)): Path<(Uuid, String, String)>, + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result>, ApiError> { + Ok(Json( + read_person_node_sql( + &store, + (&context, principal), + (location_id, &target, &node_key), + ) + .await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs new file mode 100644 index 0000000000..02fcaff030 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs @@ -0,0 +1,208 @@ +//! Person operations use person sessions, never the machine-resource bearer router. + +use axum::extract::{Path, Query, State}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::errors::ApiError; +use crate::api::models::{MutationResponse, ReadQuery}; +use crate::credentials::models::AuthenticatedPerson; +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::declarations::main::change_person_trigger::change_person_trigger; +use crate::declarations::models::TriggerChangeRequest; +use crate::runs::main::cancel_person_run::cancel_person_run; +use crate::runs::main::list_person_runs::list_person_runs; +use crate::runs::main::request_person_job_run::request_person_job_run; +use crate::runs::main::request_rerun::request_rerun; +use crate::runs::models::{CancelRunRequest, JobRunRequest, RerunRequest, RunRecord}; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::main::read_person_catalog::read_person_catalog; +use crate::warehouse_metadata::main::read_person_fingerprint::read_person_fingerprint; +use crate::warehouse_metadata::main::read_person_node_status::read_person_node_status; +use crate::warehouse_metadata::main::read_person_statistics::read_person_statistics; +use crate::warehouse_metadata::models::{ + CatalogPage, CatalogQuery, FingerprintObservation, FingerprintQuery, NodeStatus, + NodeStatusQuery, StatisticsPage, StatisticsQuery, StatisticsRetention, +}; + +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/statistics", operation_id = "readPersonStatistics", + params(("location_id" = Uuid, Path), ("target" = String, Path), + ("database" = Option, Query, description = "Exact database; omitted matches SQL NULL"), + ("schema" = String, Query), ("name" = String, Query), + ("after_observed_at" = Option>, Query), + ("after_event_id" = Option, Query), ("limit" = Option, Query)), + responses((status = 200, body = StatisticsPage), (status = 400, description = "Invalid relation or cursor"), + (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn statistics( + (State(store), Extension(context)): (State, Extension), + (Extension(default_role), Extension(retention)): ( + Extension, + Extension, + ), + (Path((location_id, target)), Query(options)): (Path<(Uuid, String)>, Query), + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result, ApiError> { + Ok(Json( + read_person_statistics( + &store, + (&context, principal), + (location_id, &target, &options), + (default_role, retention), + ) + .await?, + )) +} + +#[utoipa::path(post, path = "/api/control/runs", operation_id = "requestJobRun", + params(("x-sqb-csrf" = String, Header)), request_body = JobRunRequest, + responses((status = 200, body = MutationResponse), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn request_job( + (State(store), Extension(context)): (State, Extension), + Extension(AuthenticatedPerson(principal, _)): Extension, + Json(request): Json, +) -> Result, ApiError> { + Ok(Json(MutationResponse { + applied: request_person_job_run(&store, (&context, principal), &request).await?, + })) +} + +#[utoipa::path(post, path = "/api/control/runs/cancel", operation_id = "cancelPersonRun", + params(("x-sqb-csrf" = String, Header)), request_body = CancelRunRequest, + responses((status = 200, body = MutationResponse), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn cancel_run( + (State(store), Extension(context)): (State, Extension), + Extension(AuthenticatedPerson(principal, _)): Extension, + Json(request): Json, +) -> Result, ApiError> { + Ok(Json(MutationResponse { + applied: cancel_person_run(&store, (&context, principal), &request).await?, + })) +} + +#[utoipa::path(post, path = "/api/control/runs/rerun", operation_id = "rerunPersonRun", + params(("x-sqb-csrf" = String, Header)), request_body = RerunRequest, + responses((status = 200, body = MutationResponse), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn rerun( + (State(store), Extension(context)): (State, Extension), + Extension(AuthenticatedPerson(principal, _)): Extension, + Json(request): Json, +) -> Result, ApiError> { + Ok(Json(MutationResponse { + applied: request_rerun(&store, (&context, Some(principal)), &request).await?, + })) +} + +#[utoipa::path(post, path = "/api/control/triggers", operation_id = "changeTrigger", + params(("x-sqb-csrf" = String, Header)), request_body = TriggerChangeRequest, + responses((status = 200, body = MutationResponse), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn change_trigger( + (State(store), Extension(context)): (State, Extension), + Extension(AuthenticatedPerson(principal, _)): Extension, + Json(request): Json, +) -> Result, ApiError> { + Ok(Json(MutationResponse { + applied: change_person_trigger(&store, (&context, principal), &request).await?, + })) +} + +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/runs", operation_id = "listPersonRuns", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("limit" = Option, Query)), + responses((status = 200, body = Vec), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn recent_runs( + (State(store), Extension(context)): (State, Extension), + Extension(default_role): Extension, + (Path((location_id, target)), Query(options)): (Path<(Uuid, String)>, Query), + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result>, ApiError> { + Ok(Json( + list_person_runs( + &store, + (&context, principal), + (location_id, &target, options.limit.unwrap_or(100)), + default_role, + ) + .await?, + )) +} + +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/catalog", operation_id = "readPersonCatalog", + params(("location_id" = Uuid, Path), ("target" = String, Path), + ("database" = Option, Query, description = "Exact database; omitted matches SQL NULL"), + ("schema" = String, Query, description = "Exact physical schema"), + ("after_observed_at" = Option>, Query), + ("after_event_id" = Option, Query), + ("limit" = Option, Query, description = "Page size; defaults to 100, clamped to 1..100")), + responses((status = 200, body = CatalogPage), (status = 400, description = "Invalid namespace or cursor"), + (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn catalog( + (State(store), Extension(context)): (State, Extension), + Extension(default_role): Extension, + (Path((location_id, target)), Query(options)): (Path<(Uuid, String)>, Query), + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result, ApiError> { + Ok(Json( + read_person_catalog( + &store, + (&context, principal), + (location_id, &target, &options), + default_role, + ) + .await?, + )) +} + +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/fingerprint", operation_id = "readPersonFingerprint", + params(("location_id" = Uuid, Path), ("target" = String, Path), + ("node_type" = String, Query), ("node_name" = String, Query), + ("database" = Option, Query, description = "Exact destination database; omitted matches SQL NULL"), + ("schema" = Option, Query, description = "Exact destination schema; omitted matches SQL NULL"), + ("name" = Option, Query, description = "Exact destination name; omitted matches SQL NULL")), + responses((status = 200, body = Option), (status = 400, description = "Invalid node"), + (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn fingerprint( + (State(store), Extension(context)): (State, Extension), + Extension(default_role): Extension, + (Path((location_id, target)), Query(options)): (Path<(Uuid, String)>, Query), + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result>, ApiError> { + Ok(Json( + read_person_fingerprint( + &store, + (&context, principal), + (location_id, &target, &options), + default_role, + ) + .await?, + )) +} + +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/node-status", operation_id = "readPersonNodeStatus", + params(("location_id" = Uuid, Path), ("target" = String, Path), + ("node_key" = String, Query, description = "One exact published node key")), + responses((status = 200, body = Option), (status = 400, description = "Invalid node key"), + (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn node_status( + (State(store), Extension(context)): (State, Extension), + Extension(default_role): Extension, + (Path((location_id, target)), Query(options)): (Path<(Uuid, String)>, Query), + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result>, ApiError> { + Ok(Json( + read_person_node_status( + &store, + (&context, principal), + (location_id, &target, &options), + default_role, + ) + .await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs new file mode 100644 index 0000000000..7613b7fada --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs @@ -0,0 +1,90 @@ +//! Machine resource routes share bearer authentication and bounded request bodies. + +use axum::extract::DefaultBodyLimit; +use axum::routing::{get, post}; +use axum::{Extension, Router, middleware}; + +use crate::api::_helpers::authentication::bearer::authenticate; +use crate::api::_helpers::resources::{ + changes, location_health, location_work, run_reads, runs, snapshots, trigger_reads, + warehouse_metadata, +}; +use crate::api::constants::BODY_BYTES; +use crate::api::models::ChangeFeed; +use crate::store::models::ControlStore; + +pub(crate) fn router(store: &ControlStore, feed: ChangeFeed) -> Router { + Router::new() + .route( + "/api/locations/{location_id}/targets/{target}/changes", + get(changes::subscribe), + ) + .route( + "/api/locations/{location_id}/work/poll", + post(location_work::poll), + ) + .route( + "/api/locations/{location_id}/work/{request_id}", + get(location_work::result).post(location_work::complete), + ) + .route( + "/api/locations/{location_id}/health", + get(location_health::recent).post(location_health::report), + ) + .route( + "/api/locations/{location_id}/targets/{target}/metadata", + post(warehouse_metadata::report), + ) + .route( + "/api/locations/{location_id}/targets/{target}/metadata/statistics-due", + post(warehouse_metadata::statistics_due), + ) + .route( + "/api/locations/{location_id}/targets/{target}/metadata/watermark", + post(warehouse_metadata::watermark), + ) + .route( + "/api/locations/{location_id}/targets/{target}/snapshots", + get(snapshots::current).post(snapshots::publish), + ) + .route( + "/api/locations/{location_id}/targets/{target}/snapshots/metadata", + get(snapshots::metadata), + ) + .route( + "/api/locations/{location_id}/targets/{target}/runs", + get(run_reads::recent), + ) + .route( + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/nodes", + get(run_reads::nodes), + ) + .route( + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/invocations", + get(run_reads::invocations), + ) + .route( + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/recall/{view}", + get(run_reads::recall), + ) + .route( + "/api/locations/{location_id}/targets/{target}/triggers", + get(trigger_reads::current), + ) + .route( + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/events", + get(trigger_reads::events), + ) + .route( + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/ticks", + get(trigger_reads::ticks), + ) + .route("/api/runs/{run_id}/execution", get(runs::execution)) + .route("/api/runs/{run_id}/events", post(runs::report)) + .route("/api/runs/{run_id}/heartbeat", post(runs::heartbeat)) + .route("/api/runs/{run_id}/output", post(runs::output)) + .route("/api/runs/{run_id}/recall", post(runs::recall)) + .layer(DefaultBodyLimit::max(BODY_BYTES)) + .layer(Extension(feed)) + .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/run_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/run_reads.rs new file mode 100644 index 0000000000..ae2dc26f62 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/run_reads.rs @@ -0,0 +1,133 @@ +//! Location credentials can observe only their own runs; they cannot request or mutate work. + +use axum::extract::{Path, Query, State}; +use axum::http::StatusCode; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::bearer::{require_location, require_resource}; +use crate::api::errors::ApiError; +use crate::api::models::{ReadQuery, ReadResource, RunRecallPath, RunRecallQuery}; +use crate::credentials::models::TokenIdentity; +use crate::runs::main::list_run_nodes::list_run_nodes; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{RunNodeRecord, RunRecord}; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::list_recall_invocations::list_recall_invocations; +use crate::store::main::query::query; +use crate::store::main::read_recall_view::read_recall_view; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::types::Row; + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs", operation_id = "list_runs", + params(("location_id" = Uuid, Path), ("target" = String, Path), + ("limit" = Option, Query, description = "Maximum recent runs; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized")), security(("bearer" = [])))] +pub(crate) async fn recent( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Query(options): Query, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json( + list_runs(&store, (location_id, &target), options.limit.unwrap_or(100)).await?, + )) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs/{run_id}/nodes", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("run_id" = Uuid, Path)), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "Run not found")), + security(("bearer" = [])))] +pub(crate) async fn nodes( + State(store): State, + Extension(identity): Extension, + Path((location_id, target, run_id)): Path<(Uuid, String, Uuid)>, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + require_resource(&store, (location_id, &target), (ReadResource::Run, run_id)).await?; + Ok(Json(list_run_nodes(&store, run_id).await?)) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs/{run_id}/invocations", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("run_id" = Uuid, Path), + ("limit" = Option, Query, description = "Maximum invocation namespaces; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "Run not found")), + security(("bearer" = [])))] +pub(crate) async fn invocations( + State(store): State, + Extension(identity): Extension, + Path((location_id, target, run_id)): Path<(Uuid, String, Uuid)>, + Query(options): Query, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + require_resource(&store, (location_id, &target), (ReadResource::Run, run_id)).await?; + let namespaces = run_namespaces(&store, run_id).await?; + Ok(Json( + list_recall_invocations(&store, &namespaces, options.limit.unwrap_or(100)).await?, + )) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs/{run_id}/recall/{view}", operation_id = "read_run_recall", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("run_id" = Uuid, Path), + ("view" = String, Path), ("invocation" = String, Query, description = "Exact scheduler or worker invocation ID"), + ("limit" = Option, Query, description = "Maximum rows; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec), (status = 400, description = "Unknown Recall view"), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized"), + (status = 404, description = "Run or invocation not found")), security(("bearer" = [])))] +pub(crate) async fn recall( + State(store): State, + Extension(identity): Extension, + Path(RunRecallPath { + location_id, + target, + run_id, + view, + }): Path, + Query(options): Query, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + require_resource(&store, (location_id, &target), (ReadResource::Run, run_id)).await?; + let namespaces = run_namespaces(&store, run_id).await?; + if !namespaces.iter().any(|namespace| { + options.invocation == *namespace || options.invocation.starts_with(&format!("{namespace}/")) + }) { + return Err(ApiError { + status: StatusCode::NOT_FOUND, + message: "run invocation not found".into(), + decision_id: None, + }); + } + Ok(Json( + read_recall_view( + &store, + (&view, &options.invocation), + options.limit.unwrap_or(100), + ) + .await?, + )) +} + +/// Immutable claim evidence remains available after terminal publication removes liveness. +async fn run_namespaces(store: &ControlStore, run_id: Uuid) -> Result, ApiError> { + let scheduler = format!("scheduler/{run_id}"); + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, claims) = query( + transaction, + "SELECT payload FROM recall_events WHERE invocation_id=$1 AND event_type='run_claimed' ORDER BY invocation_sequence DESC LIMIT 1", + vec![Parameter::Text(scheduler.clone())], + ).await?; + commit(transaction).await?; + let mut namespaces = vec![scheduler]; + if let Some(claim_id) = claims + .first() + .and_then(|row| row["payload"]["claim"]["claim_id"].as_str()) + { + namespaces.push(format!("worker/{claim_id}")); + } + Ok(namespaces) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/runs.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/runs.rs new file mode 100644 index 0000000000..5933d5e27d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/runs.rs @@ -0,0 +1,127 @@ +//! Worker resource reports bind to the authenticated run and claim, never a caller-supplied actor. + +use axum::extract::{Path, State}; +use axum::http::StatusCode; +use axum::{Extension, Json}; +use sqlbuild_recall::records::models::OutputChunk; +use sqlbuild_recall::store::models::InsertOutcome; +use uuid::Uuid; + +use crate::api::errors::ApiError; +use crate::api::models::{HeartbeatResponse, MutationResponse}; +use crate::credentials::models::{CredentialScope, TokenIdentity}; +use crate::runs::main::change_run::change_run; +use crate::runs::main::heartbeat_run::heartbeat_run; +use crate::runs::main::publish_run_output::publish_run_output; +use crate::runs::main::publish_run_recall::publish_run_recall; +use crate::runs::main::run_execution::run_execution; +use crate::runs::models::{ + OutputPublication, RecallBatch, RunAuthority, RunChangeRequest, RunExecution, +}; +use crate::store::models::ControlStore; + +#[utoipa::path(get, path = "/api/runs/{run_id}/execution", params(("run_id" = Uuid, Path)), + responses((status = 200, body = RunExecution), (status = 401, description = "Inactive claim or invalid credential"), + (status = 403, description = "Run not authorized")), security(("bearer" = [])))] +pub(crate) async fn execution( + State(store): State, + Extension(identity): Extension, + Path(run_id): Path, +) -> Result, ApiError> { + let claim_id = require_worker(&identity, run_id)?; + let execution = run_execution(&store, (run_id, claim_id)) + .await? + .ok_or_else(|| ApiError { + status: StatusCode::UNAUTHORIZED, + message: "run claim is no longer active".into(), + decision_id: None, + })?; + Ok(Json(execution)) +} + +#[utoipa::path(post, path = "/api/runs/{run_id}/events", operation_id = "report_run_event", params(("run_id" = Uuid, Path)), + request_body = RunChangeRequest, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Run not authorized")), + security(("bearer" = [])))] +pub(crate) async fn report( + State(store): State, + Extension(identity): Extension, + Path(run_id): Path, + Json(request): Json, +) -> Result, ApiError> { + let claim_id = require_worker(&identity, run_id)?; + if request.run_id != run_id || request.claim_id != Some(claim_id) { + return Err(ApiError { + status: StatusCode::FORBIDDEN, + message: "report does not match the authenticated run and claim".into(), + decision_id: None, + }); + } + let applied = change_run(&store, RunAuthority::Worker { run_id, claim_id }, &request).await?; + Ok(Json(MutationResponse { applied })) +} + +#[utoipa::path(post, path = "/api/runs/{run_id}/heartbeat", params(("run_id" = Uuid, Path)), + responses((status = 200, body = HeartbeatResponse), (status = 401, description = "Invalid credential"), + (status = 403, description = "Run not authorized")), security(("bearer" = [])))] +pub(crate) async fn heartbeat( + State(store): State, + Extension(identity): Extension, + Path(run_id): Path, +) -> Result, ApiError> { + let claim_id = require_worker(&identity, run_id)?; + Ok(Json(HeartbeatResponse { + cancel_requested: heartbeat_run(&store, (run_id, claim_id)).await?, + })) +} + +#[utoipa::path(post, path = "/api/runs/{run_id}/output", params(("run_id" = Uuid, Path)), + request_body = Vec, + responses((status = 200, body = OutputPublication), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Run not authorized")), + security(("bearer" = [])))] +pub(crate) async fn output( + State(store): State, + Extension(identity): Extension, + Path(run_id): Path, + Json(chunks): Json>, +) -> Result, ApiError> { + let claim_id = require_worker(&identity, run_id)?; + Ok(Json( + publish_run_output(&store, (run_id, claim_id), &chunks).await?, + )) +} + +#[utoipa::path(post, path = "/api/runs/{run_id}/recall", params(("run_id" = Uuid, Path)), + request_body = RecallBatch, + responses((status = 200, body = InsertOutcome), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Run not authorized")), + security(("bearer" = [])))] +pub(crate) async fn recall( + State(store): State, + Extension(identity): Extension, + Path(run_id): Path, + Json(batch): Json, +) -> Result, ApiError> { + let claim_id = require_worker(&identity, run_id)?; + Ok(Json( + publish_run_recall(&store, (run_id, claim_id), &batch).await?, + )) +} + +fn require_worker(identity: &TokenIdentity, run_id: Uuid) -> Result { + if let CredentialScope::Run { + run_id: permitted, + claim_id, + } = identity.scope + && permitted == run_id + { + return Ok(claim_id); + } + Err(ApiError { + status: StatusCode::FORBIDDEN, + message: "credential does not authorize this run".into(), + decision_id: None, + }) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/snapshots.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/snapshots.rs new file mode 100644 index 0000000000..99afc12f8c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/snapshots.rs @@ -0,0 +1,106 @@ +//! A location credential publishes and reads only its own location's snapshots. + +use axum::extract::{Path, State}; +use axum::http::StatusCode; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::bearer::require_location; +use crate::api::_helpers::resources::docker_image; +use crate::api::errors::ApiError; +use crate::api::models::{DevelopmentScope, DockerImageSource}; +use crate::credentials::models::TokenIdentity; +use crate::snapshots::main::current_snapshot::current_snapshot; +use crate::snapshots::main::current_snapshot_record::current_snapshot_record; +use crate::snapshots::main::publish_development_snapshot::publish_development_snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::{CurrentSnapshot, SnapshotInput, SnapshotRecord}; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/snapshots", + params(("location_id" = Uuid, Path), ("target" = String, Path)), + request_body = SnapshotInput, + responses((status = 200, body = SnapshotRecord), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized"), + (status = 409, description = "Stale generation")), security(("bearer" = [])))] +pub(crate) async fn publish( + State(store): State, + (Extension(identity), Extension(images)): ( + Extension, + Extension>>, + ), + (Path((location_id, target)), development): ( + Path<(Uuid, String)>, + Option>, + ), + Json(input): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + if input.location_id != location_id || input.target != target { + return Err(ApiError { + status: StatusCode::BAD_REQUEST, + message: "snapshot does not match the requested location and target".into(), + decision_id: None, + }); + } + let pause_new = if let Some(Extension(scope)) = development { + if scope.location_id != location_id || scope.target != target { + return Err(ApiError { + status: StatusCode::FORBIDDEN, + message: "snapshot is outside the development scope".into(), + decision_id: None, + }); + } + !scope.triggers + } else { + false + }; + let input = docker_image::pin(&images, input).await?; + Ok(Json(if pause_new { + publish_development_snapshot(&store, &input).await? + } else { + publish_snapshot(&store, &input).await? + })) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/snapshots", + params(("location_id" = Uuid, Path), ("target" = String, Path)), + responses((status = 200, body = CurrentSnapshot), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "No successful snapshot")), + security(("bearer" = [])))] +pub(crate) async fn current( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + let current = current_snapshot(&store, location_id, &target) + .await? + .ok_or_else(|| ApiError { + status: StatusCode::NOT_FOUND, + message: "no successful snapshot".into(), + decision_id: None, + })?; + Ok(Json(current)) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/snapshots/metadata", + params(("location_id" = Uuid, Path), ("target" = String, Path)), + responses((status = 200, body = SnapshotRecord), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "No successful snapshot")), + security(("bearer" = [])))] +pub(crate) async fn metadata( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + let record = current_snapshot_record(&store, location_id, &target) + .await? + .ok_or_else(|| ApiError { + status: StatusCode::NOT_FOUND, + message: "no successful snapshot".into(), + decision_id: None, + })?; + Ok(Json(record)) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/trigger_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/trigger_reads.rs new file mode 100644 index 0000000000..a3bf1e5327 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/trigger_reads.rs @@ -0,0 +1,77 @@ +//! Trigger definitions and immutable history are visible only within the authenticated location. + +use axum::extract::{Path, Query, State}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::bearer::{require_location, require_resource}; +use crate::api::errors::ApiError; +use crate::api::models::{ReadQuery, ReadResource}; +use crate::credentials::models::TokenIdentity; +use crate::declarations::main::list_trigger_events::list_trigger_events; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::{TriggerChangeRecord, TriggerRecord}; +use crate::scheduler::main::list_ticks::list_ticks; +use crate::scheduler::models::TickRecord; +use crate::store::models::ControlStore; + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/triggers", operation_id = "list_triggers", + params(("location_id" = Uuid, Path), ("target" = String, Path)), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized")), security(("bearer" = [])))] +pub(crate) async fn current( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(list_triggers(&store, location_id, &target).await?)) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/events", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("trigger_id" = Uuid, Path), + ("limit" = Option, Query, description = "Maximum events; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "Trigger not found")), + security(("bearer" = [])))] +pub(crate) async fn events( + State(store): State, + Extension(identity): Extension, + Path((location_id, target, trigger_id)): Path<(Uuid, String, Uuid)>, + Query(options): Query, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + require_resource( + &store, + (location_id, &target), + (ReadResource::Trigger, trigger_id), + ) + .await?; + Ok(Json( + list_trigger_events(&store, trigger_id, options.limit.unwrap_or(100)).await?, + )) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/ticks", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("trigger_id" = Uuid, Path), + ("limit" = Option, Query, description = "Maximum recent ticks; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "Trigger not found")), + security(("bearer" = [])))] +pub(crate) async fn ticks( + State(store): State, + Extension(identity): Extension, + Path((location_id, target, trigger_id)): Path<(Uuid, String, Uuid)>, + Query(options): Query, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + require_resource( + &store, + (location_id, &target), + (ReadResource::Trigger, trigger_id), + ) + .await?; + Ok(Json( + list_ticks(&store, trigger_id, options.limit.unwrap_or(100)).await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs new file mode 100644 index 0000000000..9354aada8d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs @@ -0,0 +1,90 @@ +//! An authenticated location can mirror only its own registered reader's target. + +use axum::extract::{Path, State}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::bearer::require_location; +use crate::api::errors::ApiError; +use crate::credentials::models::TokenIdentity; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::main::ingest_audit_results::ingest_audit_results; +use crate::warehouse_metadata::main::ingest_catalog::ingest_catalog; +use crate::warehouse_metadata::main::ingest_fingerprints::ingest_fingerprints; +use crate::warehouse_metadata::main::ingest_janitor_events::ingest_janitor_events; +use crate::warehouse_metadata::main::ingest_microbatches::ingest_microbatches; +use crate::warehouse_metadata::main::ingest_migrations::ingest_migrations; +use crate::warehouse_metadata::main::ingest_old_name_views::ingest_old_name_views; +use crate::warehouse_metadata::main::ingest_source_freshness::ingest_source_freshness; +use crate::warehouse_metadata::main::ingest_statistics::ingest_statistics; +use crate::warehouse_metadata::main::read_statistics_due::read_statistics_due; +use crate::warehouse_metadata::main::read_watermark::read_watermark; +use crate::warehouse_metadata::models::{ + MetadataBatch, StatisticsDueRead, StatisticsRead, WarehouseHistoryKind, WarehouseSyncResponse, + WarehouseWatermarkRead, +}; + +#[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/metadata", operation_id = "report_metadata", + params(("location_id" = Uuid, Path), ("target" = String, Path)), request_body = MetadataBatch, + responses((status = 200, body = WarehouseSyncResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized"), + (status = 409, description = "Reader generation fenced")), security(("bearer" = [])))] +pub(crate) async fn report( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(batch): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + let scope = (location_id, target.as_str()); + let response = match batch.kind { + WarehouseHistoryKind::Fingerprints => ingest_fingerprints(&store, scope, &batch).await, + WarehouseHistoryKind::SourceFreshness => { + ingest_source_freshness(&store, scope, &batch).await + } + WarehouseHistoryKind::AuditResults => ingest_audit_results(&store, scope, &batch).await, + WarehouseHistoryKind::JanitorEvents => ingest_janitor_events(&store, scope, &batch).await, + WarehouseHistoryKind::Migrations => ingest_migrations(&store, scope, &batch).await, + WarehouseHistoryKind::OldNameViews => ingest_old_name_views(&store, scope, &batch).await, + WarehouseHistoryKind::Microbatches => ingest_microbatches(&store, scope, &batch).await, + WarehouseHistoryKind::Catalog => ingest_catalog(&store, scope, &batch).await, + WarehouseHistoryKind::Statistics => ingest_statistics(&store, scope, &batch).await, + }?; + Ok(Json(response)) +} + +#[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/metadata/statistics-due", + params(("location_id" = Uuid, Path), ("target" = String, Path)), request_body = StatisticsDueRead, + responses((status = 200, body = StatisticsRead), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized"), + (status = 409, description = "Reader generation fenced")), security(("bearer" = [])))] +pub(crate) async fn statistics_due( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(request): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json( + read_statistics_due(&store, (location_id, &target), &request).await?, + )) +} + +#[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/metadata/watermark", + params(("location_id" = Uuid, Path), ("target" = String, Path)), request_body = WarehouseWatermarkRead, + responses((status = 200, body = Option>), + (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized"), + (status = 409, description = "Reader generation fenced")), + security(("bearer" = [])))] +pub(crate) async fn watermark( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(request): Json, +) -> Result>>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json( + read_watermark(&store, (location_id, &target), &request).await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/constants.rs b/crates/sqlbuild-orchestration/src/api/constants.rs new file mode 100644 index 0000000000..b360c21b28 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/constants.rs @@ -0,0 +1,15 @@ +//! Shared HTTP transport bounds for the server and native clients. + +pub const BODY_BYTES: usize = 64 * 1024 * 1024; +/// Two 1024-byte passwords can each expand sixfold through JSON escaping. +pub(crate) const PASSWORD_BODY_BYTES: usize = 16 * 1024; +pub(crate) const PASSWORD_COOKIE: &str = "__Host-sqb-session"; +pub(crate) const PASSWORD_CSRF_COOKIE: &str = "__Host-sqb-csrf"; +pub(crate) const PASSWORD_CSRF_HEADER: &str = "x-sqb-csrf"; +pub(crate) const PROXY_USER_HEADER: &str = "x-sqb-user"; +pub(crate) const OIDC_FLOW_COOKIE: &str = "__Host-sqb-oidc"; +pub(crate) const OIDC_RESPONSE_BYTES: usize = 1024 * 1024; +#[cfg(unix)] +pub(crate) const DOCKER_IMAGE_HEX_LENGTH: usize = 64; +pub(crate) const HTTPS_SCHEME: &str = "https"; +pub(crate) const ROOT_PATH: &str = "/"; diff --git a/crates/sqlbuild-orchestration/src/api/errors.rs b/crates/sqlbuild-orchestration/src/api/errors.rs new file mode 100644 index 0000000000..ded2c84b0e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/errors.rs @@ -0,0 +1,78 @@ +//! HTTP errors never return database diagnostics, connection strings, or credentials. + +use axum::Json; +use axum::http::{HeaderValue, StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use uuid::Uuid; + +use crate::api::models::ApiErrorBody; +use crate::store::errors::ControlError; + +#[derive(Debug, thiserror::Error)] +pub enum ClientError { + #[error("invalid API connection: {0}")] + Configuration(&'static str), + #[error("API request failed: {0}")] + Transport(#[from] reqwest::Error), + #[error("API rejected the request with HTTP {0}")] + Rejected(reqwest::StatusCode), +} + +#[derive(Debug)] +pub struct ApiError { + pub(crate) status: StatusCode, + pub(crate) message: String, + pub(crate) decision_id: Option, +} + +impl From for ApiError { + fn from(error: ControlError) -> Self { + match error { + ControlError::Invalid(message) => Self { + status: StatusCode::BAD_REQUEST, + message, + decision_id: None, + }, + ControlError::Denied { + reason, + decision_id, + } => Self { + status: StatusCode::FORBIDDEN, + message: format!("{reason}; decision {decision_id}"), + decision_id: Some(decision_id), + }, + ControlError::StaleGeneration | ControlError::Fenced => Self { + status: StatusCode::CONFLICT, + message: error.to_string(), + decision_id: None, + }, + _ => Self { + status: StatusCode::INTERNAL_SERVER_ERROR, + message: "control-store operation failed".into(), + decision_id: None, + }, + } + } +} + +impl IntoResponse for ApiError { + fn into_response(self) -> Response { + let mut response = ( + self.status, + Json(ApiErrorBody { + error: self.message, + decision_id: self.decision_id, + }), + ) + .into_response(); + response + .headers_mut() + .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + if self.status == StatusCode::UNAUTHORIZED { + response + .headers_mut() + .insert(header::WWW_AUTHENTICATE, HeaderValue::from_static("Bearer")); + } + response + } +} diff --git a/crates/sqlbuild-orchestration/src/api/main/client_response.rs b/crates/sqlbuild-orchestration/src/api/main/client_response.rs new file mode 100644 index 0000000000..2eb643424f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/client_response.rs @@ -0,0 +1,15 @@ +//! Share bounded JSON transport without exposing server bodies or credential-bearing URLs. + +use serde::de::DeserializeOwned; + +use crate::api::errors::ClientError; + +pub(crate) async fn client_response( + request: reqwest::RequestBuilder, +) -> Result { + let response = request.send().await.map_err(reqwest::Error::without_url)?; + if !response.status().is_success() { + return Err(ClientError::Rejected(response.status())); + } + Ok(response.json().await.map_err(reqwest::Error::without_url)?) +} diff --git a/crates/sqlbuild-orchestration/src/api/main/connect_client.rs b/crates/sqlbuild-orchestration/src/api/main/connect_client.rs new file mode 100644 index 0000000000..3c717eb4f6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/connect_client.rs @@ -0,0 +1,31 @@ +//! Validate the API origin before attaching a sensitive bearer credential. + +use reqwest::header::HeaderValue; + +use crate::api::_helpers::resources::client::http_client; +use crate::api::errors::ClientError; +use crate::api::models::ControlClient; + +pub(crate) fn connect_client(server_url: &str, bearer: &str) -> Result { + let base_url = reqwest::Url::parse(server_url) + .map_err(|_| ClientError::Configuration("server URL must be an absolute HTTP URL"))?; + if !matches!(base_url.scheme(), "http" | "https") + || base_url.host_str().is_none() + || !base_url.username().is_empty() + || base_url.password().is_some() + || base_url.query().is_some() + || base_url.fragment().is_some() + { + return Err(ClientError::Configuration( + "server URL must use HTTP or HTTPS without credentials, query or fragment", + )); + } + let mut authorization = HeaderValue::from_str(&format!("Bearer {bearer}")) + .map_err(|_| ClientError::Configuration("bearer is not a valid HTTP credential"))?; + authorization.set_sensitive(true); + Ok(ControlClient { + client: http_client(authorization.clone())?, + authorization, + base_url, + }) +} diff --git a/crates/sqlbuild-orchestration/src/api/main/mod.rs b/crates/sqlbuild-orchestration/src/api/main/mod.rs new file mode 100644 index 0000000000..db4d8c57d7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod client_response; +pub(crate) mod connect_client; +pub mod openapi_document; +pub mod parse_origin; +pub mod serve_api; diff --git a/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs b/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs new file mode 100644 index 0000000000..21bea0101a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs @@ -0,0 +1,47 @@ +//! Generate the HTTP contract from the request/response types used by the actual endpoints. + +use utoipa::OpenApi; +use utoipa::openapi::security::{ApiKey, ApiKeyValue, HttpAuthScheme, HttpBuilder, SecurityScheme}; + +use crate::api::constants::{PASSWORD_COOKIE, PROXY_USER_HEADER}; +use crate::api::models::ControlApi; + +#[must_use] +pub fn openapi_document() -> utoipa::openapi::OpenApi { + let mut document = ControlApi::openapi(); + document + .components + .get_or_insert_with(Default::default) + .add_security_scheme( + "bearer", + SecurityScheme::Http(HttpBuilder::new().scheme(HttpAuthScheme::Bearer).build()), + ); + document + .components + .get_or_insert_with(Default::default) + .add_security_scheme( + "session", + SecurityScheme::ApiKey(ApiKey::Cookie(ApiKeyValue::new(PASSWORD_COOKIE))), + ); + document + .components + .get_or_insert_with(Default::default) + .add_security_scheme( + "cliSession", + SecurityScheme::ApiKey(ApiKey::Header(ApiKeyValue::with_description( + "Authorization", + "Session from sqb login; never combine with Cookie. Writes also require x-sqb-csrf.", + ))), + ); + document + .components + .get_or_insert_with(Default::default) + .add_security_scheme( + "proxy", + SecurityScheme::ApiKey(ApiKey::Header(ApiKeyValue::with_description( + PROXY_USER_HEADER, + "Upstream-replaced subject; only accepted from explicitly configured socket peers", + ))), + ); + document +} diff --git a/crates/sqlbuild-orchestration/src/api/main/parse_origin.rs b/crates/sqlbuild-orchestration/src/api/main/parse_origin.rs new file mode 100644 index 0000000000..74cc32b878 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/parse_origin.rs @@ -0,0 +1,28 @@ +//! Browser authentication and CLI handoffs share one strict HTTPS origin contract. + +use crate::api::constants::{HTTPS_SCHEME, ROOT_PATH}; +use crate::store::errors::ControlError; + +pub fn parse_origin(value: &str, mechanism: &str) -> Result { + let error = || { + ControlError::Invalid(format!( + "{mechanism} authentication requires an HTTPS origin without credentials, path, query or fragment" + )) + }; + let url = reqwest::Url::parse(value).map_err(|_| { + ControlError::Invalid(format!( + "{mechanism} authentication requires an HTTPS origin" + )) + })?; + if url.scheme() != HTTPS_SCHEME + || url.host().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.path() != ROOT_PATH + || url.query().is_some() + || url.fragment().is_some() + { + return Err(error()); + } + Ok(url.origin().ascii_serialization()) +} diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs new file mode 100644 index 0000000000..fa7bcd20ef --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -0,0 +1,75 @@ +//! One native server owns migrations and all storage writes; locations and workers speak HTTP. + +use axum::routing::get; +use axum::{Extension, Json, Router, middleware}; + +use crate::api::_helpers::authentication::{bearer::local_request, configure::configure}; +use crate::api::_helpers::resources::{changes, router::router}; +use crate::api::main::openapi_document::openapi_document; +use crate::api::models::{ApiAuthentication, ChangeFeed, DevelopmentScope, DockerImageSource}; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::subscribe_changes::subscribe_changes; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::models::StatisticsRetention; + +pub async fn serve_api( + store: ControlStore, + listener: tokio::net::TcpListener, + (authentication, images, retention): ( + ApiAuthentication, + Vec, + StatisticsRetention, + ), + (ready, shutdown): ( + impl FnOnce() + Send + 'static, + impl Future + Send + 'static, + ), +) -> Result<(), ControlError> { + let address = listener.local_addr()?; + let (development_scope, authentication_routes) = + configure(&store, authentication, address).await?; + migrate_store(&store).await?; + let (sender, listener_changes) = subscribe_changes(&store).await?; + let (stop, stopped) = tokio::sync::watch::channel(false); + let feed = ChangeFeed { + sender: sender.clone(), + stopped, + }; + let application = router(&store, feed) + .merge(authentication_routes) + .layer(Extension(std::sync::Arc::new(images))) + .layer(Extension(retention)) + .route( + "/api/openapi.json", + get(|| async { Json(openapi_document()) }), + ) + .with_state(store.clone()); + let application = development_application(application, development_scope, address); + let server = axum::serve( + listener, + application.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(async move { + shutdown.await; + stop.send_replace(true); + }); + ready(); + tokio::select! { + result = async { server.await } => { result?; Ok(()) }, + result = changes::relay(store.clone(), listener_changes, sender) => result, + } +} + +fn development_application( + application: Router, + scope: Option, + address: std::net::SocketAddr, +) -> Router { + match scope { + Some(scope) => application + .layer(Extension(scope)) + .layer(middleware::from_fn_with_state(address, local_request)), + None => application, + } +} diff --git a/crates/sqlbuild-orchestration/src/api/mod.rs b/crates/sqlbuild-orchestration/src/api/mod.rs new file mode 100644 index 0000000000..46b8c6bfe2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/mod.rs @@ -0,0 +1,12 @@ +//! Python-free HTTP endpoints; locations and workers have no direct database access. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; +pub(crate) mod types; + +#[cfg(test)] +#[path = "../../tests/api/test_contracts.rs"] +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs new file mode 100644 index 0000000000..60f122991c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -0,0 +1,352 @@ +//! Typed HTTP responses shared with the generated OpenAPI contract. + +use serde::{Deserialize, Serialize}; + +use crate::api::_helpers::authentication::password_sessions; +use crate::api::_helpers::authentication::{cli_login, oidc}; +use crate::api::_helpers::resources::{ + administrators, audit, changes, location_health, location_work, person_location_work, + person_node_sql, person_operations, run_reads, runs, snapshots, trigger_reads, + warehouse_metadata, +}; + +#[derive(Debug, utoipa::OpenApi)] +#[openapi(paths( + administrators::change, + administrators::assign, + audit::read, + person_operations::request_job, + person_operations::cancel_run, + person_operations::rerun, + person_operations::change_trigger, + person_operations::recent_runs, + person_operations::catalog, + person_operations::statistics, + person_operations::fingerprint, + person_operations::node_status, + person_location_work::request, + person_location_work::result, + person_node_sql::read, + password_sessions::login, + password_sessions::current, + password_sessions::logout, + password_sessions::change_password, + cli_login::start, + cli_login::approve, + cli_login::claim, + oidc::login, + oidc::callback, + changes::subscribe, + location_health::report, + location_health::recent, + location_work::poll, + location_work::complete, + location_work::result, + snapshots::publish, + snapshots::current, + snapshots::metadata, + runs::execution, + runs::report, + runs::heartbeat, + runs::output, + runs::recall, + run_reads::recent, + run_reads::nodes, + run_reads::invocations, + run_reads::recall, + trigger_reads::current, + trigger_reads::events, + trigger_reads::ticks, + warehouse_metadata::report, + warehouse_metadata::statistics_due, + warehouse_metadata::watermark +))] +pub(crate) struct ControlApi; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ReadQuery { + pub(crate) limit: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ReadAuditQuery { + pub(crate) after: Option, + pub(crate) limit: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RunRecallQuery { + pub(crate) invocation: String, + pub(crate) limit: Option, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct RunRecallPath { + pub(crate) location_id: uuid::Uuid, + pub(crate) target: String, + pub(crate) run_id: uuid::Uuid, + pub(crate) view: String, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) enum ReadResource { + Run, + Trigger, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct MutationResponse { + pub applied: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct HeartbeatResponse { + pub cancel_requested: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ApiErrorBody { + pub error: String, + /// Existing durable denial audit ID; absent when no decision was recorded. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub decision_id: Option, +} + +/// SSE invalidation topics; `all` means refetch the authorized REST resources. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ChangeNotice { + pub topic: String, +} + +/// Explicit local development scope; never enabled by ordinary server startup. +#[derive(Clone, Debug)] +pub struct DevelopmentScope { + pub location_id: uuid::Uuid, + pub target: String, + pub triggers: bool, +} + +/// The one way people sign in to this server, if any. +#[derive(Debug, Default)] +pub enum HumanLogin { + #[default] + None, + Development(DevelopmentScope), + Password(String), + TrustedProxy(TrustedProxyAuthentication), + Oidc(OidcAuthentication), +} + +#[derive(Debug, Default)] +pub struct ApiAuthentication { + pub login: HumanLogin, + pub default_role: crate::credentials::models::DefaultRole, +} + +/// Only the server's installation configuration selects the daemon and container to inspect. +#[derive(Clone, Debug)] +pub struct DockerImageSource { + pub location_id: uuid::Uuid, + pub target: String, + pub socket: std::path::PathBuf, + pub container: String, +} + +#[derive(Clone, Debug)] +pub(crate) struct BrowserOrigin(pub(crate) String); + +/// Explicit person bindings require an upstream-replaced X-SQB-User; no provisioning or grants. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TrustedProxyAuthentication { + pub origin: String, + pub trusted_peers: std::collections::BTreeSet, + pub identities: std::collections::BTreeMap, +} + +/// Exact issuer/subject bindings authenticate existing people without provisioning or grants. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OidcAuthentication { + pub origin: String, + pub issuer: String, + pub client_id: String, + pub client_secret_env: Option, + pub certificate_authority: Option, + pub identities: std::collections::BTreeMap, +} + +#[derive(Debug)] +pub(crate) struct OidcRuntime { + pub(crate) settings: OidcAuthentication, + pub(crate) client_secret: Option, + pub(crate) http: reqwest::Client, + pub(crate) configuration_id: String, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct OidcCallback { + pub(crate) state: Option, + pub(crate) code: Option, +} + +#[derive(Clone, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ExternalIdentityResponse { + pub principal_id: uuid::Uuid, + pub session_id: uuid::Uuid, + pub expires_at: chrono::DateTime, + pub csrf_token: String, +} + +impl std::fmt::Debug for ExternalIdentityResponse { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ExternalIdentityResponse") + .field("principal_id", &self.principal_id) + .field("session_id", &self.session_id) + .field("expires_at", &self.expires_at) + .finish_non_exhaustive() + } +} + +#[derive(Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct PasswordLoginRequest { + pub(crate) username: String, + pub(crate) password: String, +} + +impl std::fmt::Debug for PasswordLoginRequest { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PasswordLoginRequest") + .field("username", &self.username) + .finish_non_exhaustive() + } +} + +#[derive(Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct PasswordChangeRequest { + pub(crate) current_password: String, + pub(crate) new_password: String, +} + +impl std::fmt::Debug for PasswordChangeRequest { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PasswordChangeRequest") + .finish_non_exhaustive() + } +} + +#[derive(Serialize, Deserialize, utoipa::ToSchema)] +pub struct PasswordSessionResponse { + pub identity: crate::credentials::models::SessionIdentity, + pub csrf_token: String, +} + +#[derive(Debug, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct CliLoginStartRequest { + pub(crate) client_hash: String, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) enum BrowserLoginMethod { + Password, + Oidc, + Proxy, +} + +#[derive(Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct CliLoginApprovalRequest { + pub(crate) browser_code: String, + pub(crate) approve: bool, +} + +#[derive(Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct CliLoginClaimRequest { + pub(crate) flow_id: uuid::Uuid, + pub(crate) client_secret: String, +} + +#[derive(Serialize, utoipa::ToSchema)] +pub(crate) struct CliLoginSessionResponse { + pub(crate) identity: crate::credentials::models::SessionIdentity, + pub(crate) bearer: String, +} + +impl std::fmt::Debug for CliLoginApprovalRequest { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("CliLoginApprovalRequest") + .field("approve", &self.approve) + .finish_non_exhaustive() + } +} + +impl std::fmt::Debug for CliLoginClaimRequest { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("CliLoginClaimRequest") + .field("flow_id", &self.flow_id) + .finish_non_exhaustive() + } +} + +impl std::fmt::Debug for CliLoginSessionResponse { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("CliLoginSessionResponse") + .field("identity", &self.identity) + .finish_non_exhaustive() + } +} + +impl std::fmt::Debug for PasswordSessionResponse { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PasswordSessionResponse") + .field("identity", &self.identity) + .finish_non_exhaustive() + } +} + +#[derive(Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct AuthenticationDenial { + pub error: String, + pub request_id: uuid::Uuid, +} + +#[derive(Clone, Debug)] +pub(crate) struct ChangeFeed { + pub(crate) sender: tokio::sync::broadcast::Sender, + pub(crate) stopped: tokio::sync::watch::Receiver, +} + +/// Sensitive headers redact the bearer from Debug output and all wire notifications. +#[derive(Debug)] +pub(crate) struct ChangeStream { + pub(crate) store: crate::store::models::ControlStore, + pub(crate) authorization: Option, + pub(crate) location_id: uuid::Uuid, + pub(crate) target: String, + pub(crate) target_hash: String, + pub(crate) receiver: tokio::sync::broadcast::Receiver, + pub(crate) stopped: tokio::sync::watch::Receiver, + pub(crate) keepalive: tokio::time::Interval, + pub(crate) initial: bool, +} + +#[derive(Clone, Debug)] +pub(crate) struct ControlClient { + pub(crate) client: reqwest::Client, + pub(crate) authorization: reqwest::header::HeaderValue, + pub(crate) base_url: reqwest::Url, +} diff --git a/crates/sqlbuild-orchestration/src/api/types.rs b/crates/sqlbuild-orchestration/src/api/types.rs new file mode 100644 index 0000000000..2a77575870 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/types.rs @@ -0,0 +1,12 @@ +//! The provider's discovered authorization and token endpoints retain their typed availability. + +use openidconnect::{EndpointMaybeSet, EndpointNotSet, EndpointSet}; + +pub(crate) type OidcClient = openidconnect::core::CoreClient< + EndpointSet, + EndpointNotSet, + EndpointNotSet, + EndpointNotSet, + EndpointMaybeSet, + EndpointMaybeSet, +>; diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_changes.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_changes.rs new file mode 100644 index 0000000000..21516e9f8a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_changes.rs @@ -0,0 +1,107 @@ +//! Apply eligibility/membership changes without releasing the caller's authorization lock. + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::_helpers::administrators::{ + deny_change, grant_installation_admin, remaining_administrator, +}; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{AdministratorChange, AuditContext}; +use crate::store::errors::ControlError; +use crate::store::main::{commit::commit, query::query}; +use crate::store::models::{Change, ControlTransaction, Parameter}; + +pub(crate) async fn apply_change( + transaction: ControlTransaction, + context: &AuditContext, + (principal_id, change): (Uuid, AdministratorChange), +) -> Result { + let (mut transaction, rows) = query(transaction, + "SELECT p.disabled_at,a.assignment_id FROM sqb_principals p LEFT JOIN sqb_role_assignments a \ + ON p.principal_id=a.principal_id AND a.role_key='admin' AND a.location_id IS NULL \ + AND a.target IS NULL WHERE p.principal_id=$1", + vec![Parameter::Id(principal_id)]).await?; + let Some(person) = rows.first() else { + return deny_change( + transaction, + context, + principal_id, + "target must be a person", + ) + .await; + }; + let active = person["disabled_at"].is_null(); + let administrator = !person["assignment_id"].is_null(); + if active + && administrator + && matches!( + change, + AdministratorChange::Revoke | AdministratorChange::Disable + ) + { + let other; + (transaction, other) = remaining_administrator(transaction, principal_id).await?; + if !other { + return deny_change( + transaction, + context, + principal_id, + "cannot remove or disable the final installation administrator", + ) + .await; + } + } + let now = transaction.now; + + let changed; + let action = match change { + AdministratorChange::Grant => { + if !active { + return deny_change( + transaction, + context, + principal_id, + "administrator must be an active person", + ) + .await; + } + let applied; + (transaction, applied) = + grant_installation_admin(transaction, context, principal_id, "manual").await?; + commit(transaction).await?; + return Ok(applied); + } + AdministratorChange::Revoke => { + (transaction, changed) = query( + transaction, + "DELETE FROM sqb_role_assignments WHERE principal_id=$1 AND role_key='admin' \ + AND location_id IS NULL AND target IS NULL RETURNING assignment_id", + vec![Parameter::Id(principal_id)], + ) + .await?; + "role.revoked" + } + AdministratorChange::Disable | AdministratorChange::Enable => { + let disable = change == AdministratorChange::Disable; + let predicate = if disable { "IS NULL" } else { "IS NOT NULL" }; + (transaction, changed) = query(transaction, + &format!("UPDATE sqb_principals SET disabled_at=$1 WHERE principal_id=$2 AND disabled_at {predicate} RETURNING principal_id"), + vec![Parameter::OptionalTime(disable.then_some(now)),Parameter::Id(principal_id)]).await?; + if disable { + "principal.disabled" + } else { + "principal.enabled" + } + } + }; + let applied = !changed.is_empty(); + if applied { + transaction = append_audit(transaction, context, (Uuid::now_v7(), action), + (json!({"principal_id":principal_id}), json!({"administrator":administrator,"enabled":active}), + json!({"administrator":administrator && change!=AdministratorChange::Revoke,"enabled":change==AdministratorChange::Enable || (active && change!=AdministratorChange::Disable)}))).await?; + transaction.changes.insert(Change::Credentials); + } + commit(transaction).await?; + Ok(applied) +} diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_roles.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_roles.rs new file mode 100644 index 0000000000..700d64f5a2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrator_roles.rs @@ -0,0 +1,144 @@ +//! Authenticated installation administrators assign additive built-ins; code never assigns people. + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::_helpers::administrator_changes::apply_change; +use crate::authorization::main::deny_authorization::deny; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{ + AdministratorChange, AuditContext, BuiltinRole, RoleAssignmentRequest, RoleKey, +}; +use crate::store::errors::ControlError; +use crate::store::main::{commit::commit, query::query}; +use crate::store::models::{Change, ControlTransaction, Parameter}; + +pub(crate) async fn assign_role_in( + transaction: ControlTransaction, + context: &AuditContext, + request: &RoleAssignmentRequest, +) -> Result { + if request + .target + .as_ref() + .is_some_and(|target| target.trim().is_empty()) + || (request.target.is_some() && request.location_id.is_none()) + { + return Err(ControlError::Invalid( + "role target requires a location and a nonempty name".into(), + )); + } + if let RoleKey::Custom(key) = &request.role + && (key.trim().is_empty() || request.location_id.is_none()) + { + return Err(ControlError::Invalid( + "custom role assignment requires a location and a nonempty role key".into(), + )); + } + if request.role == RoleKey::Builtin(BuiltinRole::Admin) { + if request.location_id.is_some() || request.target.is_some() { + return deny( + transaction, + context, + json!(request), + "administrator assignment must be installation-wide", + ) + .await; + } + return apply_change( + transaction, + context, + ( + request.principal_id, + if request.revoke { + AdministratorChange::Revoke + } else { + AdministratorChange::Grant + }, + ), + ) + .await; + } + let transaction = if let RoleKey::Custom(key) = &request.role + && !request.revoke + { + let (transaction, published) = query(transaction, + "SELECT role_key FROM sqb_effective_roles WHERE location_id=$1 AND role_key=$2 AND ($3 IS NULL OR target=$3) LIMIT 1", + vec![Parameter::OptionalId(request.location_id), Parameter::Text(key.clone()), Parameter::OptionalText(request.target.clone())]).await?; + if published.is_empty() { + return Err(ControlError::Invalid(format!( + "role {key} is neither a built-in role nor a custom role this location has published" + ))); + } + transaction + } else { + transaction + }; + let (transaction, people) = query( + transaction, + "SELECT principal_id,disabled_at FROM sqb_principals WHERE principal_id=$1", + vec![Parameter::Id(request.principal_id)], + ) + .await?; + if people + .first() + .is_none_or(|person| !request.revoke && !person["disabled_at"].is_null()) + { + return deny( + transaction, + context, + json!(request), + "role assignment requires an enabled person; revocation requires an existing person", + ) + .await; + } + assign_scoped_role(transaction, context, request).await +} + +async fn assign_scoped_role( + transaction: ControlTransaction, + context: &AuditContext, + request: &RoleAssignmentRequest, +) -> Result { + let role = serde_json::to_value(&request.role) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let key = serde_json::to_string(&( + request.principal_id, + &request.role, + request.location_id, + &request.target, + )) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let assignment_id = Uuid::now_v7(); + let (mut transaction, changed) = if request.revoke { + query( + transaction, + "DELETE FROM sqb_role_assignments WHERE active_key=$1 RETURNING assignment_id", + vec![Parameter::Text(key)], + ) + .await? + } else { + query(transaction, "INSERT INTO sqb_role_assignments (assignment_id,principal_id,role_key,location_id,target,active_key) VALUES ($1,$2,$3,$4,$5,$6) ON CONFLICT (active_key) DO NOTHING RETURNING assignment_id", + vec![Parameter::Id(assignment_id),Parameter::Id(request.principal_id),Parameter::Text(role.as_str().unwrap_or_default().into()),Parameter::OptionalId(request.location_id),Parameter::OptionalText(request.target.clone()),Parameter::Text(key)]).await? + }; + let applied = !changed.is_empty(); + if applied { + transaction = append_audit( + transaction, + context, + ( + assignment_id, + if request.revoke { + "role.revoked" + } else { + "role.assigned" + }, + ), + (json!(request), json!(null), json!({"assignments":changed})), + ) + .await?; + transaction.changes.insert(Change::Credentials); + } + commit(transaction).await?; + Ok(applied) +} diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/administrators.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrators.rs new file mode 100644 index 0000000000..a427309cbf --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/administrators.rs @@ -0,0 +1,91 @@ +//! Installation administrator membership; callers establish bootstrap or authenticated authority. + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::main::deny_authorization::deny; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{Change, ControlTransaction, Parameter}; + +pub(crate) async fn grant_installation_admin( + transaction: ControlTransaction, + context: &AuditContext, + principal_id: Uuid, + source: &str, +) -> Result<(ControlTransaction, bool), ControlError> { + let (transaction, people) = query( + transaction, + "SELECT principal_id FROM sqb_principals WHERE principal_id=$1 AND disabled_at IS NULL", + vec![Parameter::Id(principal_id)], + ) + .await?; + if people.is_empty() { + return Err(ControlError::Invalid( + "administrator must be an active person".into(), + )); + } + let assignment_id = Uuid::now_v7(); + let (mut transaction, rows) = query( + transaction, + "INSERT INTO sqb_role_assignments (assignment_id,principal_id,role_key,active_key) \ + VALUES ($1,$2,'admin',$3) ON CONFLICT (active_key) DO NOTHING RETURNING assignment_id", + vec![ + Parameter::Id(assignment_id), + Parameter::Id(principal_id), + Parameter::Text(format!("{principal_id}:admin:installation")), + ], + ) + .await?; + let applied = !rows.is_empty(); + if applied { + transaction = append_audit( + transaction, + context, + (assignment_id, "role.assigned"), + ( + json!({"principal_id": principal_id}), + json!(null), + json!({"role_key":"admin", "source":source, "assignment_id":assignment_id}), + ), + ) + .await?; + transaction.changes.insert(Change::Credentials); + } + Ok((transaction, applied)) +} + +/// Another active installation administrator exists; a lost one is restored with grant-admin. +pub(crate) async fn remaining_administrator( + transaction: ControlTransaction, + principal_id: Uuid, +) -> Result<(ControlTransaction, bool), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT COUNT(*) AS administrators FROM sqb_role_assignments a JOIN sqb_principals p USING(principal_id) \ + WHERE a.role_key='admin' AND a.location_id IS NULL AND a.target IS NULL \ + AND p.disabled_at IS NULL AND p.principal_id<>$1", + vec![Parameter::Id(principal_id)]).await?; + Ok(( + transaction, + rows.first() + .and_then(|row| row["administrators"].as_i64()) + .is_some_and(|count| count > 0), + )) +} + +pub(crate) async fn deny_change( + transaction: ControlTransaction, + context: &AuditContext, + principal_id: Uuid, + reason: &str, +) -> Result { + deny( + transaction, + context, + json!({"principal_id":principal_id}), + reason, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/custom_roles.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/custom_roles.rs new file mode 100644 index 0000000000..e15fb43cdc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/custom_roles.rs @@ -0,0 +1,64 @@ +//! Published custom roles grant only non-administrative actions. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value; + +use crate::credentials::models::{Action, RoleDefinition}; +use crate::store::errors::ControlError; + +const ADMINISTRATIVE: [Action; 3] = [Action::UserManage, Action::RoleAssign, Action::AuditView]; + +const BUILTIN: [&str; 4] = ["viewer", "operator", "developer", "admin"]; + +fn action_name(action: Action) -> Result { + match serde_json::to_value(action) { + Ok(Value::String(name)) => Ok(name), + _ => Err(ControlError::Invalid("unnamed permission action".into())), + } +} + +pub(crate) fn decode_roles( + document: Option<&Value>, +) -> Result>, ControlError> { + let roles: BTreeMap = document.map_or_else( + || Ok(BTreeMap::new()), + |document| { + serde_json::from_value(document.clone()) + .map_err(|error| ControlError::Invalid(format!("roles: {error}"))) + }, + )?; + let mut published: BTreeMap> = BTreeMap::new(); + for (key, definition) in roles { + if key.trim().is_empty() + || BUILTIN.contains(&key.to_lowercase().as_str()) + || definition.description.trim().is_empty() + || definition + .actions + .iter() + .any(|action| ADMINISTRATIVE.contains(action)) + { + return Err(ControlError::Invalid(format!( + "role {key} needs a description, a key that is not a built-in role, and only non-administrative actions" + ))); + } + let mut grants: BTreeSet = BTreeSet::new(); + for action in definition.actions { + grants.insert(action_name(action)?); + } + published.insert(key, grants); + } + Ok(published) +} + +pub(crate) fn decode_grants(value: &Value) -> Result, ControlError> { + serde_json::from_str(value.as_str().unwrap_or("[]")) + .map_err(|error| ControlError::Schema(format!("effective role grants: {error}"))) +} + +pub(crate) fn grants_allow(value: &Value, action: Action) -> bool { + match (decode_grants(value), action_name(action)) { + (Ok(grants), Ok(name)) => grants.contains(&name), + _ => false, + } +} diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/mod.rs new file mode 100644 index 0000000000..fe8542b42c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod administrator_changes; +pub(crate) mod administrator_roles; +pub(crate) mod administrators; +pub(crate) mod custom_roles; diff --git a/crates/sqlbuild-orchestration/src/authorization/main/assign_builtin_role.rs b/crates/sqlbuild-orchestration/src/authorization/main/assign_builtin_role.rs new file mode 100644 index 0000000000..782780ae85 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/assign_builtin_role.rs @@ -0,0 +1,32 @@ +//! Only live installation administrators may assign or revoke additive built-in roles. + +use uuid::Uuid; + +use crate::authorization::_helpers::administrator_roles::assign_role_in; +use crate::authorization::main::authorize_person::authorize_person; +use crate::credentials::models::{ + Action, AuditContext, DefaultRole, PermissionScope, RoleAssignmentRequest, +}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn assign_builtin_role( + store: &ControlStore, + (context, credentials): (&AuditContext, Uuid), + request: &RoleAssignmentRequest, +) -> Result { + let (transaction, context) = authorize_person( + begin(store, TransactionMode::Write).await?, + (context, credentials), + &PermissionScope { + action: Action::RoleAssign, + location_id: None, + target: None, + nodes: Default::default(), + }, + DefaultRole::None, + ) + .await?; + assign_role_in(transaction, &context, request).await +} diff --git a/crates/sqlbuild-orchestration/src/authorization/main/authorize_person.rs b/crates/sqlbuild-orchestration/src/authorization/main/authorize_person.rs new file mode 100644 index 0000000000..6a1c545bec --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/authorize_person.rs @@ -0,0 +1,91 @@ +//! A live person and their additive grants are evaluated under the operation's store lock. + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::_helpers::custom_roles::grants_allow; +use crate::authorization::main::deny_authorization::deny; +use crate::credentials::models::{Action, AuditContext, BuiltinRole, DefaultRole, PermissionScope}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +fn role_allows(role: BuiltinRole, action: Action) -> bool { + use Action::{ + ConnectionTest, JobRun, RunAdhoc, RunCancel, RunRerun, TriggerToggle, View, ViewSql, + }; + match role { + BuiltinRole::Admin => true, + BuiltinRole::Viewer => action == View, + BuiltinRole::Operator => { + matches!(action, View | JobRun | RunCancel | RunRerun | TriggerToggle) + } + BuiltinRole::Developer => matches!( + action, + View | JobRun + | RunCancel + | RunRerun + | TriggerToggle + | ViewSql + | RunAdhoc + | ConnectionTest + ), + } +} + +pub(crate) async fn authorize_person( + transaction: ControlTransaction, + (context, principal_id): (&AuditContext, Uuid), + scope: &PermissionScope, + default_role: DefaultRole, +) -> Result<(ControlTransaction, AuditContext), ControlError> { + let mut context = AuditContext { + actor: format!("principal:{principal_id}"), + granting_role: "none".into(), + ..context.clone() + }; + let (transaction, people) = query( + transaction, + "SELECT principal_id FROM sqb_principals WHERE principal_id=$1 AND disabled_at IS NULL", + vec![Parameter::Id(principal_id)], + ) + .await?; + if people.is_empty() { + return deny( + transaction, + &context, + json!(scope), + "authenticated active person required", + ) + .await; + } + let (transaction, rows) = query(transaction, + "SELECT a.role_key,a.location_id,a.target,e.resolved_grants_json FROM sqb_role_assignments a \ + LEFT JOIN sqb_effective_roles e ON e.location_id=a.location_id AND e.role_key=a.role_key AND e.target=$2 \ + WHERE a.principal_id=$1 ORDER BY a.role_key", + vec![Parameter::Id(principal_id), Parameter::OptionalText(scope.target.clone())]).await?; + for row in rows { + let allowed = serde_json::from_value::(row["role_key"].clone()).map_or_else( + |_| grants_allow(&row["resolved_grants_json"], scope.action), + |role| role_allows(role, scope.action), + ); + if allowed + && (row["location_id"].is_null() || row["location_id"] == json!(scope.location_id)) + && (row["target"].is_null() || row["target"] == json!(scope.target)) + { + context.granting_role = row["role_key"].as_str().unwrap_or_default().into(); + return Ok((transaction, context)); + } + } + if default_role == DefaultRole::Viewer && role_allows(BuiltinRole::Viewer, scope.action) { + context.granting_role = "default:viewer".into(); + return Ok((transaction, context)); + } + deny( + transaction, + &context, + json!(scope), + "missing permission in this scope; all requested nodes denied", + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/authorization/main/bootstrap_user.rs b/crates/sqlbuild-orchestration/src/authorization/main/bootstrap_user.rs new file mode 100644 index 0000000000..d7a4a62ad4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/bootstrap_user.rs @@ -0,0 +1,51 @@ +//! Direct local store ownership creates the first person as administrator; later people use grant-admin. + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::_helpers::administrators::grant_installation_admin; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::main::identities::insert_principal::insert_principal; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn bootstrap_user( + store: &ControlStore, + context: &AuditContext, + name: &str, +) -> Result { + let (transaction, rows) = query( + begin(store, TransactionMode::Write).await?, + "SELECT principal_id FROM sqb_principals LIMIT 1", + vec![], + ) + .await?; + if !rows.is_empty() { + return Err(ControlError::Invalid( + "people already exist, so bootstrap is closed; use grant-admin".into(), + )); + } + let (transaction, principal_id) = insert_principal(transaction, context, name).await?; + let (transaction, _) = + grant_installation_admin(transaction, context, principal_id, "bootstrap").await?; + commit( + append_audit( + transaction, + context, + ( + Uuid::new_v5(&principal_id, b"administrator-initialized"), + "admin.initialized", + ), + ( + json!({"installation":true}), + json!(null), + json!({"principal_id":principal_id}), + ), + ) + .await?, + ) + .await?; + Ok(principal_id) +} diff --git a/crates/sqlbuild-orchestration/src/authorization/main/change_administrator.rs b/crates/sqlbuild-orchestration/src/authorization/main/change_administrator.rs new file mode 100644 index 0000000000..f8a0ba33b7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/change_administrator.rs @@ -0,0 +1,35 @@ +//! Live administrator checks, final-admin protection and audit share one lock. + +use uuid::Uuid; + +use crate::authorization::_helpers::administrator_changes::apply_change; +use crate::authorization::main::authorize_person::authorize_person; +use crate::credentials::models::{ + Action, AdministratorChange, AuditContext, DefaultRole, PermissionScope, +}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn change_administrator( + store: &ControlStore, + (context, actor): (&AuditContext, Uuid), + (principal_id, change): (Uuid, AdministratorChange), +) -> Result { + let (transaction, context) = authorize_person( + begin(store, TransactionMode::Write).await?, + (context, actor), + &PermissionScope { + action: match change { + AdministratorChange::Grant | AdministratorChange::Revoke => Action::RoleAssign, + AdministratorChange::Disable | AdministratorChange::Enable => Action::UserManage, + }, + location_id: None, + target: None, + nodes: Default::default(), + }, + DefaultRole::None, + ) + .await?; + apply_change(transaction, &context, (principal_id, change)).await +} diff --git a/crates/sqlbuild-orchestration/src/authorization/main/deny_authorization.rs b/crates/sqlbuild-orchestration/src/authorization/main/deny_authorization.rs new file mode 100644 index 0000000000..f1a76a2ca9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/deny_authorization.rs @@ -0,0 +1,37 @@ +//! A denial is returned only after its decision ID and authoritative scope commit durably. + +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::commit::commit; +use crate::store::models::ControlTransaction; + +pub(crate) async fn deny( + transaction: ControlTransaction, + context: &AuditContext, + scope: Value, + reason: &str, +) -> Result { + let decision_id = Uuid::now_v7(); + commit( + append_audit( + transaction, + context, + (decision_id, "authorization.denied"), + ( + scope, + json!(null), + json!({"decision_id":decision_id,"allowed":false,"reason":reason}), + ), + ) + .await?, + ) + .await?; + Err(ControlError::Denied { + reason: reason.into(), + decision_id, + }) +} diff --git a/crates/sqlbuild-orchestration/src/authorization/main/grant_admin.rs b/crates/sqlbuild-orchestration/src/authorization/main/grant_admin.rs new file mode 100644 index 0000000000..2560912eff --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/grant_admin.rs @@ -0,0 +1,54 @@ +//! Local store ownership makes an existing person an administrator. + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::_helpers::administrators::grant_installation_admin; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::main::sessions::revoke_session::revoke_session; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{Change, ControlStore, Parameter, TransactionMode}; + +pub async fn grant_admin( + store: &ControlStore, + context: &AuditContext, + principal_id: Uuid, +) -> Result<(), ControlError> { + let (transaction, people) = query( + begin(store, TransactionMode::Write).await?, + "SELECT disabled_at FROM sqb_principals WHERE principal_id=$1", + vec![Parameter::Id(principal_id)], + ) + .await?; + let Some(person) = people.first() else { + return Err(ControlError::Invalid( + "grant-admin requires an existing person".into(), + )); + }; + let enabled = person["disabled_at"].is_null(); + let (transaction, _) = query( + transaction, + "UPDATE sqb_principals SET disabled_at=NULL WHERE principal_id=$1", + vec![Parameter::Id(principal_id)], + ) + .await?; + let (transaction, _) = + grant_installation_admin(transaction, context, principal_id, "local").await?; + let (mut transaction, sessions_revoked) = + revoke_session(transaction, context, (principal_id, None)).await?; + transaction = append_audit( + transaction, + context, + (Uuid::now_v7(), "admin.granted"), + ( + json!({"principal_id":principal_id}), + json!({"enabled":enabled}), + json!({"enabled":true,"role":"admin","sessions_revoked":sessions_revoked}), + ), + ) + .await?; + transaction.changes.insert(Change::Credentials); + commit(transaction).await +} diff --git a/crates/sqlbuild-orchestration/src/authorization/main/mod.rs b/crates/sqlbuild-orchestration/src/authorization/main/mod.rs new file mode 100644 index 0000000000..b5f174516c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/mod.rs @@ -0,0 +1,9 @@ +pub mod assign_builtin_role; +pub(crate) mod authorize_person; +pub mod bootstrap_user; +pub mod change_administrator; +pub(crate) mod deny_authorization; +pub mod grant_admin; +pub(crate) mod publish_custom_roles; +pub(crate) mod read_audit_events; +pub mod read_security_audit; diff --git a/crates/sqlbuild-orchestration/src/authorization/main/publish_custom_roles.rs b/crates/sqlbuild-orchestration/src/authorization/main/publish_custom_roles.rs new file mode 100644 index 0000000000..7e0fe23386 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/publish_custom_roles.rs @@ -0,0 +1,43 @@ +//! Each published `roles` document replaces the target's custom roles, which take effect at once. + +use serde_json::Value; + +use crate::authorization::_helpers::custom_roles::decode_roles; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{Change, ControlTransaction, Parameter}; + +pub(crate) async fn publish_custom_roles( + mut transaction: ControlTransaction, + record: &SnapshotRecord, + document: Option<&Value>, +) -> Result { + let roles = decode_roles(document)?; + (transaction, _) = query( + transaction, + "DELETE FROM sqb_effective_roles WHERE location_id=$1 AND target=$2", + vec![ + Parameter::Id(record.location_id), + Parameter::Text(record.target.clone()), + ], + ) + .await?; + for (key, grants) in roles { + let grants = serde_json::to_string(&grants) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + (transaction, _) = query( + transaction, + "INSERT INTO sqb_effective_roles (location_id,target,role_key,resolved_grants_json) VALUES ($1,$2,$3,$4)", + vec![ + Parameter::Id(record.location_id), + Parameter::Text(record.target.clone()), + Parameter::Text(key), + Parameter::Text(grants), + ], + ) + .await?; + } + transaction.changes.insert(Change::Credentials); + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/authorization/main/read_audit_events.rs b/crates/sqlbuild-orchestration/src/authorization/main/read_audit_events.rs new file mode 100644 index 0000000000..8791f4349e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/read_audit_events.rs @@ -0,0 +1,25 @@ +//! Authorized server consumers page security facts independently of clocks and transport. + +use crate::credentials::models::SecurityAuditEvent; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_audit_events( + transaction: ControlTransaction, + (after, limit): (i64, u32), +) -> Result<(ControlTransaction, Vec), ControlError> { + if after < 0 || !(1..=100).contains(&limit) { + return Err(ControlError::Invalid( + "audit pages require a nonnegative storage cursor and a limit between 1 and 100".into(), + )); + } + let (transaction, rows) = query(transaction, + "SELECT storage_id,event_id,occurred_at,actor,action,granting_role,request_id,ip_address,scope_json,before_json,after_json \ + FROM sqb_security_audit WHERE storage_id>$1 ORDER BY storage_id LIMIT $2", + vec![Parameter::Integer(after), Parameter::Integer(i64::from(limit))]).await?; + Ok(( + transaction, + rows.into_iter().map(decode_row).collect::>()?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/authorization/main/read_security_audit.rs b/crates/sqlbuild-orchestration/src/authorization/main/read_security_audit.rs new file mode 100644 index 0000000000..322a4cdcfe --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/main/read_security_audit.rs @@ -0,0 +1,34 @@ +//! Live installation-wide audit permission and each bounded page share one transaction. + +use uuid::Uuid; + +use crate::authorization::main::authorize_person::authorize_person; +use crate::authorization::main::read_audit_events::read_audit_events; +use crate::credentials::models::{ + Action, AuditContext, DefaultRole, PermissionScope, SecurityAuditEvent, +}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn read_security_audit( + store: &ControlStore, + (context, credentials): (&AuditContext, Uuid), + (after, limit): (i64, u32), +) -> Result, ControlError> { + let (transaction, _) = authorize_person( + begin(store, TransactionMode::Write).await?, + (context, credentials), + &PermissionScope { + action: Action::AuditView, + location_id: None, + target: None, + nodes: Default::default(), + }, + DefaultRole::None, + ) + .await?; + let (transaction, events) = read_audit_events(transaction, (after, limit)).await?; + commit(transaction).await?; + Ok(events) +} diff --git a/crates/sqlbuild-orchestration/src/authorization/mod.rs b/crates/sqlbuild-orchestration/src/authorization/mod.rs new file mode 100644 index 0000000000..d1ddfc554a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/authorization/mod.rs @@ -0,0 +1,4 @@ +//! Closed, scoped person permissions and serialized installation-administrator lifecycle. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/active_session.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/active_session.rs new file mode 100644 index 0000000000..a974d99546 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/active_session.rs @@ -0,0 +1,26 @@ +//! All person-session consumers share live person, expiry, revocation and password-version checks. + +use uuid::Uuid; + +use crate::credentials::models::StoredSession; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn active_session( + transaction: ControlTransaction, + id: Uuid, +) -> Result<(ControlTransaction, Option), ControlError> { + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT s.session_id,s.principal_id,s.expires_at,s.secret_hash,s.csrf_hash \ + FROM sqb_sessions s JOIN sqb_principals p ON p.principal_id=s.principal_id \ + LEFT JOIN sqb_password_credentials c ON c.principal_id=s.principal_id AND c.credential_id=s.credential_id \ + WHERE s.session_id=$1 AND s.revoked_at IS NULL AND s.expires_at>$2 \ + AND p.disabled_at IS NULL AND (s.credential_id IS NULL OR c.credential_id IS NOT NULL)", + vec![Parameter::Id(id), Parameter::Time(now)]).await?; + Ok(( + transaction, + rows.into_iter().next().map(decode_row).transpose()?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/audit_retention.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/audit_retention.rs new file mode 100644 index 0000000000..cdcb9bdeec --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/audit_retention.rs @@ -0,0 +1,70 @@ +//! Bounded audit expiry batches that never outrun a sink or the storage order. + +use chrono::{DateTime, Utc}; + +use crate::credentials::constants::AUDIT_RETENTION_BATCH; +use crate::credentials::models::AuditRetention; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The newest row is never deleted, so the next MAX+1 storage ID is never reused. +pub(crate) async fn prune_audit_batch( + store: &ControlStore, + (retention, sinks): (AuditRetention, &[String]), + after: i64, +) -> Result<(usize, Option), ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let cutoff: DateTime = retention.cutoff(transaction.now); + let acknowledged: String = (0..sinks.len()) + .map(|index| { + format!( + " AND EXISTS (SELECT 1 FROM sqb_security_audit_deliveries d \ + WHERE d.sink_id=${} AND d.storage_id=a.storage_id)", + index + 4 + ) + }) + .collect(); + let mut parameters = vec![ + Parameter::Time(cutoff), + Parameter::Integer(after), + Parameter::Integer(AUDIT_RETENTION_BATCH as i64), + ]; + parameters.extend(sinks.iter().cloned().map(Parameter::Text)); + let (transaction, rows) = query( + transaction, + &format!( + "SELECT a.storage_id FROM sqb_security_audit a \ + WHERE a.occurred_at<$1 AND a.storage_id>$2 \ + AND a.storage_id<(SELECT MAX(storage_id) FROM sqb_security_audit){acknowledged} \ + ORDER BY a.storage_id LIMIT $3" + ), + parameters, + ) + .await?; + let next = (rows.len() == AUDIT_RETENTION_BATCH) + .then(|| rows.last().and_then(|row| row["storage_id"].as_i64())) + .flatten(); + let expired: Vec = rows + .iter() + .filter_map(|row| row["storage_id"].as_i64()) + .map(Parameter::Integer) + .collect(); + let mut transaction = transaction; + if !expired.is_empty() { + let placeholders = (1..=expired.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + for table in ["sqb_security_audit_deliveries", "sqb_security_audit"] { + (transaction, _) = query( + transaction, + &format!("DELETE FROM {table} WHERE storage_id IN ({placeholders})"), + expired.clone(), + ) + .await?; + } + } + commit(transaction).await?; + Ok((expired.len(), next)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/browser_sessions.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/browser_sessions.rs new file mode 100644 index 0000000000..991a4e04d0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/browser_sessions.rs @@ -0,0 +1,60 @@ +//! Issue the same bounded, hashed browser credentials after the caller verifies a person. + +use chrono::{Duration, SubsecRound}; +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::credentials::_helpers::secrets::new_bearer; +use crate::credentials::constants::PASSWORD_SESSION_SECONDS; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{AuditContext, IssuedSession, SessionIdentity}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn issue_session( + transaction: ControlTransaction, + context: &AuditContext, + (principal_id, credential_id): (Uuid, Option), +) -> Result<(ControlTransaction, IssuedSession), ControlError> { + let now = transaction.now; + let session_id = Uuid::now_v7(); + let bearer = new_bearer(session_id)?; + let csrf = new_bearer(session_id)?; + let identity = SessionIdentity { + session_id, + principal_id, + expires_at: (now + Duration::seconds(PASSWORD_SESSION_SECONDS)).trunc_subsecs(6), + }; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_sessions (session_id,principal_id,secret_hash,csrf_hash,created_at,expires_at,credential_id) VALUES ($1,$2,$3,$4,$5,$6,$7)", + vec![ + Parameter::Id(session_id), + Parameter::Id(principal_id), + Parameter::Text(sha256_hex(bearer.as_bytes())), + Parameter::Text(sha256_hex(csrf.as_bytes())), + Parameter::Time(now), + Parameter::Time(identity.expires_at), + Parameter::OptionalId(credential_id), + ]).await?; + let transaction = append_audit( + transaction, + context, + (session_id, "session.created"), + ( + json!({"principal_id": principal_id}), + json!(null), + json!(&identity), + ), + ) + .await?; + Ok(( + transaction, + IssuedSession { + identity, + bearer, + csrf, + }, + )) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/locations.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/locations.rs new file mode 100644 index 0000000000..06657a0414 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/locations.rs @@ -0,0 +1,37 @@ +//! Create a server-owned location identity independent of its mutable label. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn create_location_in( + mut transaction: ControlTransaction, + name: &str, + context: &AuditContext, +) -> Result<(ControlTransaction, Uuid), ControlError> { + if name.trim().is_empty() { + return Err(ControlError::Invalid("location name is required".into())); + } + let id = Uuid::now_v7(); + let now = transaction.now; + (transaction, _) = query(transaction, + "INSERT INTO sqb_locations (location_id, name, created_by, created_at) VALUES ($1, $2, $3, $4)", + vec![Parameter::Id(id), Parameter::Text(name.into()), Parameter::Text(context.actor.clone()), Parameter::Time(now)]).await?; + let transaction = append_audit( + transaction, + context, + (Uuid::new_v5(&id, b"created"), "location.created"), + ( + json!({"location_id": id}), + json!(null), + json!({"name": name}), + ), + ) + .await?; + Ok((transaction, id)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs new file mode 100644 index 0000000000..020e67bc15 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod active_session; +pub(crate) mod audit_retention; +pub(crate) mod browser_sessions; +pub(crate) mod locations; +pub(crate) mod password_sessions; +pub(crate) mod passwords; +pub(crate) mod scopes; +pub(crate) mod secrets; +pub(crate) mod tokens; diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/password_sessions.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/password_sessions.rs new file mode 100644 index 0000000000..7ea3000f49 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/password_sessions.rs @@ -0,0 +1,36 @@ +//! Session creation rechecks the verified password version under the control-store write lock. + +use crate::credentials::_helpers::browser_sessions; +use crate::credentials::models::{AuditContext, IssuedSession, VerifiedPassword}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn issue_session( + store: &ControlStore, + context: &AuditContext, + proof: VerifiedPassword, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, rows) = query(transaction, + "SELECT p.principal_id FROM sqb_principals p JOIN sqb_password_credentials c ON p.principal_id=c.principal_id \ + WHERE p.principal_id=$1 AND c.credential_id=$2 AND p.disabled_at IS NULL", + vec![Parameter::Id(proof.principal_id), Parameter::Id(proof.credential_id)]).await?; + if rows.is_empty() { + commit(transaction).await?; + return Ok(None); + } + let context = AuditContext { + actor: format!("principal:{}", proof.principal_id), + granting_role: "password-session".into(), + ..context.clone() + }; + let (transaction, session) = browser_sessions::issue_session( + transaction, + &context, + (proof.principal_id, Some(proof.credential_id)), + ) + .await?; + commit(transaction).await?; + Ok(Some(session)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/passwords.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/passwords.rs new file mode 100644 index 0000000000..01a77c91bf --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/passwords.rs @@ -0,0 +1,55 @@ +//! Memory-hard password work runs outside the async executor and outside write transactions. + +use argon2::Argon2; +use argon2::password_hash::{Error, PasswordHash, PasswordHasher, PasswordVerifier, SaltString}; + +use crate::credentials::constants::{PASSWORD_ALGORITHM, PASSWORD_KDF_SLOTS}; +use crate::store::errors::ControlError; + +static PASSWORD_WORK: tokio::sync::Semaphore = + tokio::sync::Semaphore::const_new(PASSWORD_KDF_SLOTS); + +pub(crate) async fn hash_password(password: String) -> Result { + let permit = PASSWORD_WORK + .acquire() + .await + .map_err(|_| ControlError::Invalid("password workers are unavailable".into()))?; + tokio::task::spawn_blocking(move || { + let _permit = permit; + let mut random = [0_u8; 16]; + getrandom::fill(&mut random) + .map_err(|_| ControlError::Invalid("could not generate a password salt".into()))?; + let salt = SaltString::encode_b64(&random) + .map_err(|_| ControlError::Invalid("could not encode a password salt".into()))?; + Argon2::default() + .hash_password(password.as_bytes(), &salt) + .map(|hash| hash.to_string()) + .map_err(|_| ControlError::Invalid("could not hash the password".into())) + }) + .await? +} + +pub(crate) async fn verify_hash(hash: String, password: String) -> Result { + let permit = PASSWORD_WORK + .acquire() + .await + .map_err(|_| ControlError::Invalid("password workers are unavailable".into()))?; + tokio::task::spawn_blocking(move || { + let _permit = permit; + let hash = PasswordHash::new(&hash) + .map_err(|_| ControlError::Invalid("stored password credential is invalid".into()))?; + if hash.algorithm.as_str() != PASSWORD_ALGORITHM { + return Err(ControlError::Invalid( + "stored password credential is not Argon2id".into(), + )); + } + match Argon2::default().verify_password(password.as_bytes(), &hash) { + Ok(()) => Ok(true), + Err(Error::Password) => Ok(false), + Err(_) => Err(ControlError::Invalid( + "stored password credential could not be verified".into(), + )), + } + }) + .await? +} diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/scopes.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/scopes.rs new file mode 100644 index 0000000000..9a0feb5b5c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/scopes.rs @@ -0,0 +1,35 @@ +//! Service credentials stop being valid with their live location or run claim. + +use crate::credentials::models::CredentialScope; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Bound values in location/run/claim column order for the credential insert. +pub(crate) fn scope_parameters(scope: CredentialScope) -> Vec { + match scope { + CredentialScope::Location { location_id } => [Some(location_id), None, None], + CredentialScope::Run { run_id, claim_id } => [None, Some(run_id), Some(claim_id)], + } + .into_iter() + .map(Parameter::OptionalId) + .collect() +} + +pub(crate) async fn scope_is_active( + transaction: ControlTransaction, + scope: CredentialScope, +) -> Result<(ControlTransaction, bool), ControlError> { + let (sql, parameters) = match scope { + CredentialScope::Location { location_id } => ( + "SELECT location_id FROM sqb_locations WHERE location_id=$1 AND archived_at IS NULL", + vec![Parameter::Id(location_id)], + ), + CredentialScope::Run { run_id, claim_id } => ( + "SELECT run_id FROM sqb_runs WHERE run_id=$1 AND worker_instance=$2", + vec![Parameter::Id(run_id), Parameter::Id(claim_id)], + ), + }; + let (transaction, rows) = query(transaction, sql, parameters).await?; + Ok((transaction, !rows.is_empty())) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/secrets.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/secrets.rs new file mode 100644 index 0000000000..e0e05f0827 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/secrets.rs @@ -0,0 +1,43 @@ +//! One encoding and constant-time check for high-entropy bearer secrets. + +use std::fmt::Write; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use subtle::ConstantTimeEq; +use uuid::Uuid; + +use crate::credentials::constants::{TOKEN_SECRET_BYTES, TOKEN_SECRET_HEX_LENGTH}; +use crate::store::errors::ControlError; + +pub(crate) fn new_bearer(id: Uuid) -> Result { + let mut random = [0_u8; TOKEN_SECRET_BYTES]; + getrandom::fill(&mut random) + .map_err(|_| ControlError::Invalid("could not generate a secure token".into()))?; + let mut bearer = format!("{id}."); + for byte in random { + write!(bearer, "{byte:02x}") + .map_err(|_| ControlError::Invalid("could not format a secure token".into()))?; + } + Ok(bearer) +} + +pub(crate) fn bearer_id(bearer: &str) -> Option { + let (id, secret) = bearer.split_once('.')?; + if secret.len() != TOKEN_SECRET_HEX_LENGTH + || !secret.bytes().all(|value| value.is_ascii_hexdigit()) + { + return None; + } + let Ok(id) = Uuid::parse_str(id) else { + return None; + }; + Some(id) +} + +pub(crate) fn secret_matches(secret: &str, expected_hash: &str) -> bool { + bool::from( + sha256_hex(secret.as_bytes()) + .as_bytes() + .ct_eq(expected_hash.as_bytes()), + ) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/tokens.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/tokens.rs new file mode 100644 index 0000000000..181a03aace --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/tokens.rs @@ -0,0 +1,92 @@ +//! Token mutations compose within the caller's single audited write transaction. + +use chrono::{DateTime, SubsecRound, Utc}; +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::credentials::_helpers::scopes::{scope_is_active, scope_parameters}; +use crate::credentials::_helpers::secrets::new_bearer; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{AuditContext, CredentialScope, IssuedToken, TokenIdentity}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{Change, ControlTransaction, Parameter}; + +pub(crate) async fn issue( + transaction: ControlTransaction, + context: &AuditContext, + scope: CredentialScope, + expires_at: DateTime, +) -> Result<(ControlTransaction, IssuedToken), ControlError> { + let expires_at = expires_at.trunc_subsecs(6); + if expires_at <= transaction.now { + return Err(ControlError::Invalid( + "token expiry must be in the future".into(), + )); + } + let (transaction, active) = scope_is_active(transaction, scope).await?; + if !active { + return Err(ControlError::Invalid("token scope is not active".into())); + } + let token_id = Uuid::now_v7(); + let bearer = new_bearer(token_id)?; + let now = transaction.now; + let mut parameters = vec![ + Parameter::Id(token_id), + Parameter::Text(sha256_hex(bearer.as_bytes())), + ]; + parameters.extend(scope_parameters(scope)); + parameters.extend([ + Parameter::Json(json!(scope)), + Parameter::Time(now), + Parameter::Time(expires_at), + ]); + let (transaction, _) = query(transaction, + "INSERT INTO sqb_access_tokens (token_id, secret_hash, location_id, run_id, claim_id, scope_json, created_at, expires_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8)", parameters).await?; + let identity = TokenIdentity { + token_id, + scope, + expires_at, + }; + let transaction = append_audit( + transaction, + context, + (Uuid::new_v5(&token_id, b"issued"), "token.issued"), + (json!(scope), json!(null), json!(identity)), + ) + .await?; + Ok((transaction, IssuedToken { identity, bearer })) +} + +pub(crate) async fn revoke( + transaction: ControlTransaction, + context: &AuditContext, + token_id: Uuid, +) -> Result<(ControlTransaction, bool), ControlError> { + let now = transaction.now; + let (transaction, rows) = query(transaction, + "UPDATE sqb_access_tokens SET revoked_at=$1 WHERE token_id=$2 AND revoked_at IS NULL RETURNING scope_json", + vec![Parameter::Time(now), Parameter::Id(token_id)]).await?; + let Some(row) = rows.into_iter().next() else { + return Ok((transaction, false)); + }; + let scope = row + .get("scope_json") + .cloned() + .ok_or_else(|| ControlError::Invalid("stored token scope is missing".into()))?; + let mut transaction = append_audit( + transaction, + context, + (Uuid::new_v5(&token_id, b"revoked"), "token.revoked"), + ( + scope, + json!({"token_id": token_id, "revoked": false}), + json!({"token_id": token_id, "revoked": true}), + ), + ) + .await?; + transaction.changes.insert(Change::Credentials); + Ok((transaction, true)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/constants.rs b/crates/sqlbuild-orchestration/src/credentials/constants.rs new file mode 100644 index 0000000000..a47221ed7d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/constants.rs @@ -0,0 +1,20 @@ +//! Credential secrets use 256 bits of entropy, represented as hexadecimal in bearer strings. + +pub(crate) const TOKEN_SECRET_BYTES: usize = 32; +pub(crate) const TOKEN_SECRET_HEX_LENGTH: usize = TOKEN_SECRET_BYTES * 2; + +pub(crate) const MIN_PASSWORD_CHARACTERS: usize = 12; +pub(crate) const MAX_PASSWORD_BYTES: usize = 1024; +pub(crate) const PASSWORD_ALGORITHM: &str = "argon2id"; +pub(crate) const PASSWORD_KDF_SLOTS: usize = 4; +pub(crate) const PASSWORD_SESSION_SECONDS: i64 = 3600; +pub(crate) const OIDC_FLOW_SECONDS: i64 = 300; +pub(crate) const LOGIN_WINDOW_SECONDS: i64 = 60; +pub(crate) const LOGIN_USER_LIMIT: u64 = 5; +pub(crate) const LOGIN_PEER_LIMIT: u64 = 20; +pub(crate) const LOGIN_USERNAME_BYTES: usize = 256; +/// A noncredential PHC output makes missing-user checks perform the same Argon2id work. +pub(crate) const DUMMY_PASSWORD_HASH: &str = "$argon2id$v=19$m=19456,t=2,p=1$AAAAAAAAAAAAAAAAAAAAAA$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + +pub(crate) const DEFAULT_AUDIT_RETENTION_DAYS: u16 = 400; +pub(crate) const AUDIT_RETENTION_BATCH: usize = 100; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/audit/append_audit.rs b/crates/sqlbuild-orchestration/src/credentials/main/audit/append_audit.rs new file mode 100644 index 0000000000..41bcc482b7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/audit/append_audit.rs @@ -0,0 +1,42 @@ +//! Security changes and their audit records commit together, without bearer material. + +use serde_json::Value; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn append_audit( + transaction: ControlTransaction, + context: &AuditContext, + (event_id, action): (Uuid, &str), + (scope, before, after): (Value, Value, Value), +) -> Result { + if context.actor.trim().is_empty() || context.granting_role.trim().is_empty() { + return Err(ControlError::Invalid( + "audit actor and granting role are required".into(), + )); + } + let (transaction, rows) = query( + transaction, + "SELECT COALESCE(MAX(storage_id),0) AS last_storage_id FROM sqb_security_audit", + vec![], + ) + .await?; + let storage_id = rows + .first() + .and_then(|row| row["last_storage_id"].as_i64()) + .and_then(|last| last.checked_add(1)) + .ok_or_else(|| ControlError::Invalid("audit storage order is exhausted".into()))?; + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_security_audit (event_id, occurred_at, actor, action, granting_role, request_id, ip_address, scope_json, before_json, after_json, storage_id) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)", + vec![Parameter::Id(event_id), Parameter::Time(now), Parameter::Text(context.actor.clone()), + Parameter::Text(action.into()), Parameter::Text(context.granting_role.clone()), Parameter::Id(context.request_id), + Parameter::OptionalText(context.ip_address.map(|value| value.to_string())), + Parameter::Json(scope), Parameter::Json(before), Parameter::Json(after), Parameter::Integer(storage_id)]).await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/audit/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/audit/mod.rs new file mode 100644 index 0000000000..b3e78f125f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/audit/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod append_audit; +pub(crate) mod prune_audit; +pub mod reserve_authentication_attempt; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/audit/prune_audit.rs b/crates/sqlbuild-orchestration/src/credentials/main/audit/prune_audit.rs new file mode 100644 index 0000000000..25f83d590a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/audit/prune_audit.rs @@ -0,0 +1,39 @@ +//! Audit expiry is bounded cleanup that never outruns a sink, a retry lookup or the storage order. + +use crate::credentials::_helpers::audit_retention::prune_audit_batch; +use crate::credentials::models::AuditRetention; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// One full pass in storage order; returns rows deleted and rows still kept past the horizon. +pub(crate) async fn prune_audit( + store: &ControlStore, + retention: AuditRetention, + sinks: &[String], +) -> Result<(usize, i64), ControlError> { + let mut deleted = 0; + let mut after = 0; + loop { + let (count, next) = prune_audit_batch(store, (retention, sinks), after).await?; + deleted += count; + match next { + Some(next) => after = next, + None => break, + } + } + let transaction = begin(store, TransactionMode::Read).await?; + let cutoff = retention.cutoff(transaction.now); + let (transaction, rows) = query( + transaction, + "SELECT COUNT(*) AS retained FROM sqb_security_audit WHERE occurred_at<$1", + vec![Parameter::Time(cutoff)], + ) + .await?; + commit(transaction).await?; + let retained = rows + .first() + .and_then(|row| row["retained"].as_i64()) + .unwrap_or_default(); + Ok((deleted, retained)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/audit/reserve_authentication_attempt.rs b/crates/sqlbuild-orchestration/src/credentials/main/audit/reserve_authentication_attempt.rs new file mode 100644 index 0000000000..5c877e7e16 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/audit/reserve_authentication_attempt.rs @@ -0,0 +1,55 @@ +//! Serialized reservations bound authentication work across API replicas. + +use chrono::Duration; +use uuid::Uuid; + +use crate::credentials::constants::{LOGIN_PEER_LIMIT, LOGIN_USER_LIMIT, LOGIN_WINDOW_SECONDS}; +use crate::credentials::models::{AuditContext, AuthenticationAttempt}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn reserve_authentication_attempt( + store: &ControlStore, + context: &AuditContext, + attempt: AuthenticationAttempt<'_>, +) -> Result<(Uuid, bool), ControlError> { + let peer = context + .ip_address + .ok_or_else(|| ControlError::Invalid("login requires a trusted peer address".into()))? + .to_canonical() + .to_string(); + let (identity_key, identity_limit) = match attempt { + AuthenticationAttempt::Login(username) => (format!("login:{username}"), LOGIN_USER_LIMIT), + AuthenticationAttempt::Reauthenticate(id) => { + (format!("reauthenticate:{id}"), LOGIN_USER_LIMIT) + } + AuthenticationAttempt::IdentityProvider => { + (format!("identity-provider:{peer}"), LOGIN_PEER_LIMIT) + } + }; + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, _) = query(transaction, + "DELETE FROM sqb_password_login_attempts WHERE attempt_id IN \ + (SELECT attempt_id FROM sqb_password_login_attempts WHERE attempted_at<$1 ORDER BY attempted_at LIMIT 1000)", + vec![Parameter::Time(now - Duration::hours(1))]).await?; + let (mut transaction, rows) = query(transaction, + "SELECT (SELECT COUNT(*) FROM sqb_password_login_attempts WHERE identity_key=$1 AND attempted_at>$3) AS user_count, \ + (SELECT COUNT(*) FROM sqb_password_login_attempts WHERE peer_address=$2 AND attempted_at>$3) AS peer_count", + vec![Parameter::Text(identity_key.clone()), Parameter::Text(peer.clone()), Parameter::Time(now - Duration::seconds(LOGIN_WINDOW_SECONDS))]).await?; + let allowed = rows[0]["user_count"] + .as_u64() + .is_some_and(|count| count < identity_limit) + && rows[0]["peer_count"] + .as_u64() + .is_some_and(|count| count < LOGIN_PEER_LIMIT); + let id = Uuid::now_v7(); + if allowed { + (transaction, _) = query(transaction, + "INSERT INTO sqb_password_login_attempts (attempt_id, identity_key, peer_address, attempted_at) VALUES ($1,$2,$3,$4)", + vec![Parameter::Id(id), Parameter::Text(identity_key.clone()), Parameter::Text(peer), Parameter::Time(now)]).await?; + } + commit(transaction).await?; + Ok((id, allowed)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/cli_login/approve_login.rs b/crates/sqlbuild-orchestration/src/credentials/main/cli_login/approve_login.rs new file mode 100644 index 0000000000..3356e7e57d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/cli_login/approve_login.rs @@ -0,0 +1,65 @@ +//! An authenticated browser person approves or rejects one pending CLI login. + +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::credentials::_helpers::secrets::bearer_id; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{AuditContext, AuthenticatedPerson}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn approve_login( + store: &ControlStore, + context: &AuditContext, + AuthenticatedPerson(principal_id, session_id): AuthenticatedPerson, + (browser_code, approve): (&str, bool), +) -> Result { + let Some(id) = bearer_id(browser_code) else { + return Ok(false); + }; + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT flow_id FROM sqb_cli_login_flows WHERE flow_id=$1 AND browser_hash=$2 AND expires_at>$3 AND principal_id IS NULL", + vec![Parameter::Id(id), Parameter::Text(sha256_hex(browser_code.as_bytes())), Parameter::Time(now)]).await?; + if rows.is_empty() { + return Ok(false); + } + let (transaction, _) = if approve { + query( + transaction, + "UPDATE sqb_cli_login_flows SET principal_id=$2,session_id=$3 WHERE flow_id=$1", + vec![ + Parameter::Id(id), + Parameter::Id(principal_id), + Parameter::OptionalId(session_id), + ], + ) + .await? + } else { + query( + transaction, + "DELETE FROM sqb_cli_login_flows WHERE flow_id=$1", + vec![Parameter::Id(id)], + ) + .await? + }; + commit( + append_audit( + transaction, + context, + (Uuid::new_v5(&id, b"decision"), "cli.login.decided"), + ( + json!({"flow_id":id,"principal_id":principal_id}), + json!(null), + json!({"approved":approve}), + ), + ) + .await?, + ) + .await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/cli_login/claim_login.rs b/crates/sqlbuild-orchestration/src/credentials/main/cli_login/claim_login.rs new file mode 100644 index 0000000000..854d27a71e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/cli_login/claim_login.rs @@ -0,0 +1,80 @@ +//! Exchange consumes approval and issues a bounded session while the approver stays live. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::_helpers::{ + active_session::active_session, browser_sessions::issue_session, secrets::secret_matches, +}; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{AuditContext, ClaimedLogin, StoredLogin}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn claim_login( + store: &ControlStore, + context: &AuditContext, + (id, client_secret): (Uuid, &str), +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query( + transaction, + "SELECT client_hash,principal_id,session_id FROM sqb_cli_login_flows WHERE flow_id=$1 AND expires_at>$2", + vec![Parameter::Id(id), Parameter::Time(now)], + ) + .await?; + let Some(row) = rows.into_iter().next() else { + return Ok(ClaimedLogin::Unavailable); + }; + let flow: StoredLogin = decode_row(row)?; + if !secret_matches(client_secret, &flow.client_hash) { + return Ok(ClaimedLogin::Unavailable); + } + let Some(principal_id) = flow.principal_id else { + return Ok(ClaimedLogin::Pending); + }; + let (transaction, approver) = match flow.session_id { + Some(session_id) => active_session(transaction, session_id).await?, + None => (transaction, None), + }; + if flow.session_id.is_some() && approver.is_none() { + return Ok(ClaimedLogin::Unavailable); + } + let (transaction, people) = query( + transaction, + "SELECT c.credential_id FROM sqb_principals p LEFT JOIN sqb_password_credentials c \ + ON c.principal_id=p.principal_id WHERE p.principal_id=$1 AND p.disabled_at IS NULL", + vec![Parameter::Id(principal_id)], + ) + .await?; + let Some(person) = people.first() else { + return Ok(ClaimedLogin::Unavailable); + }; + let credential_id = serde_json::from_value::>(person["credential_id"].clone()) + .map_err(|error| ControlError::Schema(error.to_string()))?; + let (transaction, _) = query( + transaction, + "DELETE FROM sqb_cli_login_flows WHERE flow_id=$1", + vec![Parameter::Id(id)], + ) + .await?; + let (transaction, session) = + issue_session(transaction, context, (principal_id, credential_id)).await?; + commit( + append_audit( + transaction, + context, + (Uuid::new_v5(&id, b"exchanged"), "cli.login.exchanged"), + ( + json!({"flow_id":id,"principal_id":session.identity.principal_id}), + json!(null), + json!({"session_id":session.identity.session_id}), + ), + ) + .await?, + ) + .await?; + Ok(ClaimedLogin::Authenticated(session)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/cli_login/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/cli_login/mod.rs new file mode 100644 index 0000000000..e46edbe52f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/cli_login/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod approve_login; +pub(crate) mod claim_login; +pub(crate) mod start_login; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/cli_login/start_login.rs b/crates/sqlbuild-orchestration/src/credentials/main/cli_login/start_login.rs new file mode 100644 index 0000000000..77cb358898 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/cli_login/start_login.rs @@ -0,0 +1,57 @@ +//! Only a digest of the CLI-held exchange secret is sent when starting a login. + +use chrono::{Duration, SubsecRound}; +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::credentials::_helpers::secrets::new_bearer; +use crate::credentials::constants::{OIDC_FLOW_SECONDS, TOKEN_SECRET_HEX_LENGTH}; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{AuditContext, StartedLogin}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn start_login( + store: &ControlStore, + context: &AuditContext, + client_hash: &str, +) -> Result { + if client_hash.len() != TOKEN_SECRET_HEX_LENGTH + || !client_hash.bytes().all(|byte| byte.is_ascii_hexdigit()) + { + return Err(ControlError::Invalid("invalid CLI exchange digest".into())); + } + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let id = Uuid::now_v7(); + let browser_code = new_bearer(id)?; + let expires_at = (now + Duration::seconds(OIDC_FLOW_SECONDS)).trunc_subsecs(6); + let (transaction, _) = query(transaction, + "DELETE FROM sqb_cli_login_flows WHERE flow_id IN \ + (SELECT flow_id FROM sqb_cli_login_flows WHERE expires_at<=$1 ORDER BY expires_at LIMIT 1000)", + vec![Parameter::Time(now)]).await?; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_cli_login_flows (flow_id,client_hash,browser_hash,expires_at) VALUES ($1,$2,$3,$4)", + vec![Parameter::Id(id), Parameter::Text(client_hash.into()), + Parameter::Text(sha256_hex(browser_code.as_bytes())), Parameter::Time(expires_at)]).await?; + commit( + append_audit( + transaction, + context, + (id, "cli.login.started"), + ( + json!({"flow_id":id}), + json!(null), + json!({"expires_at":expires_at}), + ), + ) + .await?, + ) + .await?; + Ok(StartedLogin { + browser_code, + expires_at, + }) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/identities/create_principal.rs b/crates/sqlbuild-orchestration/src/credentials/main/identities/create_principal.rs new file mode 100644 index 0000000000..1c69ab44ba --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/identities/create_principal.rs @@ -0,0 +1,20 @@ +//! Creating a person never implicitly assigns a role. + +use uuid::Uuid; + +use crate::credentials::main::identities::insert_principal::insert_principal; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn create_principal( + store: &ControlStore, + context: &AuditContext, + name: &str, +) -> Result { + let (transaction, id) = + insert_principal(begin(store, TransactionMode::Write).await?, context, name).await?; + commit(transaction).await?; + Ok(id) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/identities/insert_principal.rs b/crates/sqlbuild-orchestration/src/credentials/main/identities/insert_principal.rs new file mode 100644 index 0000000000..e664ebae45 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/identities/insert_principal.rs @@ -0,0 +1,44 @@ +//! Create a role-free identity; bootstrap authority is a separate, one-time operation. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn insert_principal( + transaction: ControlTransaction, + context: &AuditContext, + name: &str, +) -> Result<(ControlTransaction, Uuid), ControlError> { + if name.trim().is_empty() { + return Err(ControlError::Invalid("principal name is required".into())); + } + let id = Uuid::now_v7(); + let now = transaction.now; + let (transaction, _) = query( + transaction, + "INSERT INTO sqb_principals (principal_id, name, created_at) VALUES ($1,$2,$3)", + vec![ + Parameter::Id(id), + Parameter::Text(name.into()), + Parameter::Time(now), + ], + ) + .await?; + let transaction = append_audit( + transaction, + context, + (Uuid::new_v5(&id, b"created"), "principal.created"), + ( + json!({"principal_id": id}), + json!(null), + json!({"name": name}), + ), + ) + .await?; + Ok((transaction, id)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/identities/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/identities/mod.rs new file mode 100644 index 0000000000..54451634e7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/identities/mod.rs @@ -0,0 +1,2 @@ +pub mod create_principal; +pub(crate) mod insert_principal; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/locations/create_location.rs b/crates/sqlbuild-orchestration/src/credentials/main/locations/create_location.rs new file mode 100644 index 0000000000..39a04ea9f4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/locations/create_location.rs @@ -0,0 +1,21 @@ +//! Register a location and its security audit atomically. + +use uuid::Uuid; + +use crate::credentials::_helpers::locations::create_location_in; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +/// Names are labels, never credentials or stable identifiers. +pub async fn create_location( + store: &ControlStore, + name: &str, + context: &AuditContext, +) -> Result { + let (transaction, id) = + create_location_in(begin(store, TransactionMode::Write).await?, name, context).await?; + commit(transaction).await?; + Ok(id) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/locations/development_location.rs b/crates/sqlbuild-orchestration/src/credentials/main/locations/development_location.rs new file mode 100644 index 0000000000..4049e823a3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/locations/development_location.rs @@ -0,0 +1,53 @@ +//! A dev project/target retains its opaque location identity across process restarts. + +use uuid::Uuid; + +use crate::credentials::_helpers::locations::create_location_in; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn development_location( + store: &ControlStore, + project_path: &str, + target: &str, + context: &AuditContext, +) -> Result { + let (transaction, rows) = query( + begin(store, TransactionMode::Write).await?, + "SELECT location_id FROM sqb_development_locations WHERE project_path=$1 AND target=$2", + vec![ + Parameter::Text(project_path.into()), + Parameter::Text(target.into()), + ], + ) + .await?; + if let Some(row) = rows.first() { + let id = row["location_id"] + .as_str() + .unwrap_or_default() + .parse() + .map_err(|_| ControlError::Schema("invalid development location identity".into()))?; + commit(transaction).await?; + return Ok(id); + } + let (transaction, id) = create_location_in( + transaction, + &format!("development-{}", Uuid::now_v7()), + context, + ) + .await?; + let (transaction, _) = query( + transaction, + "INSERT INTO sqb_development_locations (project_path,target,location_id) VALUES ($1,$2,$3)", + vec![ + Parameter::Text(project_path.into()), + Parameter::Text(target.into()), + Parameter::Id(id), + ], + ) + .await?; + commit(transaction).await?; + Ok(id) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/locations/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/locations/mod.rs new file mode 100644 index 0000000000..23453c831d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/locations/mod.rs @@ -0,0 +1,2 @@ +pub mod create_location; +pub mod development_location; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs new file mode 100644 index 0000000000..06dec347c3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs @@ -0,0 +1,6 @@ +pub mod audit; +pub(crate) mod cli_login; +pub mod identities; +pub mod locations; +pub mod sessions; +pub mod tokens; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/sessions/authenticate_bound_person.rs b/crates/sqlbuild-orchestration/src/credentials/main/sessions/authenticate_bound_person.rs new file mode 100644 index 0000000000..e3fa4cfdce --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/sessions/authenticate_bound_person.rs @@ -0,0 +1,47 @@ +//! Explicit OIDC bindings issue browser sessions without provisioning accounts or roles. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::_helpers::browser_sessions::issue_session; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{AuditContext, IssuedSession}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn authenticate_bound_person( + store: &ControlStore, + context: &AuditContext, + principal_id: Uuid, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, rows) = query( + transaction, + "SELECT principal_id FROM sqb_principals WHERE principal_id=$1 AND disabled_at IS NULL", + vec![Parameter::Id(principal_id)], + ) + .await?; + if rows.is_empty() { + commit(transaction).await?; + return Ok(None); + } + let context = AuditContext { + actor: format!("principal:{principal_id}"), + granting_role: "oidc".into(), + ..context.clone() + }; + let (transaction, session) = issue_session(transaction, &context, (principal_id, None)).await?; + let identity = json!({"principal_id": principal_id}); + commit( + append_audit( + transaction, + &context, + (context.request_id, "oidc.authenticated"), + (identity.clone(), json!(null), identity), + ) + .await?, + ) + .await?; + Ok(Some(session)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/sessions/authenticate_session.rs b/crates/sqlbuild-orchestration/src/credentials/main/sessions/authenticate_session.rs new file mode 100644 index 0000000000..b3990000f2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/sessions/authenticate_session.rs @@ -0,0 +1,29 @@ +//! Live person eligibility, password versions and revocation apply to every browser request. + +use crate::credentials::_helpers::active_session::active_session; +use crate::credentials::_helpers::secrets::{bearer_id, secret_matches}; +use crate::credentials::models::{SessionAccess, SessionIdentity}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn authenticate_session( + store: &ControlStore, + bearer: &str, + access: SessionAccess<'_>, +) -> Result, ControlError> { + let Some(id) = bearer_id(bearer) else { + return Ok(None); + }; + let (transaction, session) = + active_session(begin(store, TransactionMode::Read).await?, id).await?; + commit(transaction).await?; + let Some(session) = session else { + return Ok(None); + }; + let csrf_matches = match access { + SessionAccess::Read => true, + SessionAccess::Write { csrf } => secret_matches(csrf, &session.csrf_hash), + }; + Ok((secret_matches(bearer, &session.secret_hash) && csrf_matches).then_some(session.identity)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/sessions/claim_oidc_flow.rs b/crates/sqlbuild-orchestration/src/credentials/main/sessions/claim_oidc_flow.rs new file mode 100644 index 0000000000..4420e480e8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/sessions/claim_oidc_flow.rs @@ -0,0 +1,66 @@ +//! Atomically consume browser-bound flow state before any identity-provider exchange. + +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::credentials::_helpers::secrets::secret_matches; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{AuditContext, StoredOidcFlow}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn claim_oidc_flow( + store: &ControlStore, + context: &AuditContext, + configuration_id: &str, + (state, browser): (&str, &str), +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query( + transaction, + "SELECT flow_id, browser_hash, nonce, pkce_verifier FROM sqb_oidc_login_flows \ + WHERE state_hash=$1 AND configuration_id=$2 AND expires_at>$3", + vec![ + Parameter::Text(sha256_hex(state.as_bytes())), + Parameter::Text(configuration_id.into()), + Parameter::Time(now), + ], + ) + .await?; + let Some(row) = rows.into_iter().next() else { + commit(transaction).await?; + return Ok(None); + }; + let flow: StoredOidcFlow = decode_row(row)?; + if !secret_matches(browser, &flow.browser_hash) { + commit(transaction).await?; + return Ok(None); + } + let (transaction, _) = query( + transaction, + "DELETE FROM sqb_oidc_login_flows WHERE flow_id=$1", + vec![Parameter::Id(flow.flow_id)], + ) + .await?; + commit( + append_audit( + transaction, + context, + ( + Uuid::new_v5(&flow.flow_id, b"consumed"), + "oidc.flow.consumed", + ), + ( + json!({"flow_id": flow.flow_id}), + json!({"pending": true}), + json!({"pending": false}), + ), + ) + .await?, + ) + .await?; + Ok(Some(flow)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/sessions/create_oidc_flow.rs b/crates/sqlbuild-orchestration/src/credentials/main/sessions/create_oidc_flow.rs new file mode 100644 index 0000000000..c001b9ec53 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/sessions/create_oidc_flow.rs @@ -0,0 +1,47 @@ +//! Create one-use browser-bound PKCE attempts with durable audit evidence. + +use crate::credentials::constants::OIDC_FLOW_SECONDS; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{AuditContext, OidcFlowStart}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use chrono::Duration; +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +pub(crate) async fn create_oidc_flow( + store: &ControlStore, + context: &AuditContext, + configuration_id: &str, + flow: &OidcFlowStart, +) -> Result<(), ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, _) = query(transaction, + "DELETE FROM sqb_oidc_login_flows WHERE flow_id IN \ + (SELECT flow_id FROM sqb_oidc_login_flows WHERE expires_at<=$1 ORDER BY expires_at LIMIT 1000)", + vec![Parameter::Time(now)]).await?; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_oidc_login_flows (flow_id, configuration_id, state_hash, browser_hash, nonce, pkce_verifier, expires_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7)", + vec![Parameter::Id(context.request_id), Parameter::Text(configuration_id.into()), + Parameter::Text(sha256_hex(flow.state.secret().as_bytes())), + Parameter::Text(sha256_hex(flow.browser.secret().as_bytes())), + Parameter::Text(flow.nonce.secret().clone()), Parameter::Text(flow.pkce_verifier.secret().clone()), + Parameter::Time(now + Duration::seconds(OIDC_FLOW_SECONDS))]).await?; + commit( + append_audit( + transaction, + context, + (context.request_id, "oidc.flow.created"), + ( + json!({"flow_id": context.request_id}), + json!(null), + json!({"pending": true}), + ), + ) + .await?, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/sessions/login_password.rs b/crates/sqlbuild-orchestration/src/credentials/main/sessions/login_password.rs new file mode 100644 index 0000000000..b8b50163cb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/sessions/login_password.rs @@ -0,0 +1,49 @@ +//! Password login reserves shared throttle capacity before memory-hard verification. + +use uuid::Uuid; + +use crate::credentials::_helpers::password_sessions::issue_session; +use crate::credentials::constants::{LOGIN_USERNAME_BYTES, MAX_PASSWORD_BYTES}; +use crate::credentials::main::audit::reserve_authentication_attempt::reserve_authentication_attempt; +use crate::credentials::main::sessions::verify_password::verify_password; +use crate::credentials::models::{AuditContext, AuthenticationAttempt, PasswordLogin}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn login_password( + store: &ControlStore, + context: &AuditContext, + (username, password): (&str, &str), +) -> Result { + let bounded = username.len() <= LOGIN_USERNAME_BYTES && password.len() <= MAX_PASSWORD_BYTES; + let username = if bounded { username } else { "" }; + let (_, allowed) = + reserve_authentication_attempt(store, context, AuthenticationAttempt::Login(username)) + .await?; + if !allowed { + return Ok(PasswordLogin::Denied { throttled: true }); + } + if bounded { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT principal_id FROM sqb_principals WHERE name=$1", + vec![Parameter::Text(username.into())], + ) + .await?; + commit(transaction).await?; + let principal_id = rows + .first() + .map(|row| serde_json::from_value(row["principal_id"].clone())) + .transpose() + .map_err(|_| ControlError::Invalid("stored person identity is invalid".into()))? + .unwrap_or(Uuid::nil()); + if let Some(proof) = verify_password(store, principal_id, password).await? + && let Some(session) = issue_session(store, context, proof).await? + { + return Ok(PasswordLogin::Authenticated(session)); + } + } + Ok(PasswordLogin::Denied { throttled: false }) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/sessions/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/sessions/mod.rs new file mode 100644 index 0000000000..5826bb2fdb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/sessions/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod authenticate_bound_person; +pub mod authenticate_session; +pub(crate) mod claim_oidc_flow; +pub(crate) mod create_oidc_flow; +pub mod login_password; +pub(crate) mod revoke_session; +pub mod set_password; +pub mod verify_password; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/sessions/revoke_session.rs b/crates/sqlbuild-orchestration/src/credentials/main/sessions/revoke_session.rs new file mode 100644 index 0000000000..31ee9de52f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/sessions/revoke_session.rs @@ -0,0 +1,52 @@ +//! Revoke one session or every browser session without releasing the caller's write lock. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{Change, ControlTransaction, Parameter}; + +/// A missing session ID revokes all mechanisms; the caller audits its password/reset operation. +pub(crate) async fn revoke_session( + transaction: ControlTransaction, + context: &AuditContext, + (principal_id, session_id): (Uuid, Option), +) -> Result<(ControlTransaction, usize), ControlError> { + let now = transaction.now; + let mut parameters = vec![Parameter::Time(now), Parameter::Id(principal_id)]; + let predicate = match session_id { + Some(id) => { + parameters.push(Parameter::Id(id)); + "AND session_id=$3" + } + None => "", + }; + let (mut transaction, rows) = query( + transaction, + &format!("UPDATE sqb_sessions SET revoked_at=$1 WHERE principal_id=$2 {predicate} AND revoked_at IS NULL RETURNING session_id"), + parameters, + ) + .await?; + let revoked = rows.len(); + if revoked == 0 { + return Ok((transaction, 0)); + } + if let Some(id) = session_id { + transaction = append_audit( + transaction, + context, + (Uuid::new_v5(&id, b"revoked"), "session.revoked"), + ( + json!({"principal_id":principal_id}), + json!({"session_id":id,"revoked":false}), + json!({"session_id":id,"revoked":true}), + ), + ) + .await?; + } + transaction.changes.insert(Change::Credentials); + Ok((transaction, revoked)) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/sessions/set_password.rs b/crates/sqlbuild-orchestration/src/credentials/main/sessions/set_password.rs new file mode 100644 index 0000000000..28cc5461ba --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/sessions/set_password.rs @@ -0,0 +1,98 @@ +//! Only active people have passwords; changing one rotates its identity and audits atomically. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::_helpers::passwords::hash_password; +use crate::credentials::constants::{MAX_PASSWORD_BYTES, MIN_PASSWORD_CHARACTERS}; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::main::sessions::revoke_session::revoke_session; +use crate::credentials::models::{AuditContext, PasswordAuthority}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{Change, ControlStore, Parameter, TransactionMode}; + +/// The trusted caller authorizes password management before invoking this storage operation. +pub async fn set_password( + store: &ControlStore, + context: &AuditContext, + authority: PasswordAuthority, + password: &str, +) -> Result<(), ControlError> { + if password.len() > MAX_PASSWORD_BYTES || password.chars().count() < MIN_PASSWORD_CHARACTERS { + return Err(ControlError::Invalid(format!( + "password must contain at least {MIN_PASSWORD_CHARACTERS} characters and at most {MAX_PASSWORD_BYTES} UTF-8 bytes" + ))); + } + let (principal_id, expected) = match authority { + PasswordAuthority::Operator(id) => (id, None), + PasswordAuthority::Reauthenticated { proof, session_id } => { + (proof.principal_id, Some((proof.credential_id, session_id))) + } + }; + let hash = hash_password(password.into()).await?; + let transaction = begin(store, TransactionMode::Write).await?; + let (mut transaction, rows) = query( + transaction, + "SELECT c.credential_id FROM sqb_principals p \ + LEFT JOIN sqb_password_credentials c ON c.principal_id=p.principal_id \ + WHERE p.principal_id=$1 AND p.disabled_at IS NULL", + vec![Parameter::Id(principal_id)], + ) + .await?; + if rows.is_empty() { + return Err(ControlError::Invalid( + "password requires an active person".into(), + )); + } + if let Some((id, session_id)) = expected { + let now = transaction.now; + let sessions; + (transaction, sessions) = query( + transaction, + "SELECT session_id FROM sqb_sessions WHERE session_id=$1 AND principal_id=$2 \ + AND credential_id=$3 AND revoked_at IS NULL AND expires_at>$4", + vec![ + Parameter::Id(session_id), + Parameter::Id(principal_id), + Parameter::Id(id), + Parameter::Time(now), + ], + ) + .await?; + if rows[0]["credential_id"] != json!(id) || sessions.is_empty() { + return Err(ControlError::Invalid( + "password credential or session changed; authenticate again".into(), + )); + } + } + let before = if rows[0]["credential_id"].is_null() { + json!(null) + } else { + json!({"credential_id": rows[0]["credential_id"]}) + }; + let credential_id = Uuid::now_v7(); + let now = transaction.now; + let (transaction, _) = query( + transaction, + "INSERT INTO sqb_password_credentials (principal_id, credential_id, password_hash, changed_at) \ + VALUES ($1,$2,$3,$4) ON CONFLICT (principal_id) DO UPDATE \ + SET credential_id=excluded.credential_id, password_hash=excluded.password_hash, changed_at=excluded.changed_at", + vec![Parameter::Id(principal_id), Parameter::Id(credential_id), Parameter::Text(hash), Parameter::Time(now)], + ) + .await?; + let (transaction, _) = revoke_session(transaction, context, (principal_id, None)).await?; + let mut transaction = append_audit( + transaction, + context, + (credential_id, "password.changed"), + ( + json!({"principal_id": principal_id}), + before, + json!({"credential_id": credential_id}), + ), + ) + .await?; + transaction.changes.insert(Change::Credentials); + commit(transaction).await +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/sessions/verify_password.rs b/crates/sqlbuild-orchestration/src/credentials/main/sessions/verify_password.rs new file mode 100644 index 0000000000..b13083274c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/sessions/verify_password.rs @@ -0,0 +1,51 @@ +//! A password proves one credential version; it never directly grants a role or an API token. + +use uuid::Uuid; + +use crate::credentials::_helpers::passwords::verify_hash; +use crate::credentials::constants::{DUMMY_PASSWORD_HASH, MAX_PASSWORD_BYTES}; +use crate::credentials::models::VerifiedPassword; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Network callers must apply their shared login throttle before invoking this primitive. +pub async fn verify_password( + store: &ControlStore, + principal_id: Uuid, + password: &str, +) -> Result, ControlError> { + if password.len() > MAX_PASSWORD_BYTES { + return Ok(None); + } + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT credential_id, password_hash FROM sqb_password_credentials c \ + JOIN sqb_principals p ON p.principal_id=c.principal_id \ + WHERE p.principal_id=$1 AND p.disabled_at IS NULL", + vec![Parameter::Id(principal_id)], + ) + .await?; + commit(transaction).await?; + let row = rows.into_iter().next(); + let (identity, hash) = if let Some(row) = row { + let credential_id = serde_json::from_value(row["credential_id"].clone()) + .map_err(|_| ControlError::Invalid("stored password identity is invalid".into()))?; + let hash = serde_json::from_value(row["password_hash"].clone()) + .map_err(|_| ControlError::Invalid("stored password hash is invalid".into()))?; + ( + Some(VerifiedPassword { + principal_id, + credential_id, + }), + hash, + ) + } else { + (None, DUMMY_PASSWORD_HASH.into()) + }; + Ok(verify_hash(hash, password.into()) + .await? + .then_some(identity) + .flatten()) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/tokens/authenticate_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/tokens/authenticate_token.rs new file mode 100644 index 0000000000..c2e8a86845 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/tokens/authenticate_token.rs @@ -0,0 +1,46 @@ +//! Every request sees revocation and current scope state, including across server replicas. + +use crate::credentials::_helpers::scopes::scope_is_active; +use crate::credentials::_helpers::secrets::{bearer_id, secret_matches}; +use crate::credentials::models::{StoredToken, TokenIdentity}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Invalid, revoked, expired and inactive-scope credentials are indistinguishable to callers. +pub async fn authenticate_token( + store: &ControlStore, + bearer: &str, +) -> Result, ControlError> { + let Some(id) = bearer_id(bearer) else { + return Ok(None); + }; + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query( + transaction, + "SELECT token_id, scope_json, secret_hash, expires_at FROM sqb_access_tokens \ + WHERE token_id=$1 AND revoked_at IS NULL AND expires_at>$2", + vec![Parameter::Id(id), Parameter::Time(now)], + ) + .await?; + let Some(row) = rows.into_iter().next() else { + commit(transaction).await?; + return Ok(None); + }; + let token: StoredToken = serde_json::from_value(serde_json::json!(row)) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if !secret_matches(bearer, &token.secret_hash) { + commit(transaction).await?; + return Ok(None); + } + let (transaction, active) = scope_is_active(transaction, token.scope_json).await?; + commit(transaction).await?; + Ok(active.then_some(TokenIdentity { + token_id: token.token_id, + scope: token.scope_json, + expires_at: token.expires_at, + })) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/tokens/issue_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/tokens/issue_token.rs new file mode 100644 index 0000000000..94840e1952 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/tokens/issue_token.rs @@ -0,0 +1,23 @@ +//! Random bearer secrets are returned once; only their hashes and bounded scopes are durable. + +use chrono::{DateTime, Utc}; + +use crate::credentials::_helpers::tokens::issue; +use crate::credentials::models::{AuditContext, CredentialScope, IssuedToken}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +/// The caller authorizes token management before entering this storage operation. +pub async fn issue_token( + store: &ControlStore, + context: &AuditContext, + scope: CredentialScope, + expires_at: DateTime, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, issued) = issue(transaction, context, scope, expires_at).await?; + commit(transaction).await?; + Ok(issued) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/tokens/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/tokens/mod.rs new file mode 100644 index 0000000000..607195c76d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/tokens/mod.rs @@ -0,0 +1,4 @@ +pub mod authenticate_token; +pub mod issue_token; +pub mod revoke_token; +pub mod rotate_token; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/tokens/revoke_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/tokens/revoke_token.rs new file mode 100644 index 0000000000..4df9aed7c4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/tokens/revoke_token.rs @@ -0,0 +1,22 @@ +//! Revocation and its security audit are one transaction, without a per-process token cache. + +use uuid::Uuid; + +use crate::credentials::_helpers::tokens::revoke; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +/// Repeating a revocation does not create a second audit entry or rewrite its actor. +pub async fn revoke_token( + store: &ControlStore, + context: &AuditContext, + token_id: Uuid, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, revoked) = revoke(transaction, context, token_id).await?; + commit(transaction).await?; + Ok(revoked) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/tokens/rotate_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/tokens/rotate_token.rs new file mode 100644 index 0000000000..a458a9d5ce --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/tokens/rotate_token.rs @@ -0,0 +1,48 @@ +//! Replace one live credential without widening its scope or extending its lifetime. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::_helpers::tokens::{issue, revoke}; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{AuditContext, IssuedToken, TokenIdentity}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The trusted caller authorizes token management; the old UUID is not an authentication proof. +pub async fn rotate_token( + store: &ControlStore, + context: &AuditContext, + token_id: Uuid, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query( + transaction, + "SELECT token_id, scope_json AS scope, expires_at FROM sqb_access_tokens \ + WHERE token_id=$1 AND revoked_at IS NULL AND expires_at>$2", + vec![Parameter::Id(token_id), Parameter::Time(now)], + ) + .await?; + let before: TokenIdentity = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("token is not active".into()))?, + )?; + let (transaction, _) = revoke(transaction, context, token_id).await?; + let (transaction, issued) = + issue(transaction, context, before.scope, before.expires_at).await?; + let transaction = append_audit( + transaction, + context, + ( + Uuid::new_v5(&issued.identity.token_id, b"rotated"), + "token.rotated", + ), + (json!(before.scope), json!(before), json!(issued.identity)), + ) + .await?; + commit(transaction).await?; + Ok(issued) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/mod.rs b/crates/sqlbuild-orchestration/src/credentials/mod.rs new file mode 100644 index 0000000000..acde203166 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/mod.rs @@ -0,0 +1,10 @@ +//! Hashed, revocable credentials scoped to one principal, location, or claimed run. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +#[path = "../../tests/credentials/test_contracts.rs"] +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/credentials/models.rs b/crates/sqlbuild-orchestration/src/credentials/models.rs new file mode 100644 index 0000000000..442b6bf16c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/models.rs @@ -0,0 +1,320 @@ +//! Credentials identify a caller; principal permissions are a separate authorization decision. + +use std::net::IpAddr; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::credentials::constants::DEFAULT_AUDIT_RETENTION_DAYS; +use crate::store::errors::ControlError; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum CredentialScope { + Location { location_id: Uuid }, + Run { run_id: Uuid, claim_id: Uuid }, +} + +/// Server-derived request metadata, never copied from an action's JSON body. +#[derive(Clone, Debug)] +pub struct AuditContext { + pub actor: String, + pub granting_role: String, + pub request_id: Uuid, + pub ip_address: Option, +} + +/// Durable security facts with an exclusive storage-order cursor. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct SecurityAuditEvent { + pub storage_id: i64, + pub event_id: Uuid, + pub occurred_at: DateTime, + pub actor: String, + pub action: String, + pub granting_role: String, + pub request_id: Uuid, + pub ip_address: Option, + pub scope_json: serde_json::Value, + pub before_json: serde_json::Value, + pub after_json: serde_json::Value, +} + +/// Installation configuration: rows past the horizon stay until every sink and retry lookup is done. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AuditRetention { + days: u16, +} + +impl AuditRetention { + pub fn days(days: u16) -> Result { + if days == 0 { + return Err(ControlError::Invalid( + "audit retention must keep at least one day".into(), + )); + } + Ok(Self { days }) + } + + pub(crate) fn cutoff(self, now: DateTime) -> DateTime { + now - chrono::Duration::days(i64::from(self.days)) + } +} + +impl Default for AuditRetention { + fn default() -> Self { + Self { + days: DEFAULT_AUDIT_RETENTION_DAYS, + } + } +} + +/// Closed permission vocabulary; unknown actions fail deserialization. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub enum Action { + #[serde(rename = "view")] + View, + #[serde(rename = "view.sql")] + ViewSql, + #[serde(rename = "view.evidence")] + ViewEvidence, + #[serde(rename = "job.run")] + JobRun, + #[serde(rename = "run.cancel")] + RunCancel, + #[serde(rename = "run.rerun")] + RunRerun, + #[serde(rename = "run.rewrite_history")] + RunRewriteHistory, + #[serde(rename = "trigger.toggle")] + TriggerToggle, + #[serde(rename = "sensor.reset_cursor")] + SensorResetCursor, + #[serde(rename = "run.adhoc")] + RunAdhoc, + #[serde(rename = "connection.test")] + ConnectionTest, + #[serde(rename = "user.manage")] + UserManage, + #[serde(rename = "role.assign")] + RoleAssign, + #[serde(rename = "audit.view")] + AuditView, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum BuiltinRole { + Viewer, + Operator, + Developer, + Admin, +} + +/// A built-in role by name, or a custom role a location publishes. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(untagged)] +pub enum RoleKey { + Builtin(BuiltinRole), + Custom(String), +} + +impl From for RoleKey { + fn from(role: BuiltinRole) -> Self { + Self::Builtin(role) + } +} + +/// One custom role entry of a published `roles` document; only non-administrative actions. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RoleDefinition { + pub(crate) description: String, + pub(crate) actions: Vec, +} + +/// Installation-owned ambient access; never permits an administrative or execution default. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DefaultRole { + #[default] + None, + Viewer, +} + +/// Scope is computed by the server from the operation, never accepted as a claimed write set. +#[derive(Clone, Debug, Serialize)] +pub(crate) struct PermissionScope { + pub(crate) action: Action, + pub(crate) location_id: Option, + pub(crate) target: Option, + pub(crate) nodes: std::collections::BTreeSet, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct RoleAssignmentRequest { + pub principal_id: Uuid, + pub role: RoleKey, + pub location_id: Option, + pub target: Option, + #[serde(default)] + pub revoke: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TokenIdentity { + pub token_id: Uuid, + pub scope: CredentialScope, + pub expires_at: DateTime, +} + +/// Proof of one password version, not authorization or an issued session. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct VerifiedPassword { + pub principal_id: Uuid, + pub credential_id: Uuid, +} + +#[derive(Clone, Copy, Debug)] +pub enum PasswordAuthority { + Operator(Uuid), + Reauthenticated { + proof: VerifiedPassword, + session_id: Uuid, + }, +} + +/// Administrator membership and person eligibility share the final-admin invariant. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum AdministratorChange { + Grant, + Revoke, + Disable, + Enable, +} + +/// The person a request authenticated as, and their session unless a trusted proxy vouched. +#[derive(Clone, Copy, Debug)] +pub(crate) struct AuthenticatedPerson(pub(crate) Uuid, pub(crate) Option); + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct SessionIdentity { + pub session_id: Uuid, + pub principal_id: Uuid, + pub expires_at: DateTime, +} + +pub enum SessionAccess<'a> { + Read, + Write { csrf: &'a str }, +} + +pub struct IssuedSession { + pub identity: SessionIdentity, + pub bearer: String, + pub csrf: String, +} + +#[derive(Serialize, utoipa::ToSchema)] +pub(crate) struct StartedLogin { + pub(crate) browser_code: String, + pub(crate) expires_at: DateTime, +} + +#[derive(Debug)] +pub(crate) enum ClaimedLogin { + Pending, + Unavailable, + Authenticated(IssuedSession), +} + +#[derive(Debug, Deserialize)] +pub(crate) struct StoredLogin { + pub(crate) client_hash: String, + pub(crate) principal_id: Option, + pub(crate) session_id: Option, +} + +impl std::fmt::Debug for StartedLogin { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("StartedLogin") + .field("expires_at", &self.expires_at) + .finish_non_exhaustive() + } +} + +impl std::fmt::Debug for IssuedSession { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("IssuedSession") + .field("identity", &self.identity) + .field("bearer", &"[redacted]") + .field("csrf", &"[redacted]") + .finish() + } +} + +#[derive(Debug)] +pub enum PasswordLogin { + Authenticated(IssuedSession), + Denied { throttled: bool }, +} + +pub enum AuthenticationAttempt<'a> { + Login(&'a str), + Reauthenticate(Uuid), + IdentityProvider, +} + +#[derive(Debug)] +pub(crate) struct OidcFlowStart { + pub(crate) state: openidconnect::CsrfToken, + pub(crate) browser: openidconnect::CsrfToken, + pub(crate) nonce: openidconnect::Nonce, + pub(crate) pkce_verifier: openidconnect::PkceCodeVerifier, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct StoredOidcFlow { + pub(crate) flow_id: Uuid, + pub(crate) browser_hash: String, + pub(crate) nonce: openidconnect::Nonce, + pub(crate) pkce_verifier: openidconnect::PkceCodeVerifier, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct StoredSession { + #[serde(flatten)] + pub(crate) identity: SessionIdentity, + pub(crate) secret_hash: String, + pub(crate) csrf_hash: String, +} + +/// The bearer is returned once. Debug output must never contain it. +#[derive(Serialize)] +pub struct IssuedToken { + pub identity: TokenIdentity, + pub bearer: String, +} + +impl std::fmt::Debug for IssuedToken { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("IssuedToken") + .field("identity", &self.identity) + .field("bearer", &"[redacted]") + .finish() + } +} + +#[derive(Debug, Deserialize)] +pub(crate) struct StoredToken { + pub(crate) token_id: Uuid, + pub(crate) scope_json: CredentialScope, + pub(crate) secret_hash: String, + pub(crate) expires_at: DateTime, +} diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/development.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/development.rs new file mode 100644 index 0000000000..636f2afab1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/development.rs @@ -0,0 +1,63 @@ +//! Development safety defaults are ordinary audited trigger changes, not edited definitions. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::declarations::_helpers::operator_change::change_trigger_in; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn trigger_ids( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), +) -> Result<(ControlTransaction, BTreeSet), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT trigger_id FROM sqb_triggers WHERE location_id=$1 AND target=$2 AND removed_in_snapshot_id IS NULL", + vec![Parameter::Id(location_id), Parameter::Text(target.into())], + ).await?; + let ids = rows + .iter() + .map(|row| { + row["trigger_id"] + .as_str() + .unwrap_or_default() + .parse() + .map_err(|_| ControlError::Schema("invalid trigger identity".into())) + }) + .collect::>()?; + Ok((transaction, ids)) +} + +pub(crate) async fn pause_triggers( + transaction: ControlTransaction, + scope: (Uuid, &str), + previous: &BTreeSet, + request_id: Uuid, +) -> Result { + let (mut transaction, ids) = trigger_ids(transaction, scope).await?; + let context = AuditContext { + actor: "local-development".into(), + granting_role: "control-store-owner".into(), + request_id, + ip_address: None, + }; + for trigger_id in ids.difference(previous) { + (transaction, _) = change_trigger_in( + transaction, + &context, + &TriggerChangeRequest { + event_id: Uuid::new_v5(&request_id, trigger_id.as_bytes()), + trigger_id: *trigger_id, + actor: context.actor.clone(), + reason: "Development starts with triggers paused".into(), + change: TriggerChange::Pause, + }, + ) + .await?; + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs new file mode 100644 index 0000000000..31491d736e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs @@ -0,0 +1,49 @@ +//! Decode the same immutable declaration documents at publication and execution boundaries. + +use std::collections::BTreeMap; + +use serde::de::DeserializeOwned; +use serde_json::Value; + +use crate::declarations::models::Definitions; +use crate::store::errors::ControlError; + +fn document( + documents: &BTreeMap, + name: &str, +) -> Result, ControlError> { + documents.get(name).map_or_else( + || Ok(BTreeMap::new()), + |value| { + serde_json::from_value(value.clone()) + .map_err(|error| ControlError::Invalid(format!("{name}: {error}"))) + }, + ) +} + +pub(crate) fn decode_definitions( + documents: &BTreeMap, +) -> Result { + let mut definitions = Definitions { + jobs: document(documents, "jobs")?, + groups: document(documents, "groups")?, + roles: documents.get("roles").cloned(), + ..Definitions::default() + }; + for (name, value) in document::(documents, "triggers")? { + if value.get("when").is_some() { + definitions.sensors.insert( + name, + serde_json::from_value(value) + .map_err(|error| ControlError::Invalid(format!("triggers: {error}")))?, + ); + } else { + definitions.triggers.insert( + name, + serde_json::from_value(value) + .map_err(|error| ControlError::Invalid(format!("triggers: {error}")))?, + ); + } + } + Ok(definitions) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs new file mode 100644 index 0000000000..1e3bd2943f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod development; +pub(crate) mod documents; +pub(crate) mod operator_change; +pub(crate) mod operator_state; +pub(crate) mod publication; +pub(crate) mod trigger_state; diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_change.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_change.rs new file mode 100644 index 0000000000..bb4e35df54 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_change.rs @@ -0,0 +1,65 @@ +//! Apply an operator change and its audit record in one transaction, with retry-safe request identity. + +use serde_json::json; + +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::declarations::_helpers::operator_state::apply_operator_change; +use crate::declarations::models::{EditableTriggerState, TriggerChangeRequest}; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn change_trigger_in( + transaction: ControlTransaction, + context: &AuditContext, + request: &TriggerChangeRequest, +) -> Result<(ControlTransaction, bool), ControlError> { + if request.actor.trim().is_empty() || request.actor != context.actor { + return Err(ControlError::Invalid( + "trigger changes require the authenticated audit actor".into(), + )); + } + let body = + serde_json::to_value(request).map_err(|error| ControlError::Invalid(error.to_string()))?; + let (transaction, previous) = query( + transaction, + "SELECT scope_json FROM sqb_security_audit WHERE event_id=$1 AND action='trigger.changed'", + vec![Parameter::Id(request.event_id)], + ) + .await?; + if let Some(previous) = previous.first() { + if previous["scope_json"].get("request") != Some(&body) { + return Err(ControlError::Invalid( + "trigger change event ID already has different content".into(), + )); + } + return Ok((transaction, false)); + } + let (transaction, rows) = query( + transaction, + "SELECT t.definition_json, t.snapshot_id, s.started_at, s.last_tick_for, s.status_override \ + FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + WHERE t.trigger_id=$1 AND t.removed_in_snapshot_id IS NULL", + vec![Parameter::Id(request.trigger_id)], + ) + .await?; + let state: EditableTriggerState = decode_row(rows.into_iter().next().ok_or_else(|| { + ControlError::Invalid("trigger does not exist in the current snapshot".into()) + })?)?; + let snapshot_id = state.snapshot_id; + let (transaction, (before, after)) = apply_operator_change(transaction, state, request).await?; + let transaction = append_audit( + transaction, + context, + (request.event_id, "trigger.changed"), + ( + json!({"trigger_id": request.trigger_id, "snapshot_id": snapshot_id, "request": body}), + before, + after, + ), + ) + .await?; + Ok((transaction, true)) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs new file mode 100644 index 0000000000..df0912c8db --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs @@ -0,0 +1,82 @@ +//! Apply the operator's projection and return the before/after audit images in the same transaction. + +use serde_json::{Value, json}; + +use crate::declarations::_helpers::trigger_state::trigger_timing; +use crate::declarations::models::{ + EditableTriggerState, TriggerChange, TriggerChangeRequest, TriggerDeclaration, TriggerStatus, +}; +use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; +use crate::sensors::main::reset_sensor_cursor::reset_sensor_cursor; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{Change, ControlTransaction, Parameter}; + +pub(crate) async fn apply_operator_change( + transaction: ControlTransaction, + mut state: EditableTriggerState, + request: &TriggerChangeRequest, +) -> Result<(ControlTransaction, (Value, Value)), ControlError> { + let (mut transaction, cursor) = match &state.definition_json { + TriggerDeclaration::Sensor(_) => { + if matches!(request.change, TriggerChange::SetCursor { .. }) { + return Err(ControlError::Invalid( + "sensor cursors use storage order; use reset-cursor to replay retained events" + .into(), + )); + } + let (transaction, cursor) = read_sensor_cursor(transaction, request.trigger_id).await?; + (transaction, Some(cursor)) + } + TriggerDeclaration::Schedule(_) => (transaction, None), + }; + let mut before = + json!({"status_override": state.status_override, "last_tick_for": state.last_tick_for}); + if let Some(cursor) = &cursor { + before["cursor"] = json!(cursor); + } + match request.change { + TriggerChange::Enable => state.status_override = Some(TriggerStatus::Running), + TriggerChange::Pause => state.status_override = Some(TriggerStatus::Paused), + TriggerChange::ResetDefault => state.status_override = None, + TriggerChange::SetCursor { time } => state.last_tick_for = Some(time), + TriggerChange::ResetCursor => state.last_tick_for = None, + } + let (status, next) = match &state.definition_json { + TriggerDeclaration::Schedule(definition) => trigger_timing( + definition, + state.started_at, + state.last_tick_for, + state.status_override, + )?, + TriggerDeclaration::Sensor(definition) => { + if matches!(request.change, TriggerChange::ResetCursor) { + transaction = reset_sensor_cursor(transaction, request.trigger_id).await?; + } + let status = state.status_override.unwrap_or(definition.default_status); + ( + status, + (status == TriggerStatus::Running).then_some(transaction.now), + ) + } + }; + let mut after = json!({"status_override": state.status_override, "last_tick_for": state.last_tick_for, "reason": request.reason}); + if let Some(cursor) = cursor { + after["cursor"] = json!(if matches!(request.change, TriggerChange::ResetCursor) { + Default::default() + } else { + cursor + }); + } + let now = transaction.now; + let (mut transaction, _) = query(transaction, + "UPDATE sqb_trigger_state SET status=$1, status_override=$2, last_tick_for=$3, next_due_at=$4, updated_at=$5 WHERE trigger_id=$6 \ + AND (status IS DISTINCT FROM $1 OR status_override IS DISTINCT FROM $2 OR last_tick_for IS DISTINCT FROM $3 \ + OR next_due_at IS DISTINCT FROM $4) RETURNING trigger_id", + vec![Parameter::Text(status.as_str().into()), Parameter::OptionalText(state.status_override.map(|value| value.as_str().into())), Parameter::OptionalTime(state.last_tick_for), + Parameter::OptionalTime(next), Parameter::Time(now), Parameter::Id(request.trigger_id)]).await?; + transaction + .changes + .insert(Change::Trigger(request.trigger_id)); + Ok((transaction, (before, after))) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/publication.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/publication.rs new file mode 100644 index 0000000000..64e9701929 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/publication.rs @@ -0,0 +1,98 @@ +//! Publish definition projections inside the current-snapshot transaction. + +use uuid::Uuid; + +use crate::declarations::_helpers::trigger_state::{retain_calendar_ticks, trigger_timing}; +use crate::declarations::models::{Definitions, PreviousTriggerState}; +use crate::sensors::main::publish_sensors::publish_sensors; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn publish_projections( + mut transaction: ControlTransaction, + snapshot: &SnapshotRecord, + definitions: Definitions, +) -> Result { + transaction = publish_jobs(transaction, snapshot, &definitions).await?; + (transaction, _) = query(transaction, + "UPDATE sqb_triggers SET removed_in_snapshot_id=$1 WHERE location_id=$2 AND target=$3 AND removed_in_snapshot_id IS NULL", + vec![Parameter::Id(snapshot.snapshot_id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone())]).await?; + transaction = publish_sensors(transaction, snapshot, &definitions).await?; + for (name, mut trigger) in definitions.triggers { + let id = Uuid::new_v5( + &snapshot.location_id, + &serde_json::to_vec(&(&snapshot.target, "schedule", &name)) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + let previous; + (transaction, previous) = query(transaction, + "SELECT t.definition_json, s.started_at, s.last_tick_for, s.status_override FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id WHERE s.trigger_id=$1", + vec![Parameter::Id(id)]).await?; + let state = previous + .into_iter() + .next() + .map(decode_row::) + .transpose()?; + retain_calendar_ticks(&name, &mut trigger, state.as_ref(), snapshot)?; + let definition = serde_json::to_value(&trigger) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_triggers (trigger_id, location_id, target, kind, name, job_key, definition_json, snapshot_id) VALUES ($1,$2,$3,'schedule',$4,$5,$6,$7) \ + ON CONFLICT (trigger_id) DO UPDATE SET job_key=excluded.job_key, definition_json=excluded.definition_json, snapshot_id=excluded.snapshot_id, removed_in_snapshot_id=NULL", + vec![Parameter::Id(id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name), + Parameter::Text(trigger.job_key.clone()), Parameter::Json(definition), Parameter::Id(snapshot.snapshot_id)]).await?; + let started_at = state.as_ref().map_or( + trigger.starts_at.unwrap_or(snapshot.published_at), + |state| state.started_at, + ); + let (status, next) = trigger_timing( + &trigger, + started_at, + state.as_ref().and_then(|state| state.last_tick_for), + state.as_ref().and_then(|state| state.status_override), + )?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_trigger_state (trigger_id, status, started_at, next_due_at, updated_at) VALUES ($1,$2,$3,$4,$5) \ + ON CONFLICT (trigger_id) DO UPDATE SET status=excluded.status, next_due_at=excluded.next_due_at, updated_at=excluded.updated_at", + vec![Parameter::Id(id), Parameter::Text(status.as_str().into()), Parameter::Time(started_at), Parameter::OptionalTime(next), Parameter::Time(snapshot.published_at)]).await?; + } + Ok(transaction) +} + +pub(crate) async fn publish_jobs( + mut transaction: ControlTransaction, + snapshot: &SnapshotRecord, + definitions: &Definitions, +) -> Result { + (transaction, _) = query( + transaction, + "DELETE FROM sqb_jobs WHERE location_id=$1 AND target=$2", + vec![ + Parameter::Id(snapshot.location_id), + Parameter::Text(snapshot.target.clone()), + ], + ) + .await?; + let jobs = definitions + .jobs + .iter() + .map(|(name, job)| (name, "job", job)) + .chain( + definitions + .groups + .iter() + .map(|(name, group)| (name, "group", &group.job)), + ); + for (name, kind, job) in jobs { + let definition = + serde_json::to_value(job).map_err(|error| ControlError::Invalid(error.to_string()))?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_jobs (location_id, target, job_key, snapshot_id, kind, definition_json) VALUES ($1,$2,$3,$4,$6,$5)", + vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), + Parameter::Id(snapshot.snapshot_id), Parameter::Json(definition), Parameter::Text(kind.into())]).await?; + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs new file mode 100644 index 0000000000..b258882455 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs @@ -0,0 +1,64 @@ +//! Recompute the effective schedule after code publication or an operator override. + +use chrono::{DateTime, TimeDelta, Utc}; + +use crate::declarations::models::{ + PreviousTriggerState, Schedule, TriggerDefinition, TriggerStatus, +}; +use crate::scheduler::constants::MAX_TICKS_PER_BATCH; +use crate::scheduler::main::next_schedule_time::next_schedule_time; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; + +/// Replacing a future horizon must retain due ticks not yet consumed by the scheduler. +pub(crate) fn retain_calendar_ticks( + name: &str, + trigger: &mut TriggerDefinition, + state: Option<&PreviousTriggerState>, + snapshot: &SnapshotRecord, +) -> Result<(), ControlError> { + if let Some(state) = state + && let Schedule::Calendar { + times: previous_times, + .. + } = &state.definition_json.schedule + && let Schedule::Calendar { times, .. } = &mut trigger.schedule + { + times.extend( + previous_times + .iter() + .filter(|time| { + **time >= state.started_at + && **time <= snapshot.published_at + && state.last_tick_for.is_none_or(|last| **time > last) + }) + .copied(), + ); + if times.len() > MAX_TICKS_PER_BATCH { + return Err(ControlError::Invalid(format!( + "calendar {name} pending and upcoming times exceed {MAX_TICKS_PER_BATCH}; let the scheduler drain pending ticks before refreshing" + ))); + } + } + Ok(()) +} + +pub(crate) fn trigger_timing( + definition: &TriggerDefinition, + started_at: DateTime, + last_tick_for: Option>, + status_override: Option, +) -> Result<(TriggerStatus, Option>), ControlError> { + let status = status_override.unwrap_or(definition.default_status); + let after = last_tick_for.unwrap_or( + started_at + .checked_sub_signed(TimeDelta::microseconds(1)) + .ok_or_else(|| ControlError::Invalid("trigger start is out of range".into()))?, + ); + let next = if status == TriggerStatus::Running { + next_schedule_time(&definition.schedule, after)? + } else { + None + }; + Ok((status, next)) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/change_person_trigger.rs b/crates/sqlbuild-orchestration/src/declarations/main/change_person_trigger.rs new file mode 100644 index 0000000000..9eea084384 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/change_person_trigger.rs @@ -0,0 +1,64 @@ +//! Resolve a trigger's real location before authorizing an authenticated runtime override. + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::main::authorize_person::authorize_person; +use crate::authorization::main::deny_authorization::deny; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::declarations::_helpers::operator_change::change_trigger_in; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn change_person_trigger( + store: &ControlStore, + (context, credentials): (&AuditContext, Uuid), + request: &TriggerChangeRequest, +) -> Result { + let (transaction, rows) = query(begin(store, TransactionMode::Write).await?, + "SELECT location_id,target FROM sqb_triggers WHERE trigger_id=$1 AND removed_in_snapshot_id IS NULL", + vec![Parameter::Id(request.trigger_id)]).await?; + let row = rows + .first() + .ok_or_else(|| ControlError::Invalid("current trigger not found".into()))?; + let scope = PermissionScope { + action: match request.change { + TriggerChange::Enable | TriggerChange::Pause | TriggerChange::ResetDefault => { + Action::TriggerToggle + } + TriggerChange::SetCursor { .. } | TriggerChange::ResetCursor => { + Action::SensorResetCursor + } + }, + location_id: serde_json::from_value(row["location_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + target: serde_json::from_value(row["target"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + nodes: Default::default(), + }; + let (transaction, context) = authorize_person( + transaction, + (context, credentials), + &scope, + DefaultRole::None, + ) + .await?; + if request.reason.trim().is_empty() { + return deny( + transaction, + &context, + json!({"trigger_id":request.trigger_id}), + "trigger override requires a reason", + ) + .await; + } + let request = TriggerChangeRequest { + actor: context.actor.clone(), + ..request.clone() + }; + let (transaction, applied) = change_trigger_in(transaction, &context, &request).await?; + commit(transaction).await?; + Ok(applied) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs new file mode 100644 index 0000000000..365b5ce2d5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs @@ -0,0 +1,24 @@ +//! Commit an operator fact with its projection and security audit. + +use crate::credentials::models::AuditContext; +use crate::declarations::_helpers::operator_change::change_trigger_in; +use crate::declarations::models::TriggerChangeRequest; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +/// Authorization belongs to the API boundary; `actor` is authenticated, not client input. +pub async fn change_trigger( + store: &ControlStore, + context: &AuditContext, + request: &TriggerChangeRequest, +) -> Result { + let (transaction, applied) = change_trigger_in( + begin(store, TransactionMode::Write).await?, + context, + request, + ) + .await?; + commit(transaction).await?; + Ok(applied) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/frozen_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/frozen_definitions.rs new file mode 100644 index 0000000000..5b04c1b88f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/frozen_definitions.rs @@ -0,0 +1,22 @@ +//! Runs resolve immutable snapshot documents, never a later deployment's job projections. + +use uuid::Uuid; + +use crate::declarations::_helpers::documents::decode_definitions; +use crate::declarations::models::Definitions; +use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn frozen_definitions( + transaction: ControlTransaction, + snapshot_id: Uuid, +) -> Result<(ControlTransaction, Definitions), ControlError> { + let (transaction, documents) = read_snapshot_documents( + transaction, + snapshot_id, + Some(&["jobs", "groups", "triggers"]), + ) + .await?; + Ok((transaction, decode_definitions(&documents)?)) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs new file mode 100644 index 0000000000..e896fba0a5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs @@ -0,0 +1,92 @@ +//! Freeze job boundaries from snapshot lineage plus explicit waits, rejecting cycles. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::declarations::models::JobDefinition; +use crate::snapshots::models::SnapshotInput; +use crate::store::errors::ControlError; + +pub fn group_dependencies( + input: &SnapshotInput, + jobs: &BTreeMap, + (members, after): (&BTreeSet, &BTreeMap>), +) -> Result>, ControlError> { + if members.is_empty() + || members.iter().any(|name| !jobs.contains_key(name)) + || after + .iter() + .any(|(name, dependencies)| !members.contains(name) || !dependencies.is_subset(members)) + { + return Err(ControlError::Invalid( + "group needs known jobs and dependencies within its membership".into(), + )); + } + let edges = lineage_dependencies(input, jobs, (members, after)); + ensure_acyclic(&edges)?; + Ok(edges) +} + +fn lineage_dependencies( + input: &SnapshotInput, + jobs: &BTreeMap, + (members, after): (&BTreeSet, &BTreeMap>), +) -> BTreeMap> { + let mut owners: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); + for name in members { + for key in &jobs[name].selection { + owners.entry(key).or_default().insert(name); + } + } + let mut edges: BTreeMap> = BTreeMap::new(); + for name in members { + let mut dependencies = after.get(name).cloned().unwrap_or_default(); + let mut seen: BTreeSet<&str> = BTreeSet::new(); + let mut pending: Vec<&str> = jobs[name].selection.iter().map(String::as_str).collect(); + while let Some(key) = pending.pop() { + if !seen.insert(key) { + continue; + } + if let Some(node) = input.nodes.get(key) + && let Some(upstream) = node + .definition + .get("deps") + .and_then(serde_json::Value::as_array) + { + for upstream in upstream.iter().filter_map(serde_json::Value::as_str) { + if let Some(owners) = owners.get(upstream) { + dependencies.extend( + owners + .iter() + .filter(|owner| **owner != name) + .map(|owner| (*owner).to_owned()), + ); + } + pending.push(upstream); + } + } + } + edges.insert(name.clone(), dependencies); + } + edges +} + +fn ensure_acyclic(edges: &BTreeMap>) -> Result<(), ControlError> { + let mut remaining = edges.clone(); + while !remaining.is_empty() { + let ready: BTreeSet = remaining + .iter() + .filter(|(_, deps)| deps.is_empty()) + .map(|(name, _)| name.clone()) + .collect(); + if ready.is_empty() { + return Err(ControlError::Invalid( + "group dependencies contain a cycle".into(), + )); + } + remaining.retain(|name, deps| { + deps.retain(|dependency| !ready.contains(dependency)); + !ready.contains(name) + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/list_trigger_events.rs b/crates/sqlbuild-orchestration/src/declarations/main/list_trigger_events.rs new file mode 100644 index 0000000000..4ef81bf97b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/list_trigger_events.rs @@ -0,0 +1,41 @@ +//! Read a trigger's operator changes from the security audit in committed order. + +use uuid::Uuid; + +use crate::declarations::models::TriggerChangeRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn list_trigger_events( + store: &ControlStore, + trigger_id: Uuid, + limit: u32, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT scope_json, occurred_at FROM sqb_security_audit WHERE action='trigger.changed' \ + AND scope_json ->> 'trigger_id'=$1 ORDER BY occurred_at, event_id LIMIT $2", + vec![ + Parameter::Text(trigger_id.to_string()), + Parameter::Integer(i64::from(limit.min(1000))), + ], + ) + .await?; + let events = rows + .into_iter() + .map(|row| { + serde_json::from_value(serde_json::json!({ + "request_json": row["scope_json"]["request"], + "snapshot_id": row["scope_json"]["snapshot_id"], + "occurred_at": row["occurred_at"], + })) + .map_err(|error| ControlError::Schema(format!("trigger change: {error}"))) + }) + .collect::, _>>()?; + commit(transaction).await?; + Ok(events) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/list_triggers.rs b/crates/sqlbuild-orchestration/src/declarations/main/list_triggers.rs new file mode 100644 index 0000000000..c3ead93c9c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/list_triggers.rs @@ -0,0 +1,30 @@ +//! Current declarations and runtime state are read in one consistent transaction. + +use uuid::Uuid; + +use crate::declarations::models::TriggerRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn list_triggers( + store: &ControlStore, + location_id: Uuid, + target: &str, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT t.trigger_id, t.name, t.job_key, t.definition_json, s.status, s.last_tick_for, s.next_due_at \ + FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + WHERE t.location_id=$1 AND t.target=$2 AND t.removed_in_snapshot_id IS NULL ORDER BY t.name", + vec![Parameter::Id(location_id), Parameter::Text(target.into())]).await?; + let triggers = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(triggers) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs new file mode 100644 index 0000000000..23cab85fba --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs @@ -0,0 +1,9 @@ +pub mod change_person_trigger; +pub mod change_trigger; +pub(crate) mod frozen_definitions; +pub mod group_dependencies; +pub mod list_trigger_events; +pub mod list_triggers; +pub mod pause_development_triggers; +pub(crate) mod publish_definitions; +pub mod snapshot_definitions; diff --git a/crates/sqlbuild-orchestration/src/declarations/main/pause_development_triggers.rs b/crates/sqlbuild-orchestration/src/declarations/main/pause_development_triggers.rs new file mode 100644 index 0000000000..5c8c182541 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/pause_development_triggers.rs @@ -0,0 +1,18 @@ +//! Pause retained triggers before a development scheduler can acquire leadership. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::declarations::_helpers::development::pause_triggers; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn pause_development_triggers( + store: &ControlStore, + scope: (Uuid, &str), +) -> Result<(), ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + commit(pause_triggers(transaction, scope, &BTreeSet::new(), Uuid::now_v7()).await?).await +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs new file mode 100644 index 0000000000..4c181629a4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs @@ -0,0 +1,38 @@ +//! Publish definition projections and optional audited development defaults atomically. + +use std::collections::BTreeSet; + +use crate::authorization::main::publish_custom_roles::publish_custom_roles; +use crate::declarations::_helpers::development::{pause_triggers, trigger_ids}; +use crate::declarations::_helpers::publication::publish_projections; +use crate::declarations::models::Definitions; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn publish_definitions( + transaction: ControlTransaction, + snapshot: &SnapshotRecord, + mut definitions: Definitions, + pause_new_triggers: bool, +) -> Result { + let roles = definitions.roles.take(); + let (transaction, previous) = if pause_new_triggers { + trigger_ids(transaction, (snapshot.location_id, &snapshot.target)).await? + } else { + (transaction, BTreeSet::new()) + }; + let transaction = publish_projections(transaction, snapshot, definitions).await?; + let transaction = publish_custom_roles(transaction, snapshot, roles.as_ref()).await?; + if pause_new_triggers { + pause_triggers( + transaction, + (snapshot.location_id, &snapshot.target), + &previous, + snapshot.snapshot_id, + ) + .await + } else { + Ok(transaction) + } +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs new file mode 100644 index 0000000000..b89bfb316c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs @@ -0,0 +1,78 @@ +//! Validate resolved declarations before acquiring the snapshot publication lock. + +use std::collections::BTreeSet; + +use crate::declarations::_helpers::documents::decode_definitions; +use crate::declarations::main::group_dependencies::group_dependencies; +use crate::declarations::models::{Definitions, Schedule}; +use crate::scheduler::constants::MAX_TICKS_PER_BATCH; +use crate::scheduler::main::parse_cron::parse_cron; +use crate::sensors::main::validate_sensors::validate_sensors; +use crate::snapshots::models::SnapshotInput; +use crate::store::errors::ControlError; + +/// Names, references and resolved selections are checked again at the native trust boundary. +pub fn snapshot_definitions(input: &SnapshotInput) -> Result { + let definitions = decode_definitions(&input.documents)?; + for (name, job) in &definitions.jobs { + if name.trim().is_empty() + || job.description.trim().is_empty() + || job.selection.is_empty() + || job + .selection + .iter() + .any(|key| !input.nodes.contains_key(key)) + { + return Err(ControlError::Invalid(format!( + "job {name} needs a description and a nonempty selection of snapshot nodes" + ))); + } + } + for (name, group) in &definitions.groups { + let members: BTreeSet = group.jobs.keys().cloned().collect(); + let dependencies = group_dependencies(input, &definitions.jobs, (&members, &group.jobs))?; + let selection: BTreeSet = group + .jobs + .keys() + .flat_map(|name| definitions.jobs[name].selection.iter().cloned()) + .collect(); + if name.trim().is_empty() + || definitions.jobs.contains_key(name) + || group.job.description.trim().is_empty() + || group.job.selection != selection + || group.jobs != dependencies + || !group.job.settings.is_empty() + { + return Err(ControlError::Invalid(format!( + "group {name} has invalid membership, selection or dependencies" + ))); + } + } + for (name, trigger) in &definitions.triggers { + if name.trim().is_empty() + || trigger.description.trim().is_empty() + || (!definitions.jobs.contains_key(&trigger.job_key) + && !definitions.groups.contains_key(&trigger.job_key)) + { + return Err(ControlError::Invalid(format!( + "trigger {name} needs a description and a known job" + ))); + } + match &trigger.schedule { + Schedule::Cron { + expression, + timezone, + } => { + parse_cron(expression, timezone)?; + } + Schedule::Calendar { times, .. } if times.len() > MAX_TICKS_PER_BATCH => { + return Err(ControlError::Invalid(format!( + "calendar {name} exceeds {MAX_TICKS_PER_BATCH} upcoming times" + ))); + } + Schedule::Calendar { .. } => {} + } + } + validate_sensors(&definitions, input)?; + Ok(definitions) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/mod.rs b/crates/sqlbuild-orchestration/src/declarations/mod.rs new file mode 100644 index 0000000000..5359000d16 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs new file mode 100644 index 0000000000..afdcab28d0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -0,0 +1,144 @@ +//! Resolved, serializable declarations; Python objects never enter scheduler state. + +use std::collections::{BTreeMap, BTreeSet}; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::sensors::models::SensorDefinition; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum TriggerStatus { + #[default] + Running, + Paused, +} + +impl TriggerStatus { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Running => "running", + Self::Paused => "paused", + } + } +} + +/// Selections have already been resolved by the code location against the compiled snapshot. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct JobDefinition { + pub description: String, + pub selection: BTreeSet, + #[serde(default)] + pub settings: BTreeMap, + #[serde(default)] + pub concurrency_groups: BTreeSet, +} + +/// A virtual parent shares run controls, but only its physical children claim node writers. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct GroupDefinition { + #[serde(flatten)] + pub job: JobDefinition, + pub jobs: BTreeMap>, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum Schedule { + Cron { + expression: String, + timezone: String, + }, + Calendar { + times: BTreeSet>, + refresh_at: DateTime, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct TriggerDefinition { + pub description: String, + pub job_key: String, + pub schedule: Schedule, + #[serde(default)] + pub starts_at: Option>, + #[serde(default)] + pub default_status: TriggerStatus, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Definitions { + pub jobs: BTreeMap, + pub groups: BTreeMap, + pub triggers: BTreeMap, + pub sensors: BTreeMap, + pub roles: Option, +} + +/// Legacy schedule documents retain their stored shape; sensors add a distinct condition shape. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(untagged)] +pub enum TriggerDeclaration { + Schedule(TriggerDefinition), + Sensor(SensorDefinition), +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct TriggerRecord { + pub trigger_id: uuid::Uuid, + pub name: String, + pub job_key: String, + pub definition_json: TriggerDeclaration, + pub status: TriggerStatus, + pub last_tick_for: Option>, + pub next_due_at: Option>, +} + +/// Operator changes are immutable facts; clearing an override restores the published code default. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum TriggerChange { + Enable, + Pause, + ResetDefault, + SetCursor { time: DateTime }, + ResetCursor, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct TriggerChangeRequest { + pub event_id: uuid::Uuid, + pub trigger_id: uuid::Uuid, + pub actor: String, + pub reason: String, + pub change: TriggerChange, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct TriggerChangeRecord { + pub request_json: TriggerChangeRequest, + pub occurred_at: DateTime, + pub snapshot_id: uuid::Uuid, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct EditableTriggerState { + pub(crate) definition_json: TriggerDeclaration, + pub(crate) snapshot_id: uuid::Uuid, + pub(crate) started_at: DateTime, + pub(crate) last_tick_for: Option>, + pub(crate) status_override: Option, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct PreviousTriggerState { + pub(crate) definition_json: TriggerDefinition, + pub(crate) started_at: DateTime, + pub(crate) last_tick_for: Option>, + pub(crate) status_override: Option, +} diff --git a/crates/sqlbuild-orchestration/src/lib.rs b/crates/sqlbuild-orchestration/src/lib.rs new file mode 100644 index 0000000000..87aebeb15e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/lib.rs @@ -0,0 +1,17 @@ +//! The Python-free SQLBuild control plane. + +#![forbid(unsafe_code)] + +pub mod api; +pub mod authorization; +pub mod credentials; +pub mod declarations; +pub mod platform; +pub mod runs; +pub mod scheduler; +pub mod sensors; +pub mod snapshots; +pub mod store; +pub mod warehouse_metadata; +pub mod webhooks; +pub mod worker; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/connection_tests.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/connection_tests.rs new file mode 100644 index 0000000000..44cec03e56 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/connection_tests.rs @@ -0,0 +1,31 @@ +//! Connection-test submission and reads share live action authority under the store lock. + +use uuid::Uuid; + +use crate::authorization::main::authorize_person::authorize_person; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +/// No credentials means a trusted direct-store owner, never an HTTP-supplied role. +pub(crate) async fn authorize_connection_test( + transaction: ControlTransaction, + (context, credentials): (&AuditContext, Option), + (location_id, target): (Uuid, &str), +) -> Result<(ControlTransaction, AuditContext), ControlError> { + let Some(credentials) = credentials else { + return Ok((transaction, context.clone())); + }; + authorize_person( + transaction, + (context, credentials), + &PermissionScope { + action: Action::ConnectionTest, + location_id: Some(location_id), + target: Some(target.into()), + nodes: Default::default(), + }, + DefaultRole::None, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs new file mode 100644 index 0000000000..51c5fd6959 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs @@ -0,0 +1,62 @@ +//! Liveness is derived across all sessions; an old instance cannot mask a healthy successor. + +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::platform::_helpers::observations::observe; +use crate::platform::models::{PlatformEvent, WatchdogLocation}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Locations report every 30 seconds; tolerate two missed reports. +pub(crate) async fn scan_location_page( + store: &ControlStore, + installation_id: Uuid, + after: Option<(Uuid, String)>, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let mut parameters = vec![Parameter::Time(transaction.now - TimeDelta::seconds(90))]; + let cursor = if let Some((location_id, target)) = after { + parameters.extend([Parameter::Id(location_id), Parameter::Text(target)]); + "AND (i.location_id>$2 OR (i.location_id=$2 AND i.target>$3))" + } else { + "" + }; + let (mut transaction, rows) = query( + transaction, + &format!( + "SELECT i.location_id, i.target, MAX(i.last_seen_at) AS last_seen_at \ + FROM sqb_location_instances i JOIN sqb_locations l ON l.location_id=i.location_id \ + WHERE l.archived_at IS NULL {cursor} GROUP BY i.location_id,i.target \ + HAVING MAX(i.last_seen_at)<$1 ORDER BY i.location_id,i.target LIMIT 100" + ), + parameters, + ) + .await?; + let locations: Vec = + rows.into_iter().map(decode_row).collect::>()?; + let next = locations + .last() + .map(|row| (row.location_id, row.target.clone())); + for location in locations { + transaction = observe( + transaction, + installation_id, + ( + PlatformEvent::LocationBroken { + location_id: location.location_id, + target: location.target, + last_seen_at: location.last_seen_at, + }, + None, + ), + ) + .await?; + } + commit(transaction).await?; + Ok(next) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/location_heartbeat.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/location_heartbeat.rs new file mode 100644 index 0000000000..a13e3fabb1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/location_heartbeat.rs @@ -0,0 +1,50 @@ +//! A heartbeat replaces one instance's liveness, never appending an execution fact. + +use serde_json::json; +use uuid::Uuid; + +use crate::platform::models::LocationHealthReport; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn location_heartbeat( + transaction: ControlTransaction, + location_id: Uuid, + report: &LocationHealthReport, +) -> Result<(ControlTransaction, bool), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT instance.location_id AS instance_location, instance.health_json AS report \ + FROM sqb_locations location LEFT JOIN sqb_location_instances instance ON instance.instance_id=$1 \ + WHERE location.location_id=$2 AND location.archived_at IS NULL", + vec![Parameter::Id(report.session_id), Parameter::Id(location_id)]).await?; + let row = rows + .first() + .ok_or_else(|| ControlError::Invalid("active location is required".into()))?; + if let Some(previous) = row.get("report").filter(|value| !value.is_null()) { + let previous: LocationHealthReport = serde_json::from_value(previous.clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if row.get("instance_location") != Some(&json!(location_id)) + || previous.target != report.target + { + return Err(ControlError::Invalid( + "location instance belongs to another scope".into(), + )); + } + if previous == *report { + return Ok((transaction, false)); + } + if previous.event_id == report.event_id { + return Err(ControlError::Invalid( + "location health event ID already has different content".into(), + )); + } + } + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_location_instances (instance_id, location_id, target, connected_at, last_seen_at, health_json) \ + VALUES ($1, $2, $3, $4, $4, $5) ON CONFLICT (instance_id) DO UPDATE SET last_seen_at=$4, health_json=$5", + vec![Parameter::Id(report.session_id), Parameter::Id(location_id), Parameter::Text(report.target.clone()), + Parameter::Time(now), Parameter::Json(json!(report))]).await?; + Ok((transaction, true)) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs new file mode 100644 index 0000000000..0c2b80f5ca --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod connection_tests; +pub(crate) mod liveness; +pub(crate) mod location_heartbeat; +pub(crate) mod observations; +pub(crate) mod retention; +pub(crate) mod watchdog; +pub(crate) mod work_client; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/observations.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/observations.rs new file mode 100644 index 0000000000..d421b402c3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/observations.rs @@ -0,0 +1,62 @@ +//! Replicas share a deterministic observation key, so each alert incident is recorded once. + +use chrono::{DateTime, DurationRound, TimeDelta, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::platform::models::PlatformEvent; +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlTransaction, Parameter}; + +/// A heartbeat is one fact per 30-second window; an alert is one fact per payload and `incident`. +pub(crate) async fn observe( + transaction: ControlTransaction, + installation_id: Uuid, + (event, incident): (PlatformEvent, Option>), +) -> Result { + let occurred_at = transaction + .now + .duration_trunc(TimeDelta::seconds(30)) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let occurrence = match event { + PlatformEvent::Heartbeat { .. } => Some(occurred_at), + _ => incident, + }; + let payload = json!({"platform": event}); + let identity = serde_json::to_vec(&(occurrence, &event)) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let event_id = Uuid::new_v5(&installation_id, &identity); + let (transaction, previous) = query( + transaction, + "SELECT 1 AS present FROM recall_events WHERE event_type=$1 AND event_id=$2 LIMIT 1", + vec![ + Parameter::Text(event.event_type().into()), + Parameter::Text(event_id.to_string()), + ], + ) + .await?; + if !previous.is_empty() { + return Ok(transaction); + } + let (transaction, outcome) = append_control_event( + transaction, + ControlEvent { + event_id, + invocation_id: format!("platform/{installation_id}/{}", occurred_at.date_naive()), + event_type: event.event_type(), + producer: "platform", + occurred_at, + run_id: None, + payload, + }, + ) + .await?; + if outcome.conflict.is_some() { + return Err(ControlError::Invalid( + "platform observation conflicts with a durable fact".into(), + )); + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs new file mode 100644 index 0000000000..6559f4d16d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/retention.rs @@ -0,0 +1,112 @@ +//! Every retention policy runs in one bounded pass; each part is safe to repeat. + +use chrono::TimeDelta; + +use crate::credentials::main::audit::prune_audit::prune_audit; +use crate::credentials::models::AuditRetention; +use crate::platform::constants::{ + HEARTBEAT_PAGE, HEARTBEAT_RETENTION_DAYS, LOCATION_RETENTION_DAYS, + PLATFORM_ALERT_RETENTION_DAYS, SENSOR_EVALUATION_RETENTION_DAYS, +}; +use crate::snapshots::main::prune_snapshots::prune_snapshots; +use crate::snapshots::models::SnapshotRetention; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use crate::warehouse_metadata::main::prune_statistics::prune_statistics; +use crate::warehouse_metadata::models::StatisticsRetention; + +/// Returns how many audit rows outlive their horizon because a sink has not acknowledged them. +pub(crate) async fn retain( + store: &ControlStore, + (audit, statistics, snapshots): (AuditRetention, StatisticsRetention, SnapshotRetention), + sinks: &[String], +) -> Result { + prune_snapshots(store, snapshots).await?; + while prune_statistics(store, statistics).await? == MAX_HISTORY_RECORDS { + tokio::task::yield_now().await; + } + while prune_platform_events(store).await? == HEARTBEAT_PAGE { + tokio::task::yield_now().await; + } + while prune_locations(store).await? == HEARTBEAT_PAGE { + tokio::task::yield_now().await; + } + while prune_sensor_evaluations(store).await? == HEARTBEAT_PAGE { + tokio::task::yield_now().await; + } + Ok(prune_audit(store, audit, sinks).await?.1) +} + +/// Old evaluation phases go once a later attempt of the same sensor exists. +async fn prune_sensor_evaluations(store: &ControlStore) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let cutoff = transaction.now - TimeDelta::days(SENSOR_EVALUATION_RETENTION_DAYS); + let (transaction, rows) = query( + transaction, + "DELETE FROM sqb_sensor_evaluation_events WHERE event_id IN (SELECT e.event_id \ + FROM sqb_sensor_evaluation_events e WHERE e.occurred_at<$1 AND EXISTS (SELECT 1 \ + FROM sqb_sensor_evaluation_events later WHERE later.trigger_id=e.trigger_id \ + AND later.phase='requested' AND later.occurred_at>e.occurred_at) \ + ORDER BY e.occurred_at LIMIT $2) RETURNING event_id", + vec![ + Parameter::Time(cutoff), + Parameter::Integer(HEARTBEAT_PAGE as i64), + ], + ) + .await?; + commit(transaction).await?; + Ok(rows.len()) +} + +/// Heartbeats only prove a replica was alive, so they go sooner than alerts. +async fn prune_platform_events(store: &ControlStore) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let heartbeats = transaction.now - TimeDelta::days(HEARTBEAT_RETENTION_DAYS); + let alerts = transaction.now - TimeDelta::days(PLATFORM_ALERT_RETENTION_DAYS); + let (transaction, rows) = query( + transaction, + "DELETE FROM recall_events WHERE storage_id IN (SELECT storage_id FROM recall_events \ + WHERE producer='platform' AND occurred_at Result { + let transaction = begin(store, TransactionMode::Write).await?; + let cutoff = transaction.now - TimeDelta::days(LOCATION_RETENTION_DAYS); + let (transaction, work) = query( + transaction, + "DELETE FROM sqb_location_work WHERE request_id IN (SELECT request_id FROM sqb_location_work \ + WHERE expires_at<$1 ORDER BY expires_at LIMIT $2) RETURNING request_id", + vec![ + Parameter::Time(cutoff), + Parameter::Integer(HEARTBEAT_PAGE as i64), + ], + ) + .await?; + let (transaction, instances) = query( + transaction, + "DELETE FROM sqb_location_instances WHERE instance_id IN (SELECT i.instance_id \ + FROM sqb_location_instances i WHERE i.last_seen_at<$1 AND NOT EXISTS (SELECT 1 \ + FROM sqb_location_work w WHERE w.session_id=i.instance_id) \ + ORDER BY i.last_seen_at LIMIT $2) RETURNING instance_id", + vec![ + Parameter::Time(cutoff), + Parameter::Integer(HEARTBEAT_PAGE as i64), + ], + ) + .await?; + commit(transaction).await?; + Ok(work.len().max(instances.len())) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs new file mode 100644 index 0000000000..2bfb02d6df --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs @@ -0,0 +1,138 @@ +//! Health scanning releases the writer lock between location and trigger pages. + +use chrono::TimeDelta; +use tokio::sync::watch; +use uuid::Uuid; + +use crate::declarations::models::Schedule; +use crate::platform::_helpers::liveness::scan_location_page; +use crate::platform::_helpers::observations::observe; +use crate::platform::models::{InstallationHealth, PlatformEvent, WatchdogTrigger}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn scan_health( + store: &ControlStore, + replica_id: Uuid, + shutdown: &watch::Receiver, +) -> Result<(), ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT i.installation_id, l.expires_at AS lease_expires_at, CASE WHEN l.expires_at>$1 THEN 1 ELSE 0 END AS leader \ + FROM sqb_installation i LEFT JOIN sqb_scheduler_lease l ON l.name='scheduler' WHERE i.singleton=1", + vec![Parameter::Time(now)]).await?; + let health: InstallationHealth = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("control store has no installation".into()))?, + )?; + let mut transaction = transaction; + if health.leader == 0 { + transaction = observe( + transaction, + health.installation_id, + (PlatformEvent::SchedulerNoLeader, health.lease_expires_at), + ) + .await?; + } + commit(transaction).await?; + let mut location_after = None; + while !*shutdown.borrow() { + location_after = scan_location_page(store, health.installation_id, location_after).await?; + if location_after.is_none() { + break; + } + tokio::task::yield_now().await; + } + let mut after = None; + while !*shutdown.borrow() { + after = scan_trigger_page(store, health.installation_id, after).await?; + if after.is_none() { + break; + } + tokio::task::yield_now().await; + } + if *shutdown.borrow() { + return Ok(()); + } + let transaction = begin(store, TransactionMode::Write).await?; + commit( + observe( + transaction, + health.installation_id, + (PlatformEvent::Heartbeat { replica_id }, None), + ) + .await?, + ) + .await?; + Ok(()) +} + +async fn scan_trigger_page( + store: &ControlStore, + installation_id: Uuid, + after: Option, +) -> Result, ControlError> { + let cursor = after.map_or("", |_| "AND t.trigger_id>$1"); + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (mut transaction, rows) = query( + transaction, + &format!( + "SELECT t.trigger_id, t.definition_json, s.next_due_at \ + FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id \ + WHERE t.kind='schedule' AND s.status='running' AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ + {cursor} ORDER BY t.trigger_id LIMIT 100" + ), + after.into_iter().map(Parameter::Id).collect(), + ) + .await?; + let triggers: Vec = + rows.into_iter().map(decode_row).collect::>()?; + let next = triggers.last().map(|trigger| trigger.trigger_id); + for trigger in triggers { + if let Some(scheduled_for) = trigger.next_due_at + && scheduled_for < now - TimeDelta::seconds(30) + { + transaction = observe( + transaction, + installation_id, + ( + PlatformEvent::TickOverdue { + trigger_id: trigger.trigger_id, + scheduled_for, + }, + None, + ), + ) + .await?; + } + if let Schedule::Calendar { times, refresh_at } = trigger.definition_json.schedule + && times + .last() + .is_none_or(|last| *last <= now + TimeDelta::days(1)) + { + transaction = observe( + transaction, + installation_id, + ( + PlatformEvent::CalendarHorizonLow { + trigger_id: trigger.trigger_id, + refresh_at, + horizon_end: times.last().copied(), + }, + None, + ), + ) + .await?; + } + } + commit(transaction).await?; + Ok(next) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs new file mode 100644 index 0000000000..6dee6aa9dd --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -0,0 +1,113 @@ +//! The authenticated client verifies every leased request's location, target and instance. + +use uuid::Uuid; + +use crate::api::errors::ClientError; +use crate::api::main::client_response::client_response; +use crate::api::models::MutationResponse; +use crate::platform::models::{ + LocationHealthReport, LocationWorkClaim, LocationWorkClient, LocationWorkPoll, + LocationWorkResult, +}; +use crate::warehouse_metadata::models::{ + MetadataBatch, StatisticsDueRead, StatisticsRead, WarehouseSyncResponse, WarehouseWatermarkRead, +}; + +impl LocationWorkClient { + pub async fn statistics_due( + &self, + target: &str, + request: &StatisticsDueRead, + ) -> Result { + self.post_metadata((target, Some("statistics-due")), request) + .await + } + + pub async fn report_health(&self, report: &LocationHealthReport) -> Result { + let endpoint = self.connection.endpoint(&[ + "api", + "locations", + &self.location_id.to_string(), + "health", + ])?; + let response: MutationResponse = + client_response(self.connection.client.post(endpoint).json(report)).await?; + Ok(response.applied) + } + + pub async fn metadata_watermark( + &self, + target: &str, + request: &WarehouseWatermarkRead, + ) -> Result>, ClientError> { + self.post_metadata((target, Some("watermark")), request) + .await + } + + /// Metadata shares the location's scoped pool, not the scheduler's execution authority. + pub async fn sync_metadata( + &self, + target: &str, + batch: &MetadataBatch, + ) -> Result { + self.post_metadata((target, None), batch).await + } + + async fn post_metadata( + &self, + (target, read): (&str, Option<&str>), + body: &impl serde::Serialize, + ) -> Result { + let location_id = self.location_id.to_string(); + let mut segments = vec![ + "api", + "locations", + &location_id, + "targets", + target, + "metadata", + ]; + segments.extend(read); + let endpoint = self.connection.endpoint(&segments)?; + client_response(self.connection.client.post(endpoint).json(body)).await + } + + pub async fn poll( + &self, + poll: &LocationWorkPoll, + ) -> Result, ClientError> { + let claim: Option = client_response( + self.connection + .client + .post(format!("{}poll", self.work_url)) + .json(poll), + ) + .await?; + if claim.as_ref().is_some_and(|claim| { + claim.request.location_id != self.location_id + || claim.request.target != poll.target + || claim.session_id != poll.session_id + }) { + return Err(ClientError::Configuration( + "work response does not match this location, target and instance", + )); + } + Ok(claim) + } + + /// Retain the same request, claim and result when retrying an uncertain acknowledgement. + pub async fn complete( + &self, + request_id: Uuid, + result: &LocationWorkResult, + ) -> Result { + let response: MutationResponse = client_response( + self.connection + .client + .post(format!("{}{request_id}", self.work_url)) + .json(result), + ) + .await?; + Ok(response.applied) + } +} diff --git a/crates/sqlbuild-orchestration/src/platform/constants.rs b/crates/sqlbuild-orchestration/src/platform/constants.rs new file mode 100644 index 0000000000..acf2969d6f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/constants.rs @@ -0,0 +1,21 @@ +//! Bounds shared by the platform event reader and webhook dispatcher. + +/// Operator-owned delivery endpoint; never inherited by project workers or printed in logs. +pub const PLATFORM_WEBHOOK_VARIABLE: &str = "SQLBUILD_PLATFORM_WEBHOOK_URL"; + +pub(crate) const PLATFORM_EVENT_PAGE_SIZE: i64 = 100; + +/// Replica heartbeats older than this are pruned; a day's date-scoped invocation is complete by then. +pub(crate) const HEARTBEAT_RETENTION_DAYS: i64 = 2; + +/// Platform alerts older than this are pruned; a condition still present then is recorded again. +pub(crate) const PLATFORM_ALERT_RETENTION_DAYS: i64 = 30; + +/// Location work past its expiry, and instances unseen for this long, are pruned. +pub(crate) const LOCATION_RETENTION_DAYS: i64 = 7; + +/// Sensor evaluations older than this are pruned, except each sensor's latest attempt. +pub(crate) const SENSOR_EVALUATION_RETENTION_DAYS: i64 = 7; + +/// Bounded retention deletions per transaction. +pub(crate) const HEARTBEAT_PAGE: usize = 1000; diff --git a/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs new file mode 100644 index 0000000000..106f446a4c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs @@ -0,0 +1,72 @@ +//! Serialized leases fence results from replaced location instances and uncertain poll retries. + +use chrono::Duration; +use uuid::Uuid; + +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::main::require_instance::require_instance; +use crate::platform::models::{LocationWorkClaim, LocationWorkPoll}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn claim_location_work( + store: &ControlStore, + location_id: Uuid, + poll: &LocationWorkPoll, +) -> Result, ControlError> { + let transaction = require_instance( + begin(store, TransactionMode::Write).await?, + (location_id, poll.session_id, &poll.target), + ) + .await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT w.request_id FROM sqb_location_work w WHERE w.location_id=$1 AND w.target=$2 \ + AND w.expires_at>$3 AND w.completed_at IS NULL \ + AND (w.claimed_until IS NULL OR w.claimed_until<=$3 OR w.session_id=$4) \ + AND (w.request_json->'work'->>'kind'='check_connection' OR EXISTS ( \ + SELECT 1 FROM sqb_sensor_evaluation_events e JOIN sqb_triggers t ON t.trigger_id=e.trigger_id \ + JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id JOIN sqb_scheduler_lease l ON l.name='scheduler' \ + WHERE e.evaluation_id=w.request_id AND e.phase='requested' AND s.status='running' \ + AND t.snapshot_id=w.snapshot_id AND t.removed_in_snapshot_id IS NULL AND l.expires_at>$3 \ + AND l.epoch=CAST(e.fact_json->'request'->>'epoch' AS BIGINT) \ + AND NOT EXISTS (SELECT 1 FROM sqb_sensor_evaluation_events f \ + WHERE f.evaluation_id=e.evaluation_id AND f.phase<>'requested'))) \ + ORDER BY w.requested_at, w.request_id LIMIT 1", + vec![Parameter::Id(location_id), Parameter::Text(poll.target.clone()), Parameter::Time(now), Parameter::Id(poll.session_id)]).await?; + let Some(row) = rows.first() else { + commit(transaction).await?; + return Ok(None); + }; + let request_id = serde_json::from_value(row["request_id"].clone()) + .map_err(|error| ControlError::Invalid(format!("invalid work identity: {error}")))?; + let (transaction, stored) = read_location_work(transaction, request_id).await?; + let stored = stored.ok_or_else(|| ControlError::Invalid("location work disappeared".into()))?; + let retained = stored.session_id == Some(poll.session_id) + && stored.claimed_until.is_some_and(|until| until > now); + let (claim_id, lease_expires_at) = if retained { + ( + stored.claim_id.ok_or(ControlError::Fenced)?, + stored.claimed_until.ok_or(ControlError::Fenced)?, + ) + } else { + ( + Uuid::now_v7(), + (now + Duration::seconds(60)).min(stored.record.expires_at), + ) + }; + let (transaction, _) = query(transaction, + "UPDATE sqb_location_work SET claim_id=$1, session_id=$2, claimed_until=$3 WHERE request_id=$4", + vec![Parameter::Id(claim_id), Parameter::Id(poll.session_id), Parameter::Time(lease_expires_at), Parameter::Id(request_id)]).await?; + commit(transaction).await?; + Ok(Some(LocationWorkClaim { + request: stored.record.request_json, + generation: stored.record.generation, + claim_id, + session_id: poll.session_id, + lease_expires_at, + })) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs new file mode 100644 index 0000000000..8af79d1995 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs @@ -0,0 +1,81 @@ +//! A completion is immutable; only the current unexpired claim may publish its result. + +use serde_json::json; +use uuid::Uuid; + +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::main::require_instance::require_instance; +use crate::platform::models::{LocationWork, LocationWorkOutcome, LocationWorkResult}; +use crate::sensors::constants::MAX_SENSOR_RUN_REQUESTS; +use crate::sensors::models::CustomSensorOutcome; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn complete_location_work( + store: &ControlStore, + (location_id, request_id): (Uuid, Uuid), + result: &LocationWorkResult, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, stored) = read_location_work(transaction, request_id).await?; + let stored = stored + .filter(|stored| stored.record.request_json.location_id == location_id) + .ok_or_else(|| ControlError::Invalid("work is not assigned to this location".into()))?; + let transaction = require_instance( + transaction, + ( + location_id, + result.session_id, + &stored.record.request_json.target, + ), + ) + .await?; + if let Some(previous) = stored.record.result_json { + if previous != *result { + return Err(ControlError::Invalid( + "location work already has a different result".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + if stored.claim_id != Some(result.claim_id) + || stored.session_id != Some(result.session_id) + || stored + .claimed_until + .is_none_or(|until| until <= transaction.now) + || stored.record.expires_at <= transaction.now + { + return Err(ControlError::Fenced); + } + let valid = match (&stored.record.request_json.work, &result.outcome) { + (_, LocationWorkOutcome::Superseded {}) + | (LocationWork::CheckConnection {}, LocationWorkOutcome::Connection { .. }) => true, + ( + LocationWork::EvaluateSensor { .. }, + LocationWorkOutcome::SensorEvaluation { outcome }, + ) => match outcome { + CustomSensorOutcome::Completed { run_keys, .. } => { + run_keys.len() <= MAX_SENSOR_RUN_REQUESTS + && run_keys.iter().all(|key| !key.trim().is_empty()) + } + CustomSensorOutcome::Skipped { reason, .. } => !reason.trim().is_empty(), + CustomSensorOutcome::Failed | CustomSensorOutcome::TimedOut => true, + }, + _ => false, + }; + if !valid { + return Err(ControlError::Invalid( + "location result does not match its work".into(), + )); + } + let now = transaction.now; + let (transaction, _) = query(transaction, + "UPDATE sqb_location_work SET result_json=$2, completed_at=$3 WHERE request_id=$1 AND completed_at IS NULL", + vec![Parameter::Id(request_id), Parameter::Json(json!(result)), Parameter::Time(now)]).await?; + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/connect_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/connect_location_work.rs new file mode 100644 index 0000000000..a0df47b88d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/connect_location_work.rs @@ -0,0 +1,21 @@ +//! Reuse one credential-bearing pool for the location's outbound work channel. + +use uuid::Uuid; + +use crate::api::errors::ClientError; +use crate::api::main::connect_client::connect_client; +use crate::platform::models::LocationWorkClient; + +pub fn connect_location_work( + (server_url, bearer): (&str, &str), + location_id: Uuid, +) -> Result { + let connection = connect_client(server_url, bearer)?; + let work_url = + connection.endpoint(&["api", "locations", &location_id.to_string(), "work", ""])?; + Ok(LocationWorkClient { + connection, + work_url, + location_id, + }) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/location_health.rs b/crates/sqlbuild-orchestration/src/platform/main/location_health.rs new file mode 100644 index 0000000000..977a61e6c4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/location_health.rs @@ -0,0 +1,63 @@ +//! Read bounded recent observations and instance liveness; consumers derive freshness. + +use uuid::Uuid; + +use crate::platform::models::LocationHealthObservation; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn location_health( + store: &ControlStore, + location_id: Uuid, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let today = transaction.now.date_naive(); + let yesterday = today - chrono::TimeDelta::days(1); + let (transaction, rows) = query( + transaction, + "SELECT received_at, payload FROM (\ + SELECT * FROM (SELECT occurred_at AS received_at, invocation_sequence, payload FROM recall_events \ + WHERE invocation_id=$1 AND event_type='location_health' ORDER BY invocation_sequence DESC LIMIT 100) current_day \ + UNION ALL \ + SELECT * FROM (SELECT occurred_at AS received_at, invocation_sequence, payload FROM recall_events \ + WHERE invocation_id=$2 AND event_type='location_health' ORDER BY invocation_sequence DESC LIMIT 100) previous_day\ + ) observations ORDER BY received_at DESC, invocation_sequence DESC LIMIT 100", + vec![ + Parameter::Text(format!("location/{location_id}/{today}")), + Parameter::Text(format!("location/{location_id}/{yesterday}")), + ], + ) + .await?; + let mut observations: Vec = rows + .into_iter() + .map(|mut row| { + let report = row + .remove("payload") + .and_then(|mut payload| payload.get_mut("report").map(serde_json::Value::take)) + .ok_or_else(|| ControlError::Invalid("location health report is missing".into()))?; + row.insert("report".into(), report); + decode_row(row) + }) + .collect::>()?; + let (transaction, instances) = query( + transaction, + "SELECT last_seen_at AS received_at, health_json AS report FROM sqb_location_instances \ + WHERE location_id=$1 ORDER BY last_seen_at DESC LIMIT 100", + vec![Parameter::Id(location_id)], + ) + .await?; + observations.extend( + instances + .into_iter() + .map(decode_row) + .collect::, _>>()?, + ); + observations.sort_by_key(|observation| std::cmp::Reverse(observation.received_at)); + observations.truncate(100); + commit(transaction).await?; + Ok(observations) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/location_work.rs new file mode 100644 index 0000000000..8a8783c56f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/location_work.rs @@ -0,0 +1,20 @@ +//! Read one request and its durable result without persisting a derived pending/expired status. + +use uuid::Uuid; + +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::models::LocationWorkRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn location_work( + store: &ControlStore, + request_id: Uuid, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, record) = read_location_work(transaction, request_id).await?; + commit(transaction).await?; + Ok(record.map(|stored| stored.record)) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs new file mode 100644 index 0000000000..ecf74ceb6f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -0,0 +1,13 @@ +pub mod claim_location_work; +pub mod complete_location_work; +pub mod connect_location_work; +pub mod location_health; +pub mod location_work; +pub(crate) mod read_location_work; +pub mod read_person_connection_test; +pub(crate) mod read_platform_events; +pub mod report_location_health; +pub mod report_remote_location_health; +pub mod request_location_work; +pub(crate) mod require_instance; +pub mod run_maintenance; diff --git a/crates/sqlbuild-orchestration/src/platform/main/read_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/read_location_work.rs new file mode 100644 index 0000000000..c10305a7b7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/read_location_work.rs @@ -0,0 +1,28 @@ +//! Read durable work inside a caller's existing transaction, including fenced sensor completion. + +use uuid::Uuid; + +use crate::platform::models::StoredLocationWork; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_location_work( + transaction: ControlTransaction, + request_id: Uuid, +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, rows) = query( + transaction, + "SELECT w.request_json, w.requested_by, w.requested_at, w.expires_at, w.generation, \ + w.claim_id, w.session_id, w.claimed_until, w.result_json, w.completed_at \ + FROM sqb_location_work w \ + WHERE w.request_id=$1", + vec![Parameter::Id(request_id)], + ) + .await?; + Ok(( + transaction, + rows.into_iter().next().map(decode_row).transpose()?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/read_person_connection_test.rs b/crates/sqlbuild-orchestration/src/platform/main/read_person_connection_test.rs new file mode 100644 index 0000000000..1bd5f7adbc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/read_person_connection_test.rs @@ -0,0 +1,32 @@ +//! Connection-test results require live action authority and never expose sensor work. + +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::platform::_helpers::connection_tests::authorize_connection_test; +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::models::{LocationWork, LocationWorkRecord}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn read_person_connection_test( + store: &ControlStore, + (context, credentials): (&AuditContext, Uuid), + (location_id, target, request_id): (Uuid, &str, Uuid), +) -> Result, ControlError> { + let (transaction, _) = authorize_connection_test( + begin(store, TransactionMode::Write).await?, + (context, Some(credentials)), + (location_id, target), + ) + .await?; + let (transaction, result) = read_location_work(transaction, request_id).await?; + let result = result.map(|stored| stored.record).filter(|record| { + record.request_json.location_id == location_id + && record.request_json.target == target + && matches!(record.request_json.work, LocationWork::CheckConnection {}) + }); + commit(transaction).await?; + Ok(result) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/read_platform_events.rs b/crates/sqlbuild-orchestration/src/platform/main/read_platform_events.rs new file mode 100644 index 0000000000..48a38850e6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/read_platform_events.rs @@ -0,0 +1,22 @@ +//! One authoritative namespace filter protects every platform stream consumer from worker input. + +use crate::platform::constants::PLATFORM_EVENT_PAGE_SIZE; +use crate::platform::models::PlatformEventEnvelope; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_platform_events( + transaction: ControlTransaction, + after: i64, +) -> Result<(ControlTransaction, Vec), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT storage_id,event_id,occurred_at,payload FROM recall_events WHERE producer='platform' \ + AND invocation_id LIKE 'platform/%' AND storage_id>$1 ORDER BY storage_id LIMIT $2", + vec![Parameter::Integer(after), Parameter::Integer(PLATFORM_EVENT_PAGE_SIZE)]).await?; + Ok(( + transaction, + rows.into_iter().map(decode_row).collect::>()?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs b/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs new file mode 100644 index 0000000000..7d2dad4bc8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs @@ -0,0 +1,110 @@ +//! Append scoped location observations without turning health into mutable lifecycle state. + +use serde_json::json; +use uuid::Uuid; + +use crate::platform::_helpers::location_heartbeat::location_heartbeat; +use crate::platform::models::{LocationHealth, LocationHealthReport}; +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlStore, Parameter, TransactionMode}; + +pub async fn report_location_health( + store: &ControlStore, + location_id: Uuid, + report: &LocationHealthReport, +) -> Result { + if report.target.trim().is_empty() { + return Err(ControlError::Invalid( + "location health target is invalid".into(), + )); + } + let generation = match report.health { + LocationHealth::Heartbeat {} => None, + LocationHealth::CompileFailed { generation } + | LocationHealth::Connection { generation, .. } + | LocationHealth::WarehouseSync { generation, .. } => Some(generation), + }; + if generation.is_some_and(|generation| generation < 1) { + return Err(ControlError::Invalid( + "location health generation must be positive".into(), + )); + } + let transaction = begin(store, TransactionMode::Write).await?; + if generation.is_none() { + let (transaction, applied) = location_heartbeat(transaction, location_id, report).await?; + commit(transaction).await?; + return Ok(applied); + } + let invocation_id = format!("location/{location_id}/{}", transaction.now.date_naive()); + let event_id = Uuid::new_v5(&location_id, report.event_id.as_bytes()); + let payload = json!({"location_id": location_id, "report": report}); + let (transaction, previous) = query( + transaction, + "SELECT payload FROM recall_events WHERE event_id=$1 LIMIT 1", + vec![Parameter::Text(event_id.to_string())], + ) + .await?; + if let Some(previous) = previous.first() { + if previous.get("payload") != Some(&payload) { + return Err(ControlError::Invalid( + "location health event ID already has different content".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + let (transaction, location) = query( + transaction, + "SELECT location_id FROM sqb_locations WHERE location_id=$1 AND archived_at IS NULL", + vec![Parameter::Id(location_id)], + ) + .await?; + if location.is_empty() { + return Err(ControlError::Invalid("active location is required".into())); + } + let transaction = if let LocationHealth::Connection { generation, .. } + | LocationHealth::WarehouseSync { generation, .. } = report.health + { + let (transaction, snapshot) = query( + transaction, + "SELECT snapshot_id FROM sqb_snapshots WHERE location_id=$1 AND target=$2 AND generation=$3", + vec![Parameter::Id(location_id), Parameter::Text(report.target.clone()), Parameter::Integer(generation)], + ).await?; + if snapshot.is_empty() { + let message = if matches!(report.health, LocationHealth::WarehouseSync { .. }) { + "warehouse sync health requires a published generation" + } else { + "connection health requires a published generation" + }; + return Err(ControlError::Invalid(message.into())); + } + transaction + } else { + transaction + }; + let occurred_at = transaction.now; + let (transaction, outcome) = append_control_event( + transaction, + ControlEvent { + event_id, + invocation_id, + event_type: "location_health", + producer: "location", + occurred_at, + run_id: None, + payload, + }, + ) + .await?; + if outcome.conflict.is_some() { + return Err(ControlError::Invalid( + "location health conflicts with a durable fact".into(), + )); + } + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/report_remote_location_health.rs b/crates/sqlbuild-orchestration/src/platform/main/report_remote_location_health.rs new file mode 100644 index 0000000000..ac238b8232 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/report_remote_location_health.rs @@ -0,0 +1,17 @@ +//! The code location has only its scoped HTTP credential, never a store handle. + +use uuid::Uuid; + +use crate::api::errors::ClientError; +use crate::platform::main::connect_location_work::connect_location_work; +use crate::platform::models::LocationHealthReport; + +pub async fn report_remote_location_health( + (server_url, bearer): (&str, &str), + location_id: Uuid, + report: &LocationHealthReport, +) -> Result { + connect_location_work((server_url, bearer), location_id)? + .report_health(report) + .await +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs new file mode 100644 index 0000000000..502f521cf8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs @@ -0,0 +1,92 @@ +//! Named work requires a trusted owner or live person authority; retries preserve the original pin. + +use chrono::Duration; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::platform::_helpers::connection_tests::authorize_connection_test; +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::models::{LocationWork, LocationWorkRequest}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn request_location_work( + store: &ControlStore, + (context, credentials): (&AuditContext, Option), + request: &LocationWorkRequest, +) -> Result { + if matches!(request.work, LocationWork::EvaluateSensor { .. }) { + return Err(ControlError::Invalid( + "sensor work must originate from a fenced evaluation".into(), + )); + } + if request.target.trim().is_empty() + || request.reason.trim().is_empty() + || context.actor.trim().is_empty() + { + return Err(ControlError::Invalid( + "location work needs a target, reason and trusted actor".into(), + )); + } + let (transaction, context) = authorize_connection_test( + begin(store, TransactionMode::Write).await?, + (context, credentials), + (request.location_id, &request.target), + ) + .await?; + let (transaction, previous) = read_location_work(transaction, request.request_id).await?; + if let Some(previous) = previous { + if previous.record.request_json != *request || previous.record.requested_by != context.actor + { + return Err(ControlError::Invalid( + "location work ID already has different content or actor".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + let (transaction, rows) = query(transaction, + "SELECT s.snapshot_id, s.generation FROM sqb_current_snapshots c \ + JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id JOIN sqb_locations l ON l.location_id=c.location_id \ + WHERE c.location_id=$1 AND c.target=$2 AND l.archived_at IS NULL", + vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone())]).await?; + let snapshot = rows + .first() + .ok_or_else(|| ControlError::Invalid("location work requires a published target".into()))?; + let snapshot_id = serde_json::from_value(snapshot["snapshot_id"].clone()) + .map_err(|error| ControlError::Invalid(format!("invalid snapshot identity: {error}")))?; + let generation = snapshot["generation"] + .as_i64() + .ok_or_else(|| ControlError::Invalid("invalid snapshot generation".into()))?; + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_location_work \ + (request_id,location_id,target,snapshot_id,generation,request_json,requested_by,requested_at,expires_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)", + vec![Parameter::Id(request.request_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), + Parameter::Id(snapshot_id), Parameter::Integer(generation), Parameter::Json(json!(request)), + Parameter::Text(context.actor.clone()), Parameter::Time(now), Parameter::Time(now + Duration::minutes(5))]).await?; + commit( + append_audit( + transaction, + &context, + ( + Uuid::new_v5(&request.request_id, b"location-work-requested"), + "location.work_requested", + ), + ( + json!({"location_id": request.location_id, "target": request.target}), + json!(null), + json!(request), + ), + ) + .await?, + ) + .await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/require_instance.rs b/crates/sqlbuild-orchestration/src/platform/main/require_instance.rs new file mode 100644 index 0000000000..9d64b6068f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/require_instance.rs @@ -0,0 +1,23 @@ +//! Named work and warehouse observations share the registered location/target boundary. + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn require_instance( + transaction: ControlTransaction, + (location_id, session_id, target): (Uuid, Uuid, &str), +) -> Result { + let (transaction, rows) = query(transaction, + "SELECT i.instance_id FROM sqb_location_instances i JOIN sqb_locations l ON l.location_id=i.location_id \ + WHERE i.instance_id=$1 AND i.location_id=$2 AND i.target=$3 AND l.archived_at IS NULL", + vec![Parameter::Id(session_id), Parameter::Id(location_id), Parameter::Text(target.into())]).await?; + if rows.is_empty() { + return Err(ControlError::Invalid( + "a registered instance of this location and target is required".into(), + )); + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/run_maintenance.rs b/crates/sqlbuild-orchestration/src/platform/main/run_maintenance.rs new file mode 100644 index 0000000000..422b2974df --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/run_maintenance.rs @@ -0,0 +1,54 @@ +//! One maintenance role per server: health alerts every cycle, bounded retention every minute. + +use std::time::{Duration, Instant}; + +use tokio::sync::watch; +use uuid::Uuid; + +use crate::credentials::models::AuditRetention; +use crate::platform::_helpers::retention::retain; +use crate::platform::_helpers::watchdog::scan_health; +use crate::snapshots::models::SnapshotRetention; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::models::StatisticsRetention; +use crate::webhooks::models::{Webhook, WebhookSource}; + +/// Only this server's security-audit sinks acknowledge; `report` hears each new retained count. +pub async fn run_maintenance( + (store, replica_id): (&ControlStore, Uuid), + (retention, sinks): ( + (AuditRetention, StatisticsRetention, SnapshotRetention), + &[Webhook], + ), + (report, mut shutdown): (impl Fn(i64), watch::Receiver), +) -> Result<(), ControlError> { + let sinks: Vec = sinks + .iter() + .filter(|sink| matches!(sink.source, WebhookSource::SecurityAudit)) + .map(|sink| sink.sink_id.clone()) + .collect(); + let mut reported = 0; + let mut retained_at: Option = None; + while !*shutdown.borrow() { + scan_health(store, replica_id, &shutdown).await?; + if retained_at.is_none_or(|at| at.elapsed() >= Duration::from_secs(60)) { + match retain(store, retention, &sinks).await { + Ok(retained) => { + if retained != reported { + report(retained); + } + reported = retained; + } + Err(ControlError::Database(error)) if ControlError::transient_database(&error) => {} + Err(error) => return Err(error), + } + retained_at = Some(Instant::now()); + } + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(30)) => {}, + result = shutdown.changed() => { if result.is_err() { break; } }, + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/platform/mod.rs b/crates/sqlbuild-orchestration/src/platform/mod.rs new file mode 100644 index 0000000000..d58521e0f6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +#[path = "../../tests/platform/test_contracts.rs"] +mod tests; diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs new file mode 100644 index 0000000000..543aaeb420 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -0,0 +1,252 @@ +//! Platform health is a typed stream of observations, independent of user code. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::api::models::ControlClient; +use crate::declarations::models::TriggerDefinition; +use crate::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; +use crate::warehouse_metadata::models::{WarehouseHistoryKind, WarehouseStateLocation}; + +/// One pooled outbound connection; the location never receives a store handle. +#[derive(Clone, Debug)] +pub struct LocationWorkClient { + pub(crate) connection: ControlClient, + pub(crate) work_url: reqwest::Url, + pub(crate) location_id: Uuid, +} + +/// Locations report bounded facts, never connection strings or user exception text. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum LocationHealth { + Heartbeat {}, + CompileFailed { + generation: i64, + }, + Connection { + generation: i64, + healthy: bool, + }, + WarehouseSync { + generation: i64, + location: WarehouseStateLocation, + table: WarehouseHistoryKind, + failure: Option, + records_sent: u64, + elapsed_millis: u64, + }, +} + +/// Closed failure stages keep warehouse errors and credentials out of health observations. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum WarehouseSyncFailure { + Watermark, + Read, + Delivery, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct LocationHealthReport { + pub event_id: Uuid, + pub session_id: Uuid, + pub target: String, + pub health: LocationHealth, +} + +/// Receipt time belongs to the server; consumers derive freshness rather than storing availability. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct LocationHealthObservation { + pub received_at: DateTime, + pub report: LocationHealthReport, +} + +/// Only named location operations cross this boundary, never arbitrary SQL or Python. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum LocationWork { + CheckConnection {}, + EvaluateSensor { + #[schema(value_type = Object)] + evaluation: Box, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct LocationWorkRequest { + pub request_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub reason: String, + pub work: LocationWork, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct ConnectionTestRequest { + pub request_id: Uuid, + pub reason: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct LocationWorkPoll { + pub session_id: Uuid, + pub target: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct LocationWorkClaim { + pub request: LocationWorkRequest, + pub generation: i64, + pub claim_id: Uuid, + pub session_id: Uuid, + pub lease_expires_at: DateTime, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum LocationWorkOutcome { + Connection { healthy: bool }, + SensorEvaluation { outcome: CustomSensorOutcome }, + Superseded {}, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct LocationWorkResult { + pub claim_id: Uuid, + pub session_id: Uuid, + pub outcome: LocationWorkOutcome, +} + +/// Expiry is derived from the deadline; only requests, leases and completed results are stored. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct LocationWorkRecord { + pub request_json: LocationWorkRequest, + pub requested_by: String, + pub requested_at: DateTime, + pub expires_at: DateTime, + pub generation: i64, + pub result_json: Option, + pub completed_at: Option>, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct StoredLocationWork { + #[serde(flatten)] + pub(crate) record: LocationWorkRecord, + pub(crate) claim_id: Option, + pub(crate) session_id: Option, + pub(crate) claimed_until: Option>, +} + +#[derive( + Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum PlatformEventKind { + Heartbeat, + SchedulerNoLeader, + TickOverdue, + CalendarHorizonLow, + LocationBroken, + LaunchFailed, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum PlatformEvent { + Heartbeat { + replica_id: Uuid, + }, + SchedulerNoLeader, + TickOverdue { + trigger_id: Uuid, + scheduled_for: DateTime, + }, + CalendarHorizonLow { + trigger_id: Uuid, + refresh_at: DateTime, + horizon_end: Option>, + }, + /// All observed instances of this location/target have missed three heartbeats. + LocationBroken { + location_id: Uuid, + target: String, + last_seen_at: DateTime, + }, + /// A launch attempt or its response failed; the worker may still start or finish. + LaunchFailed { + run_id: Uuid, + claim_id: Uuid, + code: String, + }, +} + +impl PlatformEvent { + #[must_use] + pub const fn kind(&self) -> PlatformEventKind { + match self { + Self::Heartbeat { .. } => PlatformEventKind::Heartbeat, + Self::SchedulerNoLeader => PlatformEventKind::SchedulerNoLeader, + Self::TickOverdue { .. } => PlatformEventKind::TickOverdue, + Self::CalendarHorizonLow { .. } => PlatformEventKind::CalendarHorizonLow, + Self::LocationBroken { .. } => PlatformEventKind::LocationBroken, + Self::LaunchFailed { .. } => PlatformEventKind::LaunchFailed, + } + } + + #[must_use] + pub const fn event_type(&self) -> &'static str { + match self { + Self::Heartbeat { .. } => "platform_heartbeat", + Self::SchedulerNoLeader => "platform_scheduler_no_leader", + Self::TickOverdue { .. } => "platform_tick_overdue", + Self::CalendarHorizonLow { .. } => "platform_calendar_horizon_low", + Self::LocationBroken { .. } => "platform_location_broken", + Self::LaunchFailed { .. } => "platform_launch_failed", + } + } +} + +#[derive(Debug, Deserialize)] +pub(crate) struct InstallationHealth { + pub(crate) installation_id: Uuid, + /// Distinguishes one leaderless outage from the next. + pub(crate) lease_expires_at: Option>, + pub(crate) leader: i64, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct WatchdogTrigger { + pub(crate) trigger_id: Uuid, + pub(crate) definition_json: TriggerDefinition, + pub(crate) next_due_at: Option>, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct WatchdogLocation { + pub(crate) location_id: Uuid, + pub(crate) target: String, + pub(crate) last_seen_at: DateTime, +} + +/// Stable event identity permits receiver-side deduplication after uncertain acknowledgements. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PlatformEventEnvelope { + pub storage_id: i64, + pub event_id: Uuid, + pub occurred_at: DateTime, + pub payload: PlatformEventPayload, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PlatformEventPayload { + pub platform: PlatformEvent, +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs new file mode 100644 index 0000000000..6b7c17d485 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs @@ -0,0 +1,108 @@ +//! A group takes every child's limits when admitted, so no admitted run waits on another limit. + +use std::cmp::Reverse; +use std::collections::{BTreeMap, BTreeSet}; + +use uuid::Uuid; + +use crate::runs::models::{LimitWaiter, QueueCandidate, RunKind, RunRequest}; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::ControlTransaction; + +/// Limits a run takes on admission; group children share their parent's, one child per limit at a time. +pub(crate) fn admission_limits<'a>( + request: &RunRequest, + children: impl IntoIterator, +) -> BTreeSet { + if request.parent_run_id.is_some() { + return BTreeSet::new(); + } + let mut limits = request.concurrency_groups.clone(); + for child in children { + limits.extend(child.concurrency_groups.iter().cloned()); + } + limits +} + +/// Queued top-level runs in dispatchable scopes, with the limits each will take. +pub(crate) async fn limit_waiters( + transaction: ControlTransaction, + eligible: Option<&BTreeSet<(Uuid, String)>>, +) -> Result<(ControlTransaction, Vec), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT r.request_json, r.queued_at FROM sqb_runs r WHERE r.status='queued' AND r.parent_run_id IS NULL \ + AND r.cancel_requested_at IS NULL AND r.worker_instance IS NULL", + vec![]).await?; + let waiting: Vec = rows + .into_iter() + .map(decode_row) + .collect::, _>>()? + .into_iter() + .filter(|run| { + eligible.is_none_or(|scopes| { + scopes.contains(&( + run.request_json.location_id, + run.request_json.target.clone(), + )) + }) + }) + .collect(); + let (transaction, rows) = query( + transaction, + "SELECT c.request_json FROM sqb_runs c JOIN sqb_runs p ON p.run_id=c.parent_run_id \ + WHERE p.kind='group' AND p.status='queued' AND p.cancel_requested_at IS NULL", + vec![], + ) + .await?; + let mut children: BTreeMap> = BTreeMap::new(); + for row in rows { + let child: RunRequest = serde_json::from_value(row["request_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if let Some(parent) = child.parent_run_id { + children.entry(parent).or_default().push(child); + } + } + let waiters: Vec = waiting + .into_iter() + .map(|run| { + let request = &run.request_json; + let members = if request.kind == RunKind::Group { + children + .get(&request.run_id) + .map(Vec::as_slice) + .unwrap_or_default() + } else { + &[] + }; + LimitWaiter { + rank: rank(&run), + run_id: request.run_id, + limits: admission_limits(request, members), + } + }) + .collect(); + Ok((transaction, waiters)) +} + +/// Strict per-limit queue order: an earlier waiter for any of the same limits goes first. +pub(crate) fn earlier_waiter_blocks( + waiters: &[LimitWaiter], + candidate: &QueueCandidate, + limits: &BTreeSet, +) -> bool { + let position = rank(candidate); + waiters.iter().any(|waiter| { + waiter.run_id != candidate.request_json.run_id + && waiter.rank < position + && !waiter.limits.is_disjoint(limits) + }) +} + +fn rank(run: &QueueCandidate) -> (Reverse, chrono::DateTime, Uuid) { + ( + Reverse(run.request_json.priority), + run.queued_at, + run.request_json.run_id, + ) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/context.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/context.rs new file mode 100644 index 0000000000..94c73eb962 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/context.rs @@ -0,0 +1,69 @@ +//! Read group membership and failure policy from the parent's immutable publication. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::declarations::main::frozen_definitions::frozen_definitions; +use crate::runs::models::{GroupContext, GroupRun}; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn load_group( + transaction: ControlTransaction, + parent_id: Uuid, +) -> Result<(ControlTransaction, GroupContext), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT r.request_json, r.queued_at, r.status, r.cancel_requested_at, r.node_counts_json, r.worker_instance FROM sqb_runs r \ + WHERE r.run_id=$1 AND r.kind='group'", + vec![Parameter::Id(parent_id)]).await?; + let parent: GroupRun = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("group run is missing".into()))?, + )?; + let (transaction, rows) = query(transaction, + "SELECT r.request_json, r.queued_at, r.status, r.cancel_requested_at, r.node_counts_json, r.worker_instance FROM sqb_runs r \ + WHERE r.parent_run_id=$1 ORDER BY r.run_id", + vec![Parameter::Id(parent_id)]).await?; + let children = rows.into_iter().map(decode_row).collect::>()?; + let (transaction, definitions) = + frozen_definitions(transaction, parent.request_json.snapshot_id).await?; + let (transaction, failures) = query( + transaction, + "SELECT n.node_key FROM sqb_run_nodes n JOIN sqb_runs r ON r.run_id=n.run_id \ + WHERE r.parent_run_id=$1 AND n.kind='node' AND n.status='failed'", + vec![Parameter::Id(parent_id)], + ) + .await?; + let failures: BTreeSet = failures + .iter() + .filter_map(|row| row["node_key"].as_str().map(str::to_owned)) + .collect(); + let (transaction, nodes) = read_snapshot_nodes( + transaction, + parent.request_json.snapshot_id, + Some(&failures), + ) + .await?; + let critical_failed = nodes + .values() + .any(|node| node.definition["config"]["critical"].as_bool() == Some(true)); + Ok(( + transaction, + GroupContext { + parent, + children, + definitions, + critical_failed, + }, + )) +} + +impl GroupContext { + pub(crate) const fn stopped(&self) -> bool { + self.critical_failed + } +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs new file mode 100644 index 0000000000..e43c5c3acd --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs @@ -0,0 +1,208 @@ +//! A virtual parent advances only from durable child evidence under the scheduler's fence. + +use std::collections::{BTreeMap, BTreeSet}; +use std::num::NonZeroU32; + +use uuid::Uuid; + +use crate::runs::_helpers::groups::admission::{ + admission_limits, earlier_waiter_blocks, limit_waiters, +}; +use crate::runs::_helpers::groups::context::load_group; +use crate::runs::_helpers::lifecycle::claims::reserve_claims; +use crate::runs::_helpers::lifecycle::publish_change::publish_change; +use crate::runs::models::{ + ChangingRun, GroupContext, NodeOutcome, QueueCandidate, RunChange, RunChangeRequest, + RunNodeRecord, RunOutcome, RunRequest, RunStatus, +}; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Callers have already checked leadership; the surrounding transaction fences the entire advance. +pub(crate) async fn group_change( + transaction: ControlTransaction, + run_id: Uuid, + change: RunChange, +) -> Result { + let (transaction, rows) = query(transaction, + "SELECT r.status, r.cancel_requested_at, r.node_counts_json, r.worker_instance FROM sqb_runs r WHERE r.run_id=$1", + vec![Parameter::Id(run_id)]).await?; + let state: ChangingRun = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("group run is missing".into()))?, + )?; + let event_id = Uuid::new_v5( + &run_id, + &serde_json::to_vec(&change).map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + publish_change( + transaction, + &RunChangeRequest { + event_id, + run_id, + claim_id: state.worker_instance, + change, + }, + &state, + ) + .await +} + +pub(crate) async fn advance_group( + transaction: ControlTransaction, + parent_id: Uuid, + limits: &BTreeMap, + eligible: Option<&BTreeSet<(Uuid, String)>>, +) -> Result { + let (mut transaction, group) = load_group(transaction, parent_id).await?; + let parent = &group.parent; + let children = &group.children; + if parent.state.status.is_terminal() { + return Ok(transaction); + } + let stopped = group.stopped(); + if parent.state.cancel_requested_at.is_some() || stopped { + for child in children { + if !child.state.status.is_terminal() && child.state.cancel_requested_at.is_none() { + transaction = group_change( + transaction, + child.request_json.run_id, + RunChange::CancelRequested { + actor: "system:group".into(), + reason: if stopped { + "group stopped after a child failure" + } else { + "parent group cancellation requested" + } + .into(), + }, + ) + .await?; + } + } + } else if parent.state.status == RunStatus::Queued { + let admission = RunRequest { + concurrency_groups: admission_limits( + &parent.request_json, + children.iter().map(|child| &child.request_json), + ), + ..parent.request_json.clone() + }; + let waiters; + (transaction, waiters) = limit_waiters(transaction, eligible).await?; + let candidate = QueueCandidate { + request_json: parent.request_json.clone(), + queued_at: parent.queued_at, + }; + if earlier_waiter_blocks(&waiters, &candidate, &admission.concurrency_groups) { + return Ok(transaction); + } + (transaction, _) = query(transaction, "SAVEPOINT group_claim", vec![]).await?; + let reserved; + (transaction, reserved) = reserve_claims(transaction, &admission, limits).await?; + if !reserved { + (transaction, _) = + query(transaction, "ROLLBACK TO SAVEPOINT group_claim", vec![]).await?; + } else { + transaction = group_change(transaction, parent_id, RunChange::Started).await?; + } + (transaction, _) = query(transaction, "RELEASE SAVEPOINT group_claim", vec![]).await?; + } + if !children.is_empty() + && children + .iter() + .all(|child| child.state.status.is_terminal()) + { + transaction = finish_group(transaction, &group, stopped).await?; + } + Ok(transaction) +} + +async fn finish_group( + transaction: ControlTransaction, + group: &GroupContext, + stopped: bool, +) -> Result { + let parent = &group.parent; + let parent_id = parent.request_json.run_id; + let (mut transaction, rows) = query(transaction, + "SELECT n.* FROM sqb_run_nodes n JOIN sqb_runs r ON r.run_id=n.run_id WHERE r.parent_run_id=$1 AND n.kind='node' ORDER BY r.run_id, n.node_key", + vec![Parameter::Id(parent_id)]).await?; + let mut nodes: BTreeMap = BTreeMap::new(); + let rank = |status: &str| match status { + "failed" => 5, + "cancelled" => 4, + "skipped" => 3, + "warn" => 2, + "success" => 1, + _ => 0, + }; + for row in rows { + let node: RunNodeRecord = decode_row(row)?; + if nodes + .get(&node.node_key) + .is_none_or(|previous| rank(&node.status) > rank(&previous.status)) + { + nodes.insert(node.node_key.clone(), node); + } + } + let mut outcomes: BTreeSet = BTreeSet::new(); + for key in &parent.request_json.selection { + let node = nodes + .get(key) + .ok_or_else(|| ControlError::Invalid("group node has no child evidence".into()))?; + outcomes.insert(node.status.clone()); + let outcome: NodeOutcome = serde_json::from_value(serde_json::json!(node.status)) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + transaction = group_change( + transaction, + parent_id, + RunChange::NodeFinished { + node_key: key.clone(), + outcome, + rows_affected: node.rows_affected, + error_code: node.error_code.clone(), + skipped_because_of: node.skipped_because_of.clone(), + }, + ) + .await?; + } + let good = outcomes.contains("success") || outcomes.contains("warn"); + let bad = group + .children + .iter() + .any(|child| !matches!(child.state.status, RunStatus::Success | RunStatus::Warn)) + || outcomes + .iter() + .any(|status| !matches!(status.as_str(), "success" | "warn")); + let outcome = if parent.state.cancel_requested_at.is_some() { + RunOutcome::Cancelled + } else if stopped { + RunOutcome::Failed + } else if bad && good { + RunOutcome::Partial + } else if bad { + RunOutcome::Failed + } else if outcomes.contains("warn") + || group + .children + .iter() + .any(|child| child.state.status == RunStatus::Warn) + { + RunOutcome::Warn + } else { + RunOutcome::Success + }; + group_change( + transaction, + parent_id, + RunChange::Finished { + outcome, + reason: "all group children have durable terminal outcomes".into(), + exit_code: None, + }, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/mod.rs new file mode 100644 index 0000000000..6668ea7c28 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/mod.rs @@ -0,0 +1,4 @@ +pub(in crate::runs) mod admission; +pub(in crate::runs) mod context; +pub(in crate::runs) mod lifecycle; +pub(in crate::runs) mod selection; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs new file mode 100644 index 0000000000..a4a6f16c6c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs @@ -0,0 +1,110 @@ +//! Failed nodes block only their lineage dependents inside a group. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::runs::_helpers::groups::context::load_group; +use crate::runs::models::{RunRequest, RunStatus}; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn child_ready( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result<(ControlTransaction, bool), ControlError> { + let Some(parent_id) = request.parent_run_id else { + return Ok((transaction, true)); + }; + let (transaction, group) = load_group(transaction, parent_id).await?; + let name = request + .job_key + .as_ref() + .ok_or_else(|| ControlError::Invalid("group child has no job".into()))?; + let parent_name = group + .parent + .request_json + .job_key + .as_ref() + .ok_or_else(|| ControlError::Invalid("group has no declaration".into()))?; + let dependencies = upstream_jobs(&group.definitions.groups[parent_name].jobs, name); + let ready = group.parent.state.status == RunStatus::Running + && group.parent.state.cancel_requested_at.is_none() + && !group.stopped() + && group.children.iter().all(|child| { + child + .request_json + .job_key + .as_ref() + .is_none_or(|name| !dependencies.contains(name.as_str())) + || child.state.status.is_terminal() + }); + Ok((transaction, ready)) +} + +pub(crate) async fn skipped_nodes( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result<(ControlTransaction, BTreeMap), ControlError> { + let Some(parent_id) = request.parent_run_id else { + return Ok((transaction, BTreeMap::new())); + }; + let (transaction, rows) = query(transaction, + "SELECT n.node_key, n.status, n.skipped_because_of FROM sqb_run_nodes n JOIN sqb_runs r ON r.run_id=n.run_id \ + WHERE r.parent_run_id=$1 AND r.run_id<>$2 AND n.kind='node' AND n.status IN ('failed','skipped','cancelled')", + vec![Parameter::Id(parent_id), Parameter::Id(request.run_id)]).await?; + let failures: BTreeMap = rows + .into_iter() + .filter_map(|row| { + let key = row.get("node_key")?.as_str()?.to_owned(); + let cause = row + .get("skipped_because_of") + .and_then(serde_json::Value::as_str) + .unwrap_or(&key) + .to_owned(); + Some((key, cause)) + }) + .collect(); + if failures.is_empty() { + return Ok((transaction, BTreeMap::new())); + } + let (transaction, nodes) = read_snapshot_nodes(transaction, request.snapshot_id, None).await?; + let mut skipped: BTreeMap = BTreeMap::new(); + for selected in &request.selection { + let mut seen: BTreeSet<&str> = BTreeSet::new(); + let mut pending = vec![selected.as_str()]; + while let Some(key) = pending.pop() { + if !seen.insert(key) { + continue; + } + if let Some(cause) = failures.get(key) { + skipped.insert(selected.clone(), cause.clone()); + break; + } + if let Some(node) = nodes.get(key) + && let Some(deps) = node + .definition + .get("deps") + .and_then(serde_json::Value::as_array) + { + pending.extend(deps.iter().filter_map(serde_json::Value::as_str)); + } + } + } + Ok((transaction, skipped)) +} + +/// Completed jobs omitted from a retry do not sever explicit ordering between retained jobs. +fn upstream_jobs<'a>( + edges: &'a BTreeMap>, + name: &str, +) -> BTreeSet<&'a str> { + let mut ancestors: BTreeSet<&str> = BTreeSet::new(); + let mut pending: Vec<&str> = edges[name].iter().map(String::as_str).collect(); + while let Some(name) = pending.pop() { + if ancestors.insert(name) { + pending.extend(edges[name].iter().map(String::as_str)); + } + } + ancestors +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/change_run.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/change_run.rs new file mode 100644 index 0000000000..64b822b237 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/change_run.rs @@ -0,0 +1,82 @@ +//! Retry-safe facts, lifecycle projection and terminal claim release form one transaction. + +use serde_json::json; + +use crate::credentials::main::audit::append_audit::append_audit; +use crate::runs::_helpers::lifecycle::changes::{ + check_launch_claim, validate_authority, validate_change, validate_retry, +}; +use crate::runs::_helpers::lifecycle::publish_change::publish_change; +use crate::runs::models::{ChangingRun, RunAuthority, RunChange, RunChangeRequest}; +use crate::scheduler::main::check_lease::check_lease; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Use authenticated authority and record unclaimed cancellation as terminal in the same transaction. +pub(crate) async fn change_run_in( + mut transaction: ControlTransaction, + authority: RunAuthority<'_>, + request: &RunChangeRequest, +) -> Result<(ControlTransaction, bool), ControlError> { + validate_authority(authority, request)?; + if let RunAuthority::Scheduler(lease) = authority { + transaction = check_lease(transaction, lease).await?; + } + let body = json!({"change": request}); + let (transaction, previous) = query( + transaction, + "SELECT invocation_id, payload FROM recall_events WHERE event_id=$1 LIMIT 1", + vec![Parameter::Text(request.event_id.to_string())], + ) + .await?; + if let Some(previous) = previous.first() { + validate_retry(previous, request, &body)?; + return Ok((transaction, false)); + } + let (mut transaction, rows) = query( + transaction, + "SELECT r.status, r.cancel_requested_at, r.node_counts_json, r.worker_instance FROM sqb_runs r WHERE r.run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + let state: ChangingRun = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("run does not exist".into()))?, + )?; + if matches!(request.change, RunChange::LaunchFailed { .. }) { + transaction = check_launch_claim(transaction, request).await?; + } else { + validate_change(&state, request)?; + } + let mut transaction = publish_change(transaction, request, &state).await?; + if let (RunAuthority::Operator(context), RunChange::CancelRequested { reason, .. }) = + (authority, &request.change) + { + let mut rows; + (transaction, rows) = query( + transaction, + "SELECT location_id, target, status, cancel_requested_at FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + let after = rows.remove(0); + transaction = append_audit( + transaction, + context, + (request.event_id, "run.cancel_requested"), + ( + json!({"location_id": after["location_id"], "target": after["target"], "run_id": request.run_id}), + json!({"status": state.status, "cancel_requested_at": state.cancel_requested_at}), + json!({"status": after["status"], "cancel_requested_at": after["cancel_requested_at"], "reason": reason}), + ), + ) + .await?; + } + if let RunAuthority::Scheduler(lease) = authority { + transaction = check_lease(transaction, lease).await?; + } + Ok((transaction, true)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs new file mode 100644 index 0000000000..c71085d635 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs @@ -0,0 +1,182 @@ +//! Lifecycle authority checks precede fact publication and physical claim release. + +use crate::runs::models::{ + ChangingRun, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; +use crate::store::types::Row; + +/// A launch-response observation is append-only even after completion, but requires recorded ownership. +pub(crate) async fn check_launch_claim( + transaction: ControlTransaction, + request: &RunChangeRequest, +) -> Result { + let claim_id = request.claim_id.ok_or_else(|| { + ControlError::Invalid("launch observation requires a recorded claim".into()) + })?; + let (transaction, rows) = query( + transaction, + "SELECT payload FROM recall_events WHERE event_id=$1 AND run_id=$2 \ + AND event_type='run_claimed' AND producer='scheduler' AND invocation_id=$3", + vec![ + Parameter::Text(claim_id.to_string()), + Parameter::Text(request.run_id.to_string()), + Parameter::Text(format!("scheduler/{}", request.run_id)), + ], + ) + .await?; + let recorded = rows.first().map(|row| &row["payload"]["claim"]); + if recorded.is_none_or(|claim| { + claim["claim_id"] != claim_id.to_string() + || claim["request"]["run_id"] != request.run_id.to_string() + }) { + return Err(ControlError::Invalid( + "launch observation does not belong to a recorded claim".into(), + )); + } + Ok(transaction) +} + +pub(crate) fn validate_retry( + previous: &Row, + request: &RunChangeRequest, + body: &serde_json::Value, +) -> Result<(), ControlError> { + if previous.get("payload") != Some(body) + || previous + .get("invocation_id") + .and_then(serde_json::Value::as_str) + != Some(format!("scheduler/{}", request.run_id).as_str()) + { + return Err(ControlError::Invalid( + "run change event ID already has different content".into(), + )); + } + Ok(()) +} + +pub(crate) fn validate_authority( + authority: RunAuthority<'_>, + request: &RunChangeRequest, +) -> Result<(), ControlError> { + let allowed = match authority { + RunAuthority::Scheduler(_) => !matches!( + request.change, + RunChange::CancelRequested { .. } + | RunChange::Started + | RunChange::NodeStarted { .. } + | RunChange::NodeFinished { .. } + | RunChange::CheckStarted { .. } + | RunChange::CheckFinished { .. } + ), + RunAuthority::Worker { run_id, claim_id } => { + run_id == request.run_id + && request.claim_id == Some(claim_id) + && matches!( + request.change, + RunChange::Started + | RunChange::NodeStarted { .. } + | RunChange::NodeFinished { .. } + | RunChange::CheckStarted { .. } + | RunChange::CheckFinished { .. } + | RunChange::Recovered + | RunChange::Finished { + outcome: RunOutcome::Success + | RunOutcome::Warn + | RunOutcome::Partial + | RunOutcome::Failed + | RunOutcome::Cancelled, + .. + } + ) + } + RunAuthority::Operator(context) => { + matches!(&request.change, RunChange::CancelRequested { actor, reason } + if !context.actor.trim().is_empty() && context.actor == *actor + && !context.granting_role.trim().is_empty() && !reason.trim().is_empty()) + } + }; + if !allowed { + return Err(ControlError::Invalid( + "authority cannot publish this run change".into(), + )); + } + Ok(()) +} + +pub(crate) fn validate_change( + state: &ChangingRun, + request: &RunChangeRequest, +) -> Result<(), ControlError> { + use RunStatus::{Launched, Queued, Running, Unresponsive}; + if !matches!(state.status, Queued | Launched | Running | Unresponsive) { + return Err(ControlError::Invalid("terminal runs cannot change".into())); + } + if !matches!(request.change, RunChange::CancelRequested { .. }) + && request.claim_id != state.worker_instance + { + return Err(ControlError::Invalid( + "run change does not belong to the current claim".into(), + )); + } + let valid = match request.change { + RunChange::LaunchFailed { .. } => false, + RunChange::Launched { .. } => state.worker_instance.is_some(), + RunChange::Started => { + state.worker_instance.is_some() && matches!(state.status, Queued | Launched) + } + RunChange::NodeStarted { .. } + | RunChange::NodeFinished { .. } + | RunChange::CheckStarted { .. } + | RunChange::CheckFinished { .. } => { + matches!(state.status, Running | Unresponsive) + } + RunChange::Unresponsive => state.status == Running, + RunChange::Recovered => state.status == Unresponsive, + RunChange::CancelRequested { .. } => state.cancel_requested_at.is_none(), + RunChange::Finished { outcome, .. } => { + state.worker_instance.is_some() + || matches!(outcome, RunOutcome::Failed | RunOutcome::Cancelled) + } + }; + if !valid { + return Err(ControlError::Invalid( + "run change is inconsistent with the current lifecycle".into(), + )); + } + if let RunChange::Finished { + outcome: RunOutcome::Success | RunOutcome::Warn | RunOutcome::Partial, + .. + } = request.change + { + let count = |status: &str| { + state + .node_counts_json + .get(status) + .copied() + .unwrap_or_default() + }; + let good = count("success") + count("warn"); + let bad = count("failed") + count("skipped") + count("cancelled"); + let expected = if bad > 0 && good > 0 { + RunOutcome::Partial + } else if bad > 0 { + RunOutcome::Failed + } else if count("warn") > 0 { + RunOutcome::Warn + } else { + RunOutcome::Success + }; + if count("pending") > 0 + || count("running") > 0 + || !matches!(request.change, RunChange::Finished { outcome, .. } if outcome == expected) + { + return Err(ControlError::Invalid( + "run outcome does not match its completed node facts".into(), + )); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/checks.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/checks.rs new file mode 100644 index 0000000000..b54f550a6c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/checks.rs @@ -0,0 +1,64 @@ +//! Attached checks are recorded beside nodes, but never join the frozen execution selection. + +use crate::runs::constants::NODE_RUNNING; +use crate::runs::models::{NodeOutcome, RunChange, RunChangeRequest}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn check_execution_history( + transaction: ControlTransaction, + request: &RunChangeRequest, +) -> Result { + let (node_key, outcome) = match &request.change { + RunChange::CheckStarted { node_key } => (node_key, None), + RunChange::CheckFinished { node_key, outcome } => (node_key, Some(*outcome)), + _ => return Err(ControlError::Invalid("check fact required".into())), + }; + let (transaction, nodes) = query(transaction, + "SELECT n.node_key FROM sqb_runs r JOIN sqb_snapshot_nodes n ON n.snapshot_id=r.snapshot_id \ + WHERE r.run_id=$1 AND n.node_key=$2 AND n.node_kind='check' LIMIT 1", + vec![Parameter::Id(request.run_id), Parameter::Text(node_key.clone())]).await?; + if !node_key.starts_with("check:") || nodes.is_empty() { + return Err(ControlError::Invalid( + "check is outside the frozen snapshot".into(), + )); + } + let (transaction, previous) = query( + transaction, + "SELECT status FROM sqb_run_nodes WHERE run_id=$1 AND node_key=$2 AND kind='check'", + vec![ + Parameter::Id(request.run_id), + Parameter::Text(node_key.clone()), + ], + ) + .await?; + let previous = previous.first().and_then(|row| row["status"].as_str()); + let valid = match outcome { + None => previous.is_none(), + Some(NodeOutcome::Skipped) => previous.is_none() || previous == Some(NODE_RUNNING), + Some(_) => previous == Some(NODE_RUNNING), + }; + if !valid { + return Err(ControlError::Invalid( + "check fact is inconsistent with its execution history".into(), + )); + } + let now = transaction.now; + let status = outcome.map_or(NODE_RUNNING, |outcome| outcome.as_str()); + let (transaction, _) = query( + transaction, + "INSERT INTO sqb_run_nodes (run_id, node_key, status, writes, started_at, ended_at, kind) \ + VALUES ($1,$2,$3,FALSE,$4,$5,'check') ON CONFLICT (run_id, node_key) \ + DO UPDATE SET status=excluded.status, ended_at=excluded.ended_at", + vec![ + Parameter::Id(request.run_id), + Parameter::Text(node_key.clone()), + Parameter::Text(status.into()), + Parameter::OptionalTime(outcome.is_none().then_some(now)), + Parameter::OptionalTime(outcome.is_some().then_some(now)), + ], + ) + .await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/claims.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/claims.rs new file mode 100644 index 0000000000..952a7ae359 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/claims.rs @@ -0,0 +1,159 @@ +//! Reserve write claims and concurrency slots inside the candidate's rollback savepoint. + +use std::collections::{BTreeMap, BTreeSet}; +use std::num::NonZeroU32; + +use serde_json::json; +use uuid::Uuid; + +use crate::runs::_helpers::lifecycle::events::append_run_event; +use crate::runs::models::{RunClaim, RunRequest}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Occupied slots above a lowered limit still count; group children use a parent's slot one at a time. +pub(crate) async fn reserve_claims( + mut transaction: ControlTransaction, + request: &RunRequest, + limits: &BTreeMap, +) -> Result<(ControlTransaction, bool), ControlError> { + if let Some(job_key) = &request.job_key { + let rows; + (transaction, rows) = query( + transaction, + "SELECT r.run_id FROM sqb_runs r \ + WHERE r.location_id=$1 AND r.target=$2 AND r.job_key=$3 \ + AND (r.worker_instance IS NOT NULL OR (r.kind='group' AND r.status='running')) LIMIT 1", + vec![ + Parameter::Id(request.location_id), + Parameter::Text(request.target.clone()), + Parameter::Text(job_key.clone()), + ], + ) + .await?; + if !rows.is_empty() { + return Ok((transaction, false)); + } + } + let mut siblings_hold: BTreeSet = BTreeSet::new(); + if let Some(parent_id) = request.parent_run_id { + let rows; + (transaction, rows) = query( + transaction, + "SELECT request_json FROM sqb_runs WHERE parent_run_id=$1 AND run_id<>$2 AND worker_instance IS NOT NULL", + vec![Parameter::Id(parent_id), Parameter::Id(request.run_id)], + ) + .await?; + for row in rows { + let sibling: RunRequest = serde_json::from_value(row["request_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + siblings_hold.extend(sibling.concurrency_groups); + } + } + let now = transaction.now; + for node in &request.write_set { + let inserted; + (transaction, inserted) = query(transaction, + "INSERT INTO sqb_write_claims (target, node_key, run_id, claimed_at) VALUES ($1,$2,$3,$4) \ + ON CONFLICT (target, node_key) DO NOTHING RETURNING node_key", + vec![Parameter::Text(request.target.clone()), Parameter::Text(node.clone()), Parameter::Id(request.run_id), Parameter::Time(now)]).await?; + if inserted.is_empty() { + return Ok((transaction, false)); + } + } + for group in &request.concurrency_groups { + if let Some(parent_id) = request.parent_run_id { + let inherited; + (transaction, inherited) = query( + transaction, + "SELECT slot FROM sqb_concurrency_slots WHERE group_key=$1 AND run_id=$2", + vec![Parameter::Text(group.clone()), Parameter::Id(parent_id)], + ) + .await?; + if !inherited.is_empty() { + if siblings_hold.contains(group) { + return Ok((transaction, false)); + } + continue; + } + } + let limit = limits + .get(group) + .ok_or_else(|| { + ControlError::Invalid(format!("concurrency group {group} has no configured limit")) + })? + .get(); + let slots; + (transaction, slots) = query( + transaction, + "SELECT slot FROM sqb_concurrency_slots WHERE group_key=$1 ORDER BY slot", + vec![Parameter::Text(group.clone())], + ) + .await?; + let occupied: BTreeSet = slots + .iter() + .filter_map(|row| row.get("slot").and_then(serde_json::Value::as_i64)) + .collect(); + if occupied.len() >= limit as usize { + return Ok((transaction, false)); + } + let slot = (1..=i64::from(limit)) + .find(|slot| !occupied.contains(slot)) + .ok_or_else(|| { + ControlError::Invalid("concurrency slot accounting is inconsistent".into()) + })?; + let inserted; + (transaction, inserted) = query(transaction, + "INSERT INTO sqb_concurrency_slots (group_key, slot, run_id, claimed_at) VALUES ($1,$2,$3,$4) \ + ON CONFLICT (group_key, slot) DO NOTHING RETURNING slot", + vec![Parameter::Text(group.clone()), Parameter::Integer(slot), Parameter::Id(request.run_id), Parameter::Time(now)]).await?; + if inserted.is_empty() { + return Ok((transaction, false)); + } + } + Ok((transaction, true)) +} + +/// Publish ownership only after every physical claim has been reserved in this transaction. +pub(crate) async fn publish_claim( + transaction: ControlTransaction, + request: RunRequest, +) -> Result<(ControlTransaction, RunClaim), ControlError> { + let claim = RunClaim { + claim_id: Uuid::new_v5(&request.run_id, b"run_claimed"), + request, + }; + let now = transaction.now; + let (transaction, claimed) = query( + transaction, + "UPDATE sqb_runs SET worker_instance=$2, claimed_at=$3, last_heartbeat_at=$3 \ + WHERE run_id=$1 AND worker_instance IS NULL RETURNING run_id", + vec![ + Parameter::Id(claim.request.run_id), + Parameter::Id(claim.claim_id), + Parameter::Time(now), + ], + ) + .await?; + if claimed.is_empty() { + return Err(ControlError::Invalid("run is already claimed".into())); + } + let (transaction, cursor) = append_run_event( + transaction, + (claim.request.run_id, claim.claim_id), + "run_claimed", + json!({"claim": claim}), + ) + .await?; + let (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET last_event_storage_id=$1 WHERE run_id=$2", + vec![ + Parameter::Integer(cursor), + Parameter::Id(claim.request.run_id), + ], + ) + .await?; + Ok((transaction, claim)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/events.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/events.rs new file mode 100644 index 0000000000..2bd4d871bf --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/events.rs @@ -0,0 +1,38 @@ +//! Lifecycle facts share Recall's invocation sequence and transaction. + +use serde_json::Value; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::models::{Change, ControlEvent, ControlTransaction}; + +pub(crate) async fn append_run_event( + transaction: ControlTransaction, + (run_id, event_id): (Uuid, Uuid), + kind: &str, + payload: Value, +) -> Result<(ControlTransaction, i64), ControlError> { + let event = ControlEvent { + event_id, + invocation_id: format!("scheduler/{run_id}"), + event_type: kind, + producer: "scheduler", + occurred_at: transaction.now, + run_id: Some(run_id), + payload, + }; + let (mut transaction, outcome) = append_control_event(transaction, event).await?; + if outcome.conflict.is_some() { + return Err(ControlError::Invalid( + "run event conflicts with a durable fact".into(), + )); + } + let cursor = outcome + .storage_id + .ok_or_else(|| ControlError::Invalid("run event has no storage cursor".into()))?; + if outcome.inserted { + transaction.changes.insert(Change::Run(run_id)); + } + Ok((transaction, cursor)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs new file mode 100644 index 0000000000..21b643ed02 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs @@ -0,0 +1,12 @@ +pub(in crate::runs) mod change_run; +pub(in crate::runs) mod changes; +pub(in crate::runs) mod checks; +pub(in crate::runs) mod claims; +pub(in crate::runs) mod events; +pub(in crate::runs) mod nodes; +pub(in crate::runs) mod output; +pub(in crate::runs) mod projection; +pub(in crate::runs) mod publish_change; +pub(in crate::runs) mod queued; +pub(in crate::runs) mod reads; +pub(in crate::runs) mod recall; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/nodes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/nodes.rs new file mode 100644 index 0000000000..f753703706 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/nodes.rs @@ -0,0 +1,84 @@ +//! Node lifecycle projections adjust bounded counters instead of rescanning nodes per event. + +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::runs::constants::{NODE_PENDING, NODE_RUNNING}; +use crate::runs::models::{RunChange, RunChangeRequest}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn project_node( + transaction: ControlTransaction, + request: &RunChangeRequest, + counts: &BTreeMap, +) -> Result { + let (node, status, rows, error, skipped) = match &request.change { + RunChange::NodeStarted { node_key } => (node_key, NODE_RUNNING, None, None, None), + RunChange::NodeFinished { + node_key, + outcome, + rows_affected, + error_code, + skipped_because_of, + } => ( + node_key, + outcome.as_str(), + *rows_affected, + error_code.clone(), + skipped_because_of.clone(), + ), + _ => { + return Err(ControlError::Invalid( + "node projection requires a node event".into(), + )); + } + }; + let now = transaction.now; + let (transaction, nodes) = query( + transaction, + "SELECT status FROM sqb_run_nodes WHERE run_id=$1 AND node_key=$2 AND kind='node'", + vec![Parameter::Id(request.run_id), Parameter::Text(node.clone())], + ) + .await?; + let previous = nodes + .first() + .and_then(|row| row.get("status")) + .and_then(serde_json::Value::as_str) + .ok_or_else(|| ControlError::Invalid("node is outside the frozen run selection".into()))?; + if previous != NODE_PENDING && (previous != NODE_RUNNING || status == NODE_RUNNING) { + return Err(ControlError::Invalid( + "node change is inconsistent with its current outcome".into(), + )); + } + let mut counts = counts.clone(); + let old = counts + .get_mut(previous) + .ok_or_else(|| ControlError::Invalid("node counter is missing".into()))?; + *old -= 1; + if *old < 0 { + return Err(ControlError::Invalid("node counter is negative".into())); + } + if *old == 0 { + counts.remove(previous); + } + *counts.entry(status.into()).or_default() += 1; + let (transaction, _) = query(transaction, + "UPDATE sqb_run_nodes SET status=$1, started_at=CASE WHEN $1='running' THEN $2 ELSE started_at END, \ + ended_at=$3, rows_affected=$4, error_code=$5, skipped_because_of=$6 WHERE run_id=$7 AND node_key=$8 AND kind='node'", + vec![Parameter::Text(status.into()), Parameter::Time(now), Parameter::OptionalTime((status != NODE_RUNNING).then_some(now)), + rows.map_or(Parameter::Null(sqlbuild_recall::schema::models::ColumnType::Integer), Parameter::Integer), + Parameter::OptionalText(error), Parameter::OptionalText(skipped), Parameter::Id(request.run_id), Parameter::Text(node.clone())]).await?; + let (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET node_counts_json=$1 WHERE run_id=$2", + vec![ + Parameter::Json(json!(counts)), + Parameter::Id(request.run_id), + ], + ) + .await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/output.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/output.rs new file mode 100644 index 0000000000..bb8c6e822e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/output.rs @@ -0,0 +1,80 @@ +//! Normalize Recall output under its physical claim's invocation envelope. + +use chrono::{DateTime, SecondsFormat}; +use sqlbuild_recall::records::main::output_row::output_row; +use sqlbuild_recall::records::models::{OutputChunk, RecallRow, RowEnvelope}; +use uuid::Uuid; + +use crate::runs::constants::{OUTPUT_BATCH_BYTES, OUTPUT_BATCH_CHUNKS}; +use crate::runs::models::ClaimMetadata; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn claim_envelope( + transaction: ControlTransaction, + (run_id, claim_id): (Uuid, Uuid), +) -> Result<(ControlTransaction, RowEnvelope), ControlError> { + let (transaction, rows) = query( + transaction, + "SELECT r.claimed_at, s.sqb_version FROM sqb_runs r \ + JOIN sqb_snapshots s ON s.snapshot_id=r.snapshot_id \ + WHERE r.run_id=$1 AND r.worker_instance=$2", + vec![Parameter::Id(run_id), Parameter::Id(claim_id)], + ) + .await?; + let metadata: ClaimMetadata = decode_row(rows.into_iter().next().ok_or_else(|| { + ControlError::Invalid("publication does not belong to an active run claim".into()) + })?)?; + Ok(( + transaction, + RowEnvelope { + invocation_id: format!("worker/{claim_id}"), + invocation_started_at: metadata + .claimed_at + .to_rfc3339_opts(SecondsFormat::Micros, true), + producer_version: metadata.sqb_version, + }, + )) +} + +/// Keep MAX(sequence) + 1 representable for Recall completeness accounting. +pub(crate) fn output_rows( + envelope: &RowEnvelope, + chunks: &[OutputChunk], +) -> Result, ControlError> { + if chunks.len() > OUTPUT_BATCH_CHUNKS + || chunks.iter().map(|chunk| chunk.data.len()).sum::() > OUTPUT_BATCH_BYTES + { + return Err(ControlError::Invalid( + "output batch exceeds its limit".into(), + )); + } + let mut rows = Vec::with_capacity(chunks.len()); + for chunk in chunks { + let mut chunk = chunk.clone(); + let mut integers = vec![chunk.sequence, chunk.byte_offset, chunk.dropped_before]; + if let Some(omitted) = chunk.omitted { + integers.extend([omitted.bytes, omitted.sequences]); + if !chunk.data.is_empty() || omitted.sequences == 0 { + return Err(ControlError::Invalid( + "invalid output omission marker".into(), + )); + } + } + if chunk.sequence >= i64::MAX as u64 + || integers.iter().any(|value| i64::try_from(*value).is_err()) + { + return Err(ControlError::Invalid( + "output counter is out of range".into(), + )); + } + chunk.occurred_at = DateTime::parse_from_rfc3339(&chunk.occurred_at) + .map_err(|_| ControlError::Invalid("invalid output timestamp".into()))? + .to_utc() + .to_rfc3339_opts(SecondsFormat::Micros, true); + rows.push(output_row(envelope, &chunk)); + } + Ok(rows) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs new file mode 100644 index 0000000000..da765e4e32 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs @@ -0,0 +1,136 @@ +//! Lifecycle projections and claim cleanup are part of the fact's transaction, never a later task. + +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::runs::_helpers::lifecycle::checks::check_execution_history; +use crate::runs::_helpers::lifecycle::nodes::project_node; +use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest, RunOutcome}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn project_change( + transaction: ControlTransaction, + request: &RunChangeRequest, + state: &ChangingRun, +) -> Result { + if matches!( + request.change, + RunChange::CheckStarted { .. } | RunChange::CheckFinished { .. } + ) { + return check_execution_history(transaction, request).await; + } + let mut transaction = project_run_state(transaction, request, state).await?; + if let RunChange::Launched { container_id } = &request.change { + (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET container_id=$1 WHERE run_id=$2 AND worker_instance IS NOT NULL", + vec![ + Parameter::OptionalText(container_id.clone()), + Parameter::Id(request.run_id), + ], + ) + .await?; + } + if matches!(request.change, RunChange::Finished { .. }) { + for statement in [ + "DELETE FROM sqb_write_claims WHERE run_id=$1", + "DELETE FROM sqb_concurrency_slots WHERE run_id=$1", + "UPDATE sqb_runs SET worker_instance=NULL, container_id=NULL, claimed_at=NULL, \ + last_heartbeat_at=NULL WHERE run_id=$1", + ] { + (transaction, _) = + query(transaction, statement, vec![Parameter::Id(request.run_id)]).await?; + } + } + Ok(transaction) +} + +/// Reconstruct derived status only, without releasing or recreating live physical ownership. +pub(crate) async fn project_run_state( + mut transaction: ControlTransaction, + request: &RunChangeRequest, + state: &ChangingRun, +) -> Result { + let now = transaction.now; + match &request.change { + RunChange::LaunchFailed { .. } + | RunChange::CheckStarted { .. } + | RunChange::CheckFinished { .. } => {} + RunChange::NodeStarted { .. } | RunChange::NodeFinished { .. } => { + return project_node(transaction, request, &state.node_counts_json).await; + } + RunChange::Launched { .. } => { + (transaction, _) = query(transaction, + "UPDATE sqb_runs SET status=CASE WHEN status='queued' THEN 'launched' ELSE status END, launched_at=COALESCE(launched_at,$1) WHERE run_id=$2", + vec![Parameter::Time(now), Parameter::Id(request.run_id)]).await?; + } + RunChange::Started => { + (transaction, _) = query(transaction, + "UPDATE sqb_runs SET status='running', launched_at=COALESCE(launched_at,$1), started_at=$1 WHERE run_id=$2", + vec![Parameter::Time(now), Parameter::Id(request.run_id)]).await?; + } + RunChange::Unresponsive | RunChange::Recovered => { + let status = if request.change == RunChange::Unresponsive { + "unresponsive" + } else { + "running" + }; + (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET status=$1 WHERE run_id=$2", + vec![ + Parameter::Text(status.into()), + Parameter::Id(request.run_id), + ], + ) + .await?; + } + RunChange::CancelRequested { .. } => { + (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET cancel_requested_at=$1 WHERE run_id=$2", + vec![Parameter::Time(now), Parameter::Id(request.run_id)], + ) + .await?; + } + RunChange::Finished { + outcome, reason, .. + } => { + let cancelled = *outcome == RunOutcome::Cancelled; + (transaction, _) = query(transaction, + "UPDATE sqb_run_nodes SET status=CASE WHEN status='running' THEN $1 ELSE $2 END, ended_at=$3 \ + WHERE run_id=$4 AND kind='node' AND status IN ('pending','running')", + vec![Parameter::Text(if cancelled { "cancelled" } else { "failed" }.into()), + Parameter::Text(if cancelled { "cancelled" } else { "skipped" }.into()), Parameter::Time(now), Parameter::Id(request.run_id)]).await?; + let counts; + (transaction, counts) = query(transaction, + "SELECT status, COUNT(*) AS count FROM sqb_run_nodes WHERE run_id=$1 AND kind='node' GROUP BY status", + vec![Parameter::Id(request.run_id)]).await?; + let counts: BTreeMap = counts + .into_iter() + .filter_map(|row| { + Some(( + row.get("status")?.as_str()?.into(), + row.get("count")?.as_i64()?, + )) + }) + .collect(); + (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET status=$1, status_reason=$2, ended_at=$3, node_counts_json=$5 WHERE run_id=$4", + vec![ + Parameter::Text(outcome.as_str().into()), + Parameter::Text(reason.clone()), + Parameter::Time(now), + Parameter::Id(request.run_id), + Parameter::Json(json!(counts)), + ], + ) + .await?; + } + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs new file mode 100644 index 0000000000..0ec90d7056 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs @@ -0,0 +1,107 @@ +//! Publish facts atomically; launch observations never project state or release physical ownership. + +use serde_json::json; + +use crate::platform::models::PlatformEvent; +use crate::runs::_helpers::lifecycle::events::append_run_event; +use crate::runs::_helpers::lifecycle::projection::project_change; +use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest, RunOutcome}; +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlTransaction, Parameter}; + +pub(crate) async fn publish_change( + transaction: ControlTransaction, + request: &RunChangeRequest, + state: &ChangingRun, +) -> Result { + let (mut transaction, mut cursor) = append_run_event( + transaction, + (request.run_id, request.event_id), + request.change.event_type(), + json!({"change": request}), + ) + .await?; + if let (RunChange::LaunchFailed { code }, Some(claim_id)) = (&request.change, request.claim_id) + { + let platform = PlatformEvent::LaunchFailed { + run_id: request.run_id, + claim_id, + code: code.clone(), + }; + let occurred_at = transaction.now; + let outcome; + (transaction, outcome) = append_control_event( + transaction, + ControlEvent { + event_id: uuid::Uuid::new_v5(&request.event_id, b"platform-launch-failed"), + invocation_id: format!("platform/{}", request.run_id), + event_type: platform.event_type(), + producer: "platform", + occurred_at, + run_id: Some(request.run_id), + payload: json!({"platform": platform}), + }, + ) + .await?; + if outcome.conflict.is_some() { + return Err(ControlError::Invalid( + "launch failure conflicts with a durable fact".into(), + )); + } + return Ok(transaction); + } + let mut transaction = project_change(transaction, request, state).await?; + if let Some((node_key, _)) = request.change.finished_node() { + (transaction, _) = query( + transaction, + "UPDATE sqb_run_nodes SET finished_storage_id=$1 WHERE run_id=$2 AND node_key=$3", + vec![ + Parameter::Integer(cursor), + Parameter::Id(request.run_id), + Parameter::Text(node_key.into()), + ], + ) + .await?; + } + let mut cancel_unstarted = matches!(request.change, RunChange::CancelRequested { .. }) + && state.worker_instance.is_none(); + if cancel_unstarted { + let rows; + (transaction, rows) = query( + transaction, + "SELECT kind FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + cancel_unstarted = rows.first().and_then(|row| row["kind"].as_str()) != Some("group"); + } + if cancel_unstarted { + let terminal = RunChangeRequest { + event_id: uuid::Uuid::new_v5(&request.event_id, b"cancelled-before-launch"), + run_id: request.run_id, + claim_id: None, + change: RunChange::Finished { + outcome: RunOutcome::Cancelled, + reason: "cancelled before launch".into(), + exit_code: None, + }, + }; + (transaction, cursor) = append_run_event( + transaction, + (terminal.run_id, terminal.event_id), + terminal.change.event_type(), + json!({"change": terminal}), + ) + .await?; + transaction = project_change(transaction, &terminal, state).await?; + } + let (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET last_event_storage_id=$1 WHERE run_id=$2", + vec![Parameter::Integer(cursor), Parameter::Id(request.run_id)], + ) + .await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/queued.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/queued.rs new file mode 100644 index 0000000000..9c62c054b8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/queued.rs @@ -0,0 +1,45 @@ +//! Initial run and node rows written when a run is queued. + +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::runs::models::RunRequest; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn project_queued_run( + mut transaction: ControlTransaction, + request: &RunRequest, + cursor: i64, +) -> Result { + let now = transaction.now; + let selection = serde_json::to_value(&request.selection) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let digest = sha256_hex( + &serde_json::to_vec(&request.write_set) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + (transaction, _) = query(transaction, + "INSERT INTO sqb_runs (run_id, location_id, target, snapshot_id, image_id, kind, origin, job_key, parent_run_id, root_run_id, trigger_id, tick_id, rerun_of_run_id, \ + requested_by, priority, selection_json, write_set_digest, status, queued_at, node_counts_json, last_event_storage_id, request_json) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,'queued',$18,$19,$20,$21) \ + ON CONFLICT (run_id) DO UPDATE SET location_id=excluded.location_id,target=excluded.target,snapshot_id=excluded.snapshot_id, \ + image_id=excluded.image_id,kind=excluded.kind,origin=excluded.origin,job_key=excluded.job_key,parent_run_id=excluded.parent_run_id, \ + root_run_id=excluded.root_run_id,trigger_id=excluded.trigger_id,tick_id=excluded.tick_id,rerun_of_run_id=excluded.rerun_of_run_id, \ + requested_by=excluded.requested_by,priority=excluded.priority,selection_json=excluded.selection_json,write_set_digest=excluded.write_set_digest, \ + status=excluded.status,queued_at=excluded.queued_at,node_counts_json=excluded.node_counts_json,last_event_storage_id=excluded.last_event_storage_id, \ + request_json=excluded.request_json,rewrite_history=FALSE,status_reason=NULL,launched_at=NULL,started_at=NULL,ended_at=NULL,cancel_requested_at=NULL", + vec![Parameter::Id(request.run_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), Parameter::Id(request.snapshot_id), + Parameter::OptionalText(request.image_id.clone()), Parameter::Text(request.kind.as_str().into()), Parameter::Text(request.origin.as_str().into()), + Parameter::OptionalText(request.job_key.clone()), Parameter::OptionalId(request.parent_run_id), Parameter::Id(request.root_run_id), + Parameter::OptionalId(request.trigger_id), Parameter::OptionalId(request.tick_id), Parameter::OptionalId(request.rerun_of_run_id), + Parameter::Text(request.requested_by.clone()), Parameter::Integer(request.priority), Parameter::Json(selection), Parameter::Text(digest), + Parameter::Time(now), Parameter::Json(json!({"pending": request.selection.len()})), Parameter::Integer(cursor), Parameter::Json(json!(request))]).await?; + for node in &request.selection { + (transaction, _) = query(transaction, + "INSERT INTO sqb_run_nodes (run_id, node_key, status, writes) VALUES ($1,$2,'pending',$3)", + vec![Parameter::Id(request.run_id), Parameter::Text(node.clone()), Parameter::Boolean(request.write_set.contains(node))]).await?; + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs new file mode 100644 index 0000000000..4689f8cbff --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs @@ -0,0 +1,25 @@ +//! Bounded metadata projection; it never includes SQL, sampled rows or captured output. + +use uuid::Uuid; + +use crate::runs::models::RunRecord; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn list_runs_in( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), + limit: u32, +) -> Result<(ControlTransaction, Vec), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT run_id, location_id, target, snapshot_id, job_key, status, queued_at, launched_at, started_at, ended_at, selection_json, node_counts_json, last_event_storage_id \ + FROM sqb_runs WHERE location_id=$1 AND target=$2 ORDER BY queued_at DESC, run_id DESC LIMIT $3", + vec![Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Integer(i64::from(limit.min(1000)))]).await?; + Ok(( + transaction, + rows.into_iter() + .map(decode_row) + .collect::, _>>()?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/recall.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/recall.rs new file mode 100644 index 0000000000..ecd6740f79 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/recall.rs @@ -0,0 +1,103 @@ +//! Namespace Recall facts and validate artifact ownership before publication. + +use std::collections::BTreeMap; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use sqlbuild_recall::records::models::{RecallRow, RecallValue, RowEnvelope}; +use sqlbuild_recall::schema::models::Table; +use sqlbuild_recall::store::models::InsertOutcome; + +use crate::runs::models::RecallRecord; +use crate::store::errors::ControlError; +use crate::store::main::append_recall::append_recall; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Recall statement projections join correlations globally, not only by invocation. +pub(crate) fn claim_record( + envelope: &RowEnvelope, + record: &RecallRecord, +) -> Result { + let mut values = record.values.clone(); + for name in [ + "event_id", + "run_id", + "resource_id", + "resource_attempt_id", + "operation_id", + "statement_id", + ] { + if let Some(RecallValue::Text(id)) = values.get_mut(name) { + *id = serde_json::json!([envelope.invocation_id, id]).to_string(); + } + } + match record.table { + Table::Meta | Table::Conflicts => { + return Err(ControlError::Invalid( + "Recall store metadata is server-owned".into(), + )); + } + Table::Artifacts => { + let Some(RecallValue::Blob(content)) = values.get("content") else { + return Err(ControlError::Invalid( + "Recall artifact content must be bytes".into(), + )); + }; + if values.get("digest") + != Some(&RecallValue::Text(format!( + "sha256:{}", + sha256_hex(content) + ))) + || values.get("size") != Some(&RecallValue::Integer(content.len() as i64)) + { + return Err(ControlError::Invalid( + "Recall artifact digest or size differs from its bytes".into(), + )); + } + } + _ => {} + } + RecallRow::from_values(envelope, record.table, &values) + .map_err(|error| ControlError::Invalid(error.to_string())) +} + +pub(crate) async fn append_records( + mut transaction: ControlTransaction, + envelope: &RowEnvelope, + rows: &[RecallRow], +) -> Result<(ControlTransaction, InsertOutcome), ControlError> { + let mut result = InsertOutcome::default(); + let mut delivered: BTreeMap = BTreeMap::new(); + for row in rows { + if row.table == Table::ArtifactRefs + && !rows.iter().any(|candidate| { + candidate.table == Table::Artifacts + && candidate.value("digest") == row.value("digest") + }) + { + let Some(RecallValue::Text(digest)) = row.value("digest") else { + return Err(ControlError::Invalid( + "Recall artifact digest must be text".into(), + )); + }; + let references; + (transaction, references) = query(transaction, + "SELECT 1 AS present FROM recall_artifact_refs WHERE invocation_id=$1 AND digest=$2 LIMIT 1", + vec![Parameter::Text(envelope.invocation_id.clone()), Parameter::Text(digest.clone())]).await?; + if references.is_empty() { + return Err(ControlError::Invalid( + "Recall artifact reference has no invocation-owned content".into(), + )); + } + } + let outcome; + (transaction, outcome) = append_recall(transaction, row).await?; + if let Some(conflict) = outcome.conflict { + result.conflicts.push(conflict); + } else { + *delivered.entry(row.table).or_insert(0) += 1; + } + } + result.delivered = delivered.into_iter().collect(); + Ok((transaction, result)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs new file mode 100644 index 0000000000..5d47d7cf23 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod groups; +pub(crate) mod lifecycle; +pub(crate) mod requests; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs new file mode 100644 index 0000000000..f9c69bf0c7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs @@ -0,0 +1,62 @@ +//! Freeze execution inputs from one published job or group in the caller's write transaction. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::declarations::main::frozen_definitions::frozen_definitions; +use crate::runs::models::{RunKind, RunOrigin, RunRequest}; +use crate::snapshots::main::job_write_set::job_write_set; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn frozen_group_children( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result<(ControlTransaction, Vec), ControlError> { + let (transaction, definitions) = frozen_definitions(transaction, request.snapshot_id).await?; + let group = request + .job_key + .as_ref() + .and_then(|name| definitions.groups.get(name)) + .ok_or_else(|| ControlError::Invalid("group is absent from the pinned snapshot".into()))?; + let selection_matches = if request.origin == RunOrigin::Rerun { + request.selection.is_subset(&group.job.selection) + } else { + group.job.selection == request.selection + }; + if !selection_matches || !request.write_set.is_empty() { + return Err(ControlError::Invalid( + "group scope differs from its pinned membership".into(), + )); + } + let (transaction, writers) = + job_write_set(transaction, request.snapshot_id, &request.selection).await?; + let mut children: Vec = Vec::new(); + for name in group.jobs.keys() { + let job = &definitions.jobs[name]; + let selection: BTreeSet<_> = job + .selection + .intersection(&request.selection) + .cloned() + .collect(); + if selection.is_empty() { + continue; + } + children.push(RunRequest { + run_id: Uuid::new_v5(&request.run_id, name.as_bytes()), + kind: RunKind::Job, + job_key: Some(name.clone()), + parent_run_id: Some(request.run_id), + rerun_of_run_id: request + .rerun_of_run_id + .map(|id| Uuid::new_v5(&id, name.as_bytes())), + write_set: selection.intersection(&writers).cloned().collect(), + selection, + settings: job.settings.clone(), + concurrency_groups: job.concurrency_groups.clone(), + ..request.clone() + }); + } + Ok((transaction, children)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs new file mode 100644 index 0000000000..a690f31d6d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs @@ -0,0 +1,80 @@ +//! Manual job freezing, idempotence, authorization and publication share one transaction. + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::main::authorize_person::authorize_person; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::runs::_helpers::requests::permissions::authorize_execution; +use crate::runs::_helpers::requests::publication::{ + publish_request, record_authored, request_retry, +}; +use crate::runs::constants::MANUAL_JOB_AUDIT_TAG; +use crate::runs::main::frozen_job_request::frozen_job_request; +use crate::runs::models::{JobRunRequest, RunWindow}; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn request_job_run_in( + mut transaction: ControlTransaction, + (context, credentials): (&AuditContext, Option), + request: &JobRunRequest, +) -> Result<(ControlTransaction, bool), ControlError> { + let mut context = context.clone(); + let scope = PermissionScope { + action: Action::JobRun, + location_id: Some(request.location_id), + target: Some(request.target.clone()), + nodes: Default::default(), + }; + if let Some(credentials) = credentials { + (transaction, context) = authorize_person( + transaction, + (&context, credentials), + &scope, + DefaultRole::None, + ) + .await?; + } + if context.actor.trim().is_empty() + || context.granting_role.trim().is_empty() + || request.target.trim().is_empty() + || request.job_key.trim().is_empty() + || request.reason.trim().is_empty() + { + return Err(ControlError::Invalid( + "job run needs an actor, granting role, target, job and reason".into(), + )); + } + if request.window.as_ref().is_some_and(|window| match window { + RunWindow::Timestamp { start, end } => start > end, + RunWindow::Integer { start, end } => start > end, + }) { + return Err(ControlError::Invalid( + "backfill window start must not follow end".into(), + )); + } + let event_id = Uuid::new_v5(&request.run_id, MANUAL_JOB_AUDIT_TAG); + let (transaction, retry) = + request_retry(transaction, &context, (request.run_id, &json!(request))).await?; + if retry { + return Ok((transaction, false)); + } + let (mut transaction, frozen) = frozen_job_request(transaction, &context, request).await?; + if let Some(credentials) = credentials { + (transaction, context) = authorize_execution( + transaction, + (&context, credentials), + (&frozen, Action::JobRun), + ) + .await?; + } + let (transaction, _) = publish_request(transaction, &frozen).await?; + let transaction = + record_authored(transaction, &context, (request.run_id, &json!(request))).await?; + let transaction = append_audit(transaction, &context, (event_id, "job.run_requested"), + (json!({"location_id":request.location_id,"target":request.target,"job_key":request.job_key,"run_id":request.run_id}), + json!(null), json!({"request":request,"frozen":frozen}))).await?; + Ok((transaction, true)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_rerun.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_rerun.rs new file mode 100644 index 0000000000..bd338a9933 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_rerun.rs @@ -0,0 +1,86 @@ +//! Publish a new pinned execution and its operator audit atomically, including retry identity. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{Action, AuditContext}; +use crate::runs::_helpers::requests::permissions::authorize_execution; +use crate::runs::_helpers::requests::publication::{ + publish_request, record_authored, request_retry, +}; +use crate::runs::_helpers::requests::rerun::frozen_rerun; +use crate::runs::constants::RERUN_AUDIT_TAG; +use crate::runs::models::{RerunRequest, RunRequest}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// No credentials means a trusted direct-store owner, never an HTTP-supplied role. +pub(crate) async fn request_rerun_in( + mut transaction: ControlTransaction, + (context, credentials): (&AuditContext, Option), + request: &RerunRequest, +) -> Result<(ControlTransaction, bool), ControlError> { + let mut context = context.clone(); + let mut prepared = None; + if let Some(credentials) = credentials { + let rows; + (transaction, rows) = query( + transaction, + "SELECT request_json FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + let frozen: RunRequest; + if let Some(row) = rows.into_iter().next() { + frozen = serde_json::from_value(row["request_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + (transaction, context) = authorize_execution( + transaction, + (&context, credentials), + (&frozen, Action::RunRerun), + ) + .await?; + } else { + (transaction, (context, frozen)) = + frozen_rerun(transaction, (&context, Some(credentials)), request).await?; + } + prepared = Some(frozen); + } + if context.actor.trim().is_empty() + || context.granting_role.trim().is_empty() + || request.reason.trim().is_empty() + { + return Err(ControlError::Invalid( + "rerun needs an actor, granting role and reason".into(), + )); + } + let event_id = Uuid::new_v5(&request.run_id, RERUN_AUDIT_TAG); + let (transaction, retry) = + request_retry(transaction, &context, (request.run_id, &json!(request))).await?; + if retry { + return Ok((transaction, false)); + } + let (transaction, frozen) = if let Some(frozen) = prepared { + (transaction, frozen) + } else { + let (transaction, (_, frozen)) = + frozen_rerun(transaction, (&context, None), request).await?; + (transaction, frozen) + }; + let (transaction, _) = publish_request(transaction, &frozen).await?; + let transaction = + record_authored(transaction, &context, (request.run_id, &json!(request))).await?; + let transaction = append_audit( + transaction, + &context, + (event_id, "run.rerun_requested"), + ( + json!({"location_id": frozen.location_id, "target": frozen.target, "job_key": frozen.job_key, "run_id": request.run_id}), + json!({"source_run_id": request.source_run_id}), + json!({"request": request, "frozen": frozen}), + ), + ).await?; + Ok((transaction, true)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs new file mode 100644 index 0000000000..fbd365703f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs @@ -0,0 +1,6 @@ +pub(in crate::runs) mod job; +pub(in crate::runs) mod manual_job; +pub(in crate::runs) mod manual_rerun; +pub(in crate::runs) mod permissions; +pub(in crate::runs) mod publication; +pub(in crate::runs) mod rerun; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs new file mode 100644 index 0000000000..58baf5f024 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs @@ -0,0 +1,72 @@ +//! Person authority covers the effective frozen execution, including sensitive group children. + +use uuid::Uuid; + +use crate::authorization::main::authorize_person::authorize_person; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::runs::_helpers::requests::job::frozen_group_children; +use crate::runs::models::{RunKind, RunRequest}; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn authorize_execution( + mut transaction: ControlTransaction, + (context, credentials): (&AuditContext, Uuid), + (request, action): (&RunRequest, Action), +) -> Result<(ControlTransaction, AuditContext), ControlError> { + let mut executions = vec![request.clone()]; + if request.kind == RunKind::Group { + (transaction, executions) = frozen_group_children(transaction, request).await?; + } + let mut scope = PermissionScope { + action, + location_id: Some(request.location_id), + target: Some(request.target.clone()), + nodes: executions + .iter() + .flat_map(|run| run.write_set.iter().cloned()) + .collect(), + }; + let (mut transaction, mut context) = authorize_person( + transaction, + (context, credentials), + &scope, + DefaultRole::None, + ) + .await?; + if action == Action::RunRerun && request.kind == RunKind::Adhoc { + scope.action = Action::RunAdhoc; + (transaction, context) = authorize_person( + transaction, + (&context, credentials), + &scope, + DefaultRole::None, + ) + .await?; + } + if matches!(action, Action::JobRun | Action::RunRerun) + && (request.window.is_some() + || executions.iter().any(|run| { + run.settings + .get("full_refresh") + .and_then(serde_json::Value::as_bool) + == Some(true) + || run.settings.contains_key("start_cursor") + || run.settings.contains_key("end_cursor") + || run.settings.contains_key("start_cursor_ts") + || run.settings.contains_key("end_cursor_ts") + || run.settings.contains_key("start_cursor_int") + || run.settings.contains_key("end_cursor_int") + })) + { + scope.action = Action::RunRewriteHistory; + (transaction, context) = authorize_person( + transaction, + (&context, credentials), + &scope, + DefaultRole::None, + ) + .await?; + } + Ok((transaction, context)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/publication.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/publication.rs new file mode 100644 index 0000000000..9b5d67be48 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/publication.rs @@ -0,0 +1,114 @@ +//! Validate and publish frozen requests inside the caller's audit transaction. + +use std::collections::BTreeSet; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::runs::main::queue_run::queue_run; +use crate::runs::models::RunRequest; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Check the original authored request before reading mutable publication pointers or outcomes. +pub(crate) async fn request_retry( + transaction: ControlTransaction, + context: &AuditContext, + (run_id, request): (Uuid, &serde_json::Value), +) -> Result<(ControlTransaction, bool), ControlError> { + let (transaction, previous) = query( + transaction, + "SELECT authored_json FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(run_id)], + ) + .await?; + let Some(previous) = previous.first() else { + return Ok((transaction, false)); + }; + if previous["authored_json"] != json!({"actor": context.actor, "request": request}) { + return Err(ControlError::Invalid( + "run ID already has different request content".into(), + )); + } + Ok((transaction, true)) +} + +/// Keep who asked for a manual run and exactly what they sent, for retries of the same run ID. +pub(crate) async fn record_authored( + transaction: ControlTransaction, + context: &AuditContext, + (run_id, request): (Uuid, &serde_json::Value), +) -> Result { + let (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET authored_json=$1 WHERE run_id=$2", + vec![ + Parameter::Json(json!({"actor": context.actor, "request": request})), + Parameter::Id(run_id), + ], + ) + .await?; + Ok(transaction) +} + +pub(crate) async fn publish_request( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result<(ControlTransaction, bool), ControlError> { + if request.requested_by.trim().is_empty() + || request.selection.is_empty() + || !request.write_set.is_subset(&request.selection) + { + return Err(ControlError::Invalid( + "run needs an actor, selection and a contained write set".into(), + )); + } + let (transaction, previous) = query( + transaction, + "SELECT request_json FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + if let Some(previous) = previous.first() { + if previous.get("request_json") != Some(&json!(request)) { + return Err(ControlError::Invalid( + "run ID already has different request content".into(), + )); + } + return Ok((transaction, false)); + } + if request.parent_run_id.is_some() { + return Err(ControlError::Invalid( + "group children are created atomically with their pinned parent".into(), + )); + } + let (transaction, nodes) = query( + transaction, + "SELECT node_key FROM sqb_snapshot_nodes WHERE snapshot_id=$1 AND EXISTS \ + (SELECT 1 FROM sqb_snapshots WHERE snapshot_id=$1 AND location_id=$2 AND target=$3 \ + AND image_id IS NOT DISTINCT FROM $4)", + vec![ + Parameter::Id(request.snapshot_id), + Parameter::Id(request.location_id), + Parameter::Text(request.target.clone()), + Parameter::OptionalText(request.image_id.clone()), + ], + ) + .await?; + let keys: BTreeSet<&str> = nodes + .iter() + .filter_map(|row| row.get("node_key").and_then(serde_json::Value::as_str)) + .collect(); + if request + .selection + .iter() + .any(|key| !keys.contains(key.as_str())) + { + return Err(ControlError::Invalid( + "run selection or image does not match the pinned location and target snapshot".into(), + )); + } + Ok((queue_run(transaction, request).await?, true)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs new file mode 100644 index 0000000000..50310ba08b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs @@ -0,0 +1,95 @@ +//! Retry immutable execution inputs, narrowing only to durable failed and skipped outcomes. + +use std::collections::BTreeSet; + +use serde_json::from_value; +use uuid::Uuid; + +use crate::authorization::main::authorize_person::authorize_person; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::runs::_helpers::requests::permissions::authorize_execution; +use crate::runs::models::{RerunRequest, RunOrigin, RunRequest, RunStatus}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn frozen_rerun( + transaction: ControlTransaction, + (context, credentials): (&AuditContext, Option), + request: &RerunRequest, +) -> Result<(ControlTransaction, (AuditContext, RunRequest)), ControlError> { + let (mut transaction, rows) = query( + transaction, + "SELECT request_json, status FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.source_run_id)], + ) + .await?; + let Some(source) = rows.first() else { + if let Some(credentials) = credentials { + authorize_person( + transaction, + (context, credentials), + &PermissionScope { + action: Action::RunRerun, + location_id: None, + target: None, + nodes: Default::default(), + }, + DefaultRole::None, + ) + .await?; + } + return Err(ControlError::Invalid("source run was not found".into())); + }; + let status: RunStatus = from_value(source["status"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let mut frozen: RunRequest = from_value(source["request_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if request.from_failure { + let rows; + (transaction, rows) = query( + transaction, + "SELECT node_key FROM sqb_run_nodes WHERE run_id=$1 AND kind='node' AND status IN ('failed','skipped')", + vec![Parameter::Id(request.source_run_id)], + ) + .await?; + let failed: BTreeSet<_> = rows + .iter() + .filter_map(|row| row["node_key"].as_str()) + .collect(); + frozen.selection.retain(|key| failed.contains(key.as_str())); + frozen + .write_set + .retain(|key| frozen.selection.contains(key)); + } + frozen.run_id = request.run_id; + frozen.origin = RunOrigin::Rerun; + frozen.parent_run_id = None; + frozen.root_run_id = request.run_id; + frozen.trigger_id = None; + frozen.tick_id = None; + frozen.rerun_of_run_id = Some(request.source_run_id); + frozen.requested_by = context.actor.clone(); + frozen.priority = request.priority; + let mut context = context.clone(); + if let Some(credentials) = credentials { + (transaction, context) = authorize_execution( + transaction, + (&context, credentials), + (&frozen, Action::RunRerun), + ) + .await?; + frozen.requested_by.clone_from(&context.actor); + } + if !status.is_terminal() { + return Err(ControlError::Invalid( + "rerun requires a terminal source run".into(), + )); + } + if request.from_failure && frozen.selection.is_empty() { + return Err(ControlError::Invalid( + "source run has no failed or skipped nodes".into(), + )); + } + Ok((transaction, (context, frozen))) +} diff --git a/crates/sqlbuild-orchestration/src/runs/constants.rs b/crates/sqlbuild-orchestration/src/runs/constants.rs new file mode 100644 index 0000000000..4d273c9a15 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/constants.rs @@ -0,0 +1,16 @@ +//! Nonterminal node states; terminal states come from NodeOutcome. + +pub(crate) const NODE_PENDING: &str = "pending"; +pub(crate) const NODE_RUNNING: &str = "running"; + +pub(crate) const HEARTBEAT_TIMEOUT_SECONDS: i64 = 45; +pub(crate) const MONITOR_BATCH_SIZE: usize = 100; + +pub(crate) const OUTPUT_BATCH_CHUNKS: usize = 256; +pub(crate) const OUTPUT_BATCH_BYTES: usize = 1_048_576; +pub(crate) const RECALL_BATCH_ROWS: usize = 500; +pub(crate) const RECALL_INVOCATION_ID_BYTES: usize = 256; + +/// Manual requests derive their audit event ID from the run ID so retries find the original. +pub(crate) const MANUAL_JOB_AUDIT_TAG: &[u8] = b"manual-job-request"; +pub(crate) const RERUN_AUDIT_TAG: &[u8] = b"rerun-request"; diff --git a/crates/sqlbuild-orchestration/src/runs/main/advance_groups.rs b/crates/sqlbuild-orchestration/src/runs/main/advance_groups.rs new file mode 100644 index 0000000000..33b0e9320e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/advance_groups.rs @@ -0,0 +1,51 @@ +//! Resume virtual parents independently of launcher capacity or process restarts. + +use std::collections::BTreeMap; +use std::num::NonZeroU32; +use std::time::{Duration, Instant}; + +use crate::runs::_helpers::groups::lifecycle::advance_group; +use crate::runs::models::RunDispatch; +use crate::scheduler::constants::RENEW_INTERVAL_SECONDS; +use crate::scheduler::main::{check_lease::check_lease, renew_lease::renew_lease}; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn advance_groups<'a>( + store: &ControlStore, + lease: &SchedulerLease, + (limits, dispatch): (&BTreeMap, impl Into>), +) -> Result<(), ControlError> { + let dispatch = dispatch.into(); + let scope = dispatch.scope.ok_or_else(|| { + ControlError::Invalid("group advancement requires a dispatch scope".into()) + })?; + let mut after = None; + let mut renewed = Instant::now(); + loop { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT run_id FROM sqb_runs WHERE kind='group' AND status IN ('queued','running') \ + AND location_id=$1 AND target=$2 AND ($3 IS NULL OR run_id>$3) ORDER BY run_id LIMIT 100", + vec![Parameter::Id(scope.0), Parameter::Text(scope.1.into()), Parameter::OptionalId(after)]).await?; + commit(transaction).await?; + if rows.is_empty() { + return Ok(()); + } + for row in rows { + if renewed.elapsed() >= Duration::from_secs(RENEW_INTERVAL_SECONDS) { + renew_lease(store, lease).await?; + renewed = Instant::now(); + } + let run_id: uuid::Uuid = serde_json::from_value(row["run_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let transaction = + check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let transaction = advance_group(transaction, run_id, limits, dispatch.eligible).await?; + commit(check_lease(transaction, lease).await?).await?; + after = Some(run_id); + } + } +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/cancel_person_run.rs b/crates/sqlbuild-orchestration/src/runs/main/cancel_person_run.rs new file mode 100644 index 0000000000..0c36271b25 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/cancel_person_run.rs @@ -0,0 +1,62 @@ +//! Cancellation derives scope and actor from durable execution inputs and live authentication. + +use uuid::Uuid; + +use crate::authorization::main::authorize_person::authorize_person; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::runs::_helpers::lifecycle::change_run::change_run_in; +use crate::runs::_helpers::requests::permissions::authorize_execution; +use crate::runs::models::{ + CancelRunRequest, RunAuthority, RunChange, RunChangeRequest, RunRequest, +}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn cancel_person_run( + store: &ControlStore, + credentials: (&AuditContext, Uuid), + request: &CancelRunRequest, +) -> Result { + let (transaction, rows) = query( + begin(store, TransactionMode::Write).await?, + "SELECT request_json FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + let Some(source) = rows.into_iter().next() else { + authorize_person( + transaction, + credentials, + &PermissionScope { + action: Action::RunCancel, + location_id: None, + target: None, + nodes: Default::default(), + }, + DefaultRole::None, + ) + .await?; + return Err(ControlError::Invalid("run does not exist".into())); + }; + let frozen: RunRequest = serde_json::from_value(source["request_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let (transaction, context) = + authorize_execution(transaction, credentials, (&frozen, Action::RunCancel)).await?; + let (transaction, applied) = change_run_in( + transaction, + RunAuthority::Operator(&context), + &RunChangeRequest { + run_id: request.run_id, + event_id: request.event_id, + claim_id: None, + change: RunChange::CancelRequested { + actor: context.actor.clone(), + reason: request.reason.clone(), + }, + }, + ) + .await?; + commit(transaction).await?; + Ok(applied) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs new file mode 100644 index 0000000000..294eb90d75 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs @@ -0,0 +1,22 @@ +//! Lifecycle changes share one transaction owner across trusted callers and person operations. + +use crate::runs::_helpers::lifecycle::change_run::change_run_in; +use crate::runs::models::{RunAuthority, RunChangeRequest}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn change_run( + store: &ControlStore, + authority: RunAuthority<'_>, + request: &RunChangeRequest, +) -> Result { + let (transaction, applied) = change_run_in( + begin(store, TransactionMode::Write).await?, + authority, + request, + ) + .await?; + commit(transaction).await?; + Ok(applied) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs b/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs new file mode 100644 index 0000000000..48adb8bd65 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs @@ -0,0 +1,92 @@ +//! Claim priority-ordered work without allowing a blocked page to starve later candidates. + +use std::collections::BTreeMap; +use std::num::NonZeroU32; + +use sqlbuild_recall::schema::models::ColumnType; + +use crate::runs::_helpers::groups::admission::{ + admission_limits, earlier_waiter_blocks, limit_waiters, +}; +use crate::runs::_helpers::groups::selection::child_ready; +use crate::runs::_helpers::lifecycle::claims::{publish_claim, reserve_claims}; +use crate::runs::models::{QueueCandidate, RunClaim, RunDispatch}; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{Connection, ControlStore, Parameter, TransactionMode}; + +/// Limits come from one installation configuration, never from competing per-job declarations. +pub async fn claim_next_run<'a>( + store: &ControlStore, + lease: &SchedulerLease, + limits: &BTreeMap, + dispatch: impl Into>, +) -> Result, ControlError> { + let dispatch = dispatch.into(); + let (mut transaction, waiters) = limit_waiters( + check_lease(begin(store, TransactionMode::Write).await?, lease).await?, + dispatch.eligible, + ) + .await?; + let locking = if matches!(transaction.connection, Some(Connection::Postgres(_))) { + " FOR UPDATE OF r SKIP LOCKED" + } else { + "" + }; + let mut after: Option = None; + loop { + let rows; + (transaction, rows) = query(transaction, + &format!("SELECT r.request_json, r.queued_at FROM sqb_runs r WHERE r.status='queued' AND r.kind<>'group' AND r.cancel_requested_at IS NULL \ + AND r.worker_instance IS NULL \ + AND ($4 IS NULL OR (r.location_id=$4 AND r.target=$5)) \ + AND ($1 IS NULL OR r.priority<$1 OR (r.priority=$1 AND (r.queued_at>$2 OR (r.queued_at=$2 AND r.run_id>$3)))) \ + ORDER BY r.priority DESC, r.queued_at, r.run_id LIMIT 100{locking}"), + vec![after.as_ref().map_or(Parameter::Null(ColumnType::Integer), |value| Parameter::Integer(value.request_json.priority)), + Parameter::OptionalTime(after.as_ref().map(|value| value.queued_at)), + Parameter::OptionalId(after.as_ref().map(|value| value.request_json.run_id)), + Parameter::OptionalId(dispatch.scope.map(|(location, _)| location)), + Parameter::OptionalText(dispatch.scope.map(|(_, target)| target.to_owned()))]).await?; + if rows.is_empty() { + commit(check_lease(transaction, lease).await?).await?; + return Ok(None); + } + for row in rows { + let candidate: QueueCandidate = decode_row(row)?; + let ready; + (transaction, ready) = child_ready(transaction, &candidate.request_json).await?; + if !ready { + after = Some(candidate); + continue; + } + let wanted = admission_limits(&candidate.request_json, []); + if earlier_waiter_blocks(&waiters, &candidate, &wanted) { + after = Some(candidate); + continue; + } + (transaction, _) = query(transaction, "SAVEPOINT candidate_claim", vec![]).await?; + let reserved; + (transaction, reserved) = + reserve_claims(transaction, &candidate.request_json, limits).await?; + if !reserved { + (transaction, _) = + query(transaction, "ROLLBACK TO SAVEPOINT candidate_claim", vec![]).await?; + (transaction, _) = + query(transaction, "RELEASE SAVEPOINT candidate_claim", vec![]).await?; + after = Some(candidate); + continue; + } + let claim; + (transaction, claim) = publish_claim(transaction, candidate.request_json).await?; + (transaction, _) = + query(transaction, "RELEASE SAVEPOINT candidate_claim", vec![]).await?; + commit(check_lease(transaction, lease).await?).await?; + return Ok(Some(claim)); + } + } +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs new file mode 100644 index 0000000000..c84da1f294 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs @@ -0,0 +1,69 @@ +//! Freeze published jobs for manual and sensor requests in their owning transaction. + +use crate::credentials::models::AuditContext; +use crate::declarations::models::JobDefinition; +use crate::runs::models::{JobRunRequest, RunKind, RunOrigin, RunRequest}; +use crate::snapshots::main::job_write_set::job_write_set; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn frozen_job_request( + transaction: ControlTransaction, + context: &AuditContext, + request: &JobRunRequest, +) -> Result<(ControlTransaction, RunRequest), ControlError> { + let (transaction, jobs) = query( + transaction, + "SELECT j.snapshot_id, j.definition_json, j.kind, s.image_id FROM sqb_jobs j \ + JOIN sqb_current_snapshots c ON c.location_id=j.location_id AND c.target=j.target AND c.snapshot_id=j.snapshot_id \ + JOIN sqb_snapshots s ON s.snapshot_id=j.snapshot_id WHERE j.location_id=$1 AND j.target=$2 AND j.job_key=$3", + vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), Parameter::Text(request.job_key.clone())], + ).await?; + let row = jobs.first().ok_or_else(|| { + ControlError::Invalid("published job not found in this location and target".into()) + })?; + let snapshot_id = serde_json::from_value(row["snapshot_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let definition: JobDefinition = serde_json::from_value(row["definition_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let image_id = serde_json::from_value(row["image_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let kind: RunKind = serde_json::from_value(row["kind"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let (transaction, write_set) = + job_write_set(transaction, snapshot_id, &definition.selection).await?; + Ok(( + transaction, + RunRequest { + run_id: request.run_id, + location_id: request.location_id, + target: request.target.clone(), + snapshot_id, + image_id, + kind, + origin: if request.window.is_some() { + RunOrigin::Backfill + } else { + RunOrigin::Manual + }, + job_key: Some(request.job_key.clone()), + parent_run_id: None, + root_run_id: request.run_id, + trigger_id: None, + tick_id: None, + rerun_of_run_id: None, + requested_by: context.actor.clone(), + priority: request.priority, + selection: definition.selection, + write_set: if kind == RunKind::Group { + Default::default() + } else { + write_set + }, + settings: definition.settings, + window: request.window.clone(), + concurrency_groups: definition.concurrency_groups, + }, + )) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs b/crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs new file mode 100644 index 0000000000..bf906f7765 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs @@ -0,0 +1,71 @@ +//! Heartbeats update only liveness; recovery is a durable transition in that same transaction. + +use serde_json::json; +use uuid::Uuid; + +use crate::runs::_helpers::lifecycle::events::append_run_event; +use crate::runs::models::{RunChange, RunChangeRequest}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The worker identity is taken from its authenticated run token. Return whether cancellation is pending. +pub async fn heartbeat_run( + store: &ControlStore, + (run_id, claim_id): (Uuid, Uuid), +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "UPDATE sqb_runs SET last_heartbeat_at=$1 WHERE run_id=$2 AND worker_instance=$3 RETURNING run_id", + vec![Parameter::Time(now), Parameter::Id(run_id), Parameter::Id(claim_id)]).await?; + if rows.is_empty() { + return Err(ControlError::Invalid( + "heartbeat does not belong to an active run claim".into(), + )); + } + let (mut transaction, rows) = query( + transaction, + "SELECT status, last_event_storage_id, cancel_requested_at FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(run_id)], + ) + .await?; + let row = rows + .first() + .ok_or_else(|| ControlError::Invalid("claimed run does not exist".into()))?; + let cancelled = row + .get("cancel_requested_at") + .is_some_and(|value| !value.is_null()); + if row.get("status").and_then(serde_json::Value::as_str) == Some("unresponsive") { + let previous = row + .get("last_event_storage_id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| { + ControlError::Invalid("run recovery is missing its preceding event".into()) + })?; + let request = RunChangeRequest { + event_id: Uuid::new_v5(&claim_id, format!("recovery/{previous}").as_bytes()), + run_id, + claim_id: Some(claim_id), + change: RunChange::Recovered, + }; + let cursor; + (transaction, cursor) = append_run_event( + transaction, + (run_id, request.event_id), + request.change.event_type(), + json!({"change": request}), + ) + .await?; + (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET status='running', last_event_storage_id=$1 WHERE run_id=$2", + vec![Parameter::Integer(cursor), Parameter::Id(run_id)], + ) + .await?; + } + commit(transaction).await?; + Ok(cancelled) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/list_person_runs.rs b/crates/sqlbuild-orchestration/src/runs/main/list_person_runs.rs new file mode 100644 index 0000000000..23d34c5126 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/list_person_runs.rs @@ -0,0 +1,34 @@ +//! Metadata reads hold live permission checks and results in the same transaction. + +use uuid::Uuid; + +use crate::authorization::main::authorize_person::authorize_person; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::runs::_helpers::lifecycle::reads::list_runs_in; +use crate::runs::models::RunRecord; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn list_person_runs( + store: &ControlStore, + (context, credentials): (&AuditContext, Uuid), + (location_id, target, limit): (Uuid, &str, u32), + default_role: DefaultRole, +) -> Result, ControlError> { + let (transaction, _) = authorize_person( + begin(store, TransactionMode::Write).await?, + (context, credentials), + &PermissionScope { + action: Action::View, + location_id: Some(location_id), + target: Some(target.into()), + nodes: Default::default(), + }, + default_role, + ) + .await?; + let (transaction, runs) = list_runs_in(transaction, (location_id, target), limit).await?; + commit(transaction).await?; + Ok(runs) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/list_run_claims.rs b/crates/sqlbuild-orchestration/src/runs/main/list_run_claims.rs new file mode 100644 index 0000000000..69dc6ac2ea --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/list_run_claims.rs @@ -0,0 +1,39 @@ +//! Recover ownership from active claims, not stale lifecycle events or elapsed heartbeat guesses. + +use uuid::Uuid; + +use crate::runs::constants::MONITOR_BATCH_SIZE; +use crate::runs::models::RunClaim; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Keyset pages include claimed-but-unstarted work and never include a neighboring scope. +pub async fn list_run_claims( + store: &ControlStore, + (location_id, target): (Uuid, &str), + after: Option, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + &format!( + "SELECT r.worker_instance AS claim_id, r.request_json AS request \ + FROM sqb_runs r \ + WHERE r.worker_instance IS NOT NULL AND r.location_id=$1 AND r.target=$2 AND ($3 IS NULL OR r.run_id>$3) \ + ORDER BY r.run_id LIMIT {MONITOR_BATCH_SIZE}" + ), + vec![ + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::OptionalId(after), + ], + ) + .await?; + let claims = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(claims) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/list_run_nodes.rs b/crates/sqlbuild-orchestration/src/runs/main/list_run_nodes.rs new file mode 100644 index 0000000000..4ef10ad211 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/list_run_nodes.rs @@ -0,0 +1,29 @@ +//! Node outcomes are scoped to one frozen run, independently of current code snapshots. + +use uuid::Uuid; + +use crate::runs::models::RunNodeRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The API authorizes visibility of the run before reading its nodes. +pub async fn list_run_nodes( + store: &ControlStore, + run_id: Uuid, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT node_key, status, started_at, ended_at, rows_affected, error_code, skipped_because_of \ + FROM sqb_run_nodes WHERE run_id=$1 AND kind='node' ORDER BY node_key", + vec![Parameter::Id(run_id)]).await?; + let nodes = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(nodes) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/list_runs.rs b/crates/sqlbuild-orchestration/src/runs/main/list_runs.rs new file mode 100644 index 0000000000..b7bf47534c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/list_runs.rs @@ -0,0 +1,22 @@ +//! Installation API reads never mix locations or targets. + +use uuid::Uuid; + +use crate::runs::_helpers::lifecycle::reads::list_runs_in; +use crate::runs::models::RunRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +/// Return recent projected runs; per-run events remain in Recall. +pub async fn list_runs( + store: &ControlStore, + (location_id, target): (Uuid, &str), + limit: u32, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, runs) = list_runs_in(transaction, (location_id, target), limit).await?; + commit(transaction).await?; + Ok(runs) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs new file mode 100644 index 0000000000..bc36f01e6c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -0,0 +1,19 @@ +pub mod advance_groups; +pub mod cancel_person_run; +pub mod change_run; +pub mod claim_next_run; +pub(crate) mod frozen_job_request; +pub mod heartbeat_run; +pub mod list_person_runs; +pub mod list_run_claims; +pub mod list_run_nodes; +pub mod list_runs; +pub mod monitor_runs; +pub mod observe_run; +pub mod publish_run_output; +pub mod publish_run_recall; +pub(crate) mod queue_run; +pub mod request_job_run; +pub mod request_person_job_run; +pub mod request_rerun; +pub mod run_execution; diff --git a/crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs b/crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs new file mode 100644 index 0000000000..48263740f6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs @@ -0,0 +1,54 @@ +//! Observe heartbeat loss under the scheduler fence without releasing a live worker's claims. + +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::runs::_helpers::lifecycle::publish_change::publish_change; +use crate::runs::constants::{HEARTBEAT_TIMEOUT_SECONDS, MONITOR_BATCH_SIZE}; +use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest}; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Serialized writers make heartbeat inspection and the resulting fact one atomic observation. +pub async fn monitor_runs( + store: &ControlStore, + lease: &SchedulerLease, +) -> Result { + let mut transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let cutoff = transaction.now - TimeDelta::seconds(HEARTBEAT_TIMEOUT_SECONDS); + let rows; + (transaction, rows) = query( + transaction, + &format!( + "SELECT r.run_id, r.status, r.cancel_requested_at, r.node_counts_json, \ + r.last_event_storage_id, r.worker_instance FROM sqb_runs r \ + WHERE r.status='running' AND r.last_heartbeat_at<$1 \ + ORDER BY r.last_heartbeat_at, r.run_id LIMIT {MONITOR_BATCH_SIZE}" + ), + vec![Parameter::Time(cutoff)], + ) + .await?; + let observed = rows.len(); + for row in rows { + let run_id: Uuid = serde_json::from_value(row["run_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let previous: i64 = serde_json::from_value(row["last_event_storage_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let state: ChangingRun = decode_row(row)?; + let request = RunChangeRequest { + event_id: Uuid::new_v5(&run_id, format!("unresponsive/{previous}").as_bytes()), + run_id, + claim_id: state.worker_instance, + change: RunChange::Unresponsive, + }; + transaction = publish_change(transaction, &request, &state).await?; + } + commit(check_lease(transaction, lease).await?).await?; + Ok(observed) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/observe_run.rs b/crates/sqlbuild-orchestration/src/runs/main/observe_run.rs new file mode 100644 index 0000000000..69b9067575 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/observe_run.rs @@ -0,0 +1,24 @@ +//! Observe one run and its current claim using the same database-clock snapshot. + +use uuid::Uuid; + +use crate::runs::models::RunObservation; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn observe_run( + store: &ControlStore, + run_id: Uuid, +) -> Result { + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, "SELECT r.status, r.cancel_requested_at, r.claimed_at, r.last_heartbeat_at, $2 AS observed_at FROM sqb_runs r WHERE r.run_id=$1", vec![Parameter::Id(run_id), Parameter::Time(now)]).await?; + let state = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("launched run is missing".into()))?, + )?; + commit(transaction).await?; + Ok(state) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs b/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs new file mode 100644 index 0000000000..8e7fce4680 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs @@ -0,0 +1,39 @@ +//! Exact-byte Recall output is namespaced by the authenticated physical claim, never a body ID. + +use sqlbuild_recall::records::models::OutputChunk; +use uuid::Uuid; + +use crate::runs::_helpers::lifecycle::output::{claim_envelope, output_rows}; +use crate::runs::models::OutputPublication; +use crate::store::errors::ControlError; +use crate::store::main::append_recall::append_recall; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{Change, ControlStore, TransactionMode}; + +/// The active-claim check and the complete batch commit share a serialized write transaction. +pub async fn publish_run_output( + store: &ControlStore, + (run_id, claim_id): (Uuid, Uuid), + chunks: &[OutputChunk], +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let (mut transaction, envelope) = claim_envelope(transaction, (run_id, claim_id)).await?; + let mut published = OutputPublication::default(); + for row in output_rows(&envelope, chunks)? { + let outcome; + (transaction, outcome) = append_recall(transaction, &row).await?; + if outcome.inserted { + published.inserted += 1; + } else if outcome.conflict.is_some() { + published.conflicts += 1; + } else { + published.duplicates += 1; + } + } + if published.inserted > 0 || published.conflicts > 0 { + transaction.changes.insert(Change::Run(run_id)); + } + commit(transaction).await?; + Ok(published) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs b/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs new file mode 100644 index 0000000000..967e89ad14 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs @@ -0,0 +1,42 @@ +//! A worker's complete Recall batch and active-claim check share one transaction. + +use sqlbuild_recall::store::models::InsertOutcome; +use uuid::Uuid; + +use crate::runs::_helpers::lifecycle::output::claim_envelope; +use crate::runs::_helpers::lifecycle::recall::{append_records, claim_record}; +use crate::runs::constants::{RECALL_BATCH_ROWS, RECALL_INVOCATION_ID_BYTES}; +use crate::runs::models::RecallBatch; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{Change, ControlStore, TransactionMode}; + +pub async fn publish_run_recall( + store: &ControlStore, + identity: (Uuid, Uuid), + batch: &RecallBatch, +) -> Result { + if batch.invocation_id.trim().is_empty() + || batch.invocation_id.len() > RECALL_INVOCATION_ID_BYTES + || batch.records.len() > RECALL_BATCH_ROWS + { + return Err(ControlError::Invalid( + "invalid Recall invocation or batch size".into(), + )); + } + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, mut envelope) = claim_envelope(transaction, identity).await?; + envelope.invocation_id = format!("{}/{}", envelope.invocation_id, batch.invocation_id); + let rows = batch + .records + .iter() + .map(|record| claim_record(&envelope, record)) + .collect::, _>>()?; + let (mut transaction, result) = append_records(transaction, &envelope, &rows).await?; + if !rows.is_empty() { + transaction.changes.insert(Change::Run(identity.0)); + } + commit(transaction).await?; + Ok(result) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs b/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs new file mode 100644 index 0000000000..f2d0ee7c95 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs @@ -0,0 +1,50 @@ +//! Publish request facts, queued facts and initial node projections in the caller's transaction. + +use serde_json::json; +use uuid::Uuid; + +use crate::runs::_helpers::lifecycle::events::append_run_event; +use crate::runs::_helpers::lifecycle::queued::project_queued_run; +use crate::runs::_helpers::requests::job::frozen_group_children; +use crate::runs::models::{RunKind, RunRequest}; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn queue_run( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result { + let mut transaction = queue_one(transaction, request).await?; + if request.kind == RunKind::Group { + let children; + (transaction, children) = frozen_group_children(transaction, request).await?; + for child in children { + transaction = queue_one(transaction, &child).await?; + } + } + Ok(transaction) +} + +async fn queue_one( + mut transaction: ControlTransaction, + request: &RunRequest, +) -> Result { + let mut cursor = None; + for kind in ["run_requested", "run_queued"] { + let storage_id; + (transaction, storage_id) = append_run_event( + transaction, + ( + request.run_id, + Uuid::new_v5(&request.run_id, kind.as_bytes()), + ), + kind, + json!({"request": request}), + ) + .await?; + cursor = Some(storage_id); + } + let cursor = + cursor.ok_or_else(|| ControlError::Invalid("queued event has no storage cursor".into()))?; + project_queued_run(transaction, request, cursor).await +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs new file mode 100644 index 0000000000..35d9f616f0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs @@ -0,0 +1,25 @@ +//! Direct-store owners queue published jobs without supplying or widening their write scope. + +use crate::credentials::models::AuditContext; +use crate::runs::_helpers::requests::manual_job::request_job_run_in; +use crate::runs::models::JobRunRequest; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +/// The context is a trusted control-store owner, not a role supplied by an HTTP caller. +pub async fn request_job_run( + store: &ControlStore, + context: &AuditContext, + request: &JobRunRequest, +) -> Result { + let (transaction, applied) = request_job_run_in( + begin(store, TransactionMode::Write).await?, + (context, None), + request, + ) + .await?; + commit(transaction).await?; + Ok(applied) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_person_job_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_person_job_run.rs new file mode 100644 index 0000000000..13797caf94 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/request_person_job_run.rs @@ -0,0 +1,25 @@ +//! Browser callers cannot take the trusted local-owner path or supply an execution write set. + +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::runs::_helpers::requests::manual_job::request_job_run_in; +use crate::runs::models::JobRunRequest; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn request_person_job_run( + store: &ControlStore, + (context, credentials): (&AuditContext, Uuid), + request: &JobRunRequest, +) -> Result { + let (transaction, applied) = request_job_run_in( + begin(store, TransactionMode::Write).await?, + (context, Some(credentials)), + request, + ) + .await?; + commit(transaction).await?; + Ok(applied) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_rerun.rs b/crates/sqlbuild-orchestration/src/runs/main/request_rerun.rs new file mode 100644 index 0000000000..a07e85159b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/request_rerun.rs @@ -0,0 +1,26 @@ +//! Trusted local reruns and browser reruns share freezing, idempotence and publication. + +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::runs::_helpers::requests::manual_rerun::request_rerun_in; +use crate::runs::models::RerunRequest; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +/// Omit session credentials only for trusted callers that directly own the control store. +pub async fn request_rerun( + store: &ControlStore, + credentials: (&AuditContext, Option), + request: &RerunRequest, +) -> Result { + let (transaction, applied) = request_rerun_in( + begin(store, TransactionMode::Write).await?, + credentials, + request, + ) + .await?; + commit(transaction).await?; + Ok(applied) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/run_execution.rs b/crates/sqlbuild-orchestration/src/runs/main/run_execution.rs new file mode 100644 index 0000000000..4cfbd8a8f9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/run_execution.rs @@ -0,0 +1,60 @@ +//! Read the immutable execution contract only for the run's currently active claim. + +use std::collections::BTreeMap; + +use uuid::Uuid; + +use crate::runs::_helpers::groups::selection::skipped_nodes; +use crate::runs::models::{ExecutionMetadata, RunClaim, RunExecution}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// A redeploy cannot change a claimed run's code or environment expectations. +pub async fn run_execution( + store: &ControlStore, + (run_id, claim_id): (Uuid, Uuid), +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT r.request_json, s.environment_hash, s.sqb_version FROM sqb_runs r \ + JOIN sqb_snapshots s ON s.snapshot_id=r.snapshot_id \ + WHERE r.run_id=$1 AND r.worker_instance=$2", + vec![Parameter::Id(run_id), Parameter::Id(claim_id)], + ) + .await?; + let Some(row) = rows.into_iter().next() else { + commit(transaction).await?; + return Ok(None); + }; + let metadata: ExecutionMetadata = decode_row(row)?; + let (transaction, rows) = query( + transaction, + "SELECT node_key, node_hash FROM sqb_snapshot_nodes WHERE snapshot_id=$1", + vec![Parameter::Id(metadata.request_json.snapshot_id)], + ) + .await?; + let node_hashes: BTreeMap = rows + .into_iter() + .map(|row| { + let fields: BTreeMap = decode_row(row)?; + Ok((fields["node_key"].clone(), fields["node_hash"].clone())) + }) + .collect::>()?; + let (transaction, skipped_nodes) = skipped_nodes(transaction, &metadata.request_json).await?; + commit(transaction).await?; + Ok(Some(RunExecution { + claim: RunClaim { + claim_id, + request: metadata.request_json, + }, + environment_hash: metadata.environment_hash, + sqb_version: metadata.sqb_version, + node_hashes, + skipped_nodes, + })) +} diff --git a/crates/sqlbuild-orchestration/src/runs/mod.rs b/crates/sqlbuild-orchestration/src/runs/mod.rs new file mode 100644 index 0000000000..529a59766a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +#[path = "../../tests/runs/test_contracts.rs"] +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs new file mode 100644 index 0000000000..7b6f9c7c2d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -0,0 +1,437 @@ +//! Frozen run requests and the run, node and claim records the control store keeps. + +use std::collections::{BTreeMap, BTreeSet}; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum RunKind { + Job, + Group, + Adhoc, +} + +impl RunKind { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Job => "job", + Self::Group => "group", + Self::Adhoc => "adhoc", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum RunOrigin { + Manual, + Schedule, + Sensor, + Rerun, + Backfill, +} + +impl RunOrigin { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Manual => "manual", + Self::Schedule => "schedule", + Self::Sensor => "sensor", + Self::Rerun => "rerun", + Self::Backfill => "backfill", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum RunStatus { + Queued, + Launched, + Running, + Unresponsive, + Success, + Warn, + Partial, + Failed, + Cancelled, +} + +impl RunStatus { + #[must_use] + pub const fn is_terminal(self) -> bool { + !matches!( + self, + Self::Queued | Self::Launched | Self::Running | Self::Unresponsive + ) + } +} + +/// Inclusive typed CLI cursor bounds; the native planner derives exclusive execution ends. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum RunWindow { + Timestamp { + start: DateTime, + end: DateTime, + }, + Integer { + start: i64, + end: i64, + }, +} + +/// The authorized selection and pinned execution inputs are immutable once requested. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct RunRequest { + pub run_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub snapshot_id: Uuid, + pub image_id: Option, + pub kind: RunKind, + pub origin: RunOrigin, + pub job_key: Option, + pub parent_run_id: Option, + pub root_run_id: Uuid, + pub trigger_id: Option, + pub tick_id: Option, + pub rerun_of_run_id: Option, + pub requested_by: String, + pub priority: i64, + pub selection: BTreeSet, + pub write_set: BTreeSet, + pub settings: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub window: Option, + pub concurrency_groups: BTreeSet, +} + +/// A trusted operator names a published job; execution inputs are never supplied by the caller. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct JobRunRequest { + pub run_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub job_key: String, + pub reason: String, + pub priority: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub window: Option, +} + +/// A retry derives its execution scope from a terminal run, never from caller selectors. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct RerunRequest { + pub run_id: Uuid, + pub source_run_id: Uuid, + pub from_failure: bool, + pub reason: String, + pub priority: i64, +} + +/// Browser cancellation cannot supply an actor, claim or lifecycle transition. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct CancelRunRequest { + pub run_id: Uuid, + pub event_id: Uuid, + pub reason: String, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct RunRecord { + pub run_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub snapshot_id: Uuid, + pub job_key: Option, + pub status: RunStatus, + pub queued_at: DateTime, + pub launched_at: Option>, + pub started_at: Option>, + pub ended_at: Option>, + pub selection_json: BTreeSet, + pub node_counts_json: BTreeMap, + pub last_event_storage_id: i64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum NodeOutcome { + Success, + Warn, + Failed, + Skipped, + Cancelled, +} + +impl NodeOutcome { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Success => "success", + Self::Warn => "warn", + Self::Failed => "failed", + Self::Skipped => "skipped", + Self::Cancelled => "cancelled", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct RunNodeRecord { + pub node_key: String, + pub status: String, + pub started_at: Option>, + pub ended_at: Option>, + pub rows_affected: Option, + pub error_code: Option, + pub skipped_because_of: Option, +} + +/// The database owns the claim before a launcher sees the pinned request. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct RunClaim { + pub claim_id: Uuid, + pub request: RunRequest, +} + +/// Workers compile locally and compare against this immutable snapshot, not the current deploy. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct RunExecution { + pub claim: RunClaim, + pub environment_hash: String, + pub sqb_version: String, + pub node_hashes: BTreeMap, + /// Derived from completed sibling facts and pinned lineage, never a mutable request rewrite. + #[serde(default)] + pub skipped_nodes: BTreeMap, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct ExecutionMetadata { + pub(crate) request_json: RunRequest, + pub(crate) environment_hash: String, + pub(crate) sqb_version: String, +} + +/// Terminal outcomes cannot be confused with a request to change an active status. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum RunOutcome { + Success, + Warn, + Partial, + Failed, + Cancelled, +} + +impl RunOutcome { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Success => "success", + Self::Warn => "warn", + Self::Partial => "partial", + Self::Failed => "failed", + Self::Cancelled => "cancelled", + } + } +} + +/// This is a server-owned instruction, not an arbitrary status supplied by a worker. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum RunChange { + Launched { + container_id: Option, + }, + /// A failed launch response is not physical evidence that the worker has stopped. + LaunchFailed { + code: String, + }, + Started, + NodeStarted { + node_key: String, + }, + NodeFinished { + node_key: String, + outcome: NodeOutcome, + rows_affected: Option, + error_code: Option, + skipped_because_of: Option, + }, + /// Observational attached-check facts never expand the frozen execution selection. + CheckStarted { + node_key: String, + }, + CheckFinished { + node_key: String, + outcome: NodeOutcome, + }, + Unresponsive, + Recovered, + CancelRequested { + actor: String, + #[serde(default)] + reason: String, + }, + Finished { + outcome: RunOutcome, + reason: String, + exit_code: Option, + }, +} + +impl RunChange { + pub(crate) const fn event_type(&self) -> &'static str { + match self { + Self::Launched { .. } => "run_launched", + Self::LaunchFailed { .. } => "run_launch_failed", + Self::Started => "run_started", + Self::NodeStarted { .. } => "run_node_started", + Self::NodeFinished { .. } => "run_node_finished", + Self::CheckStarted { .. } => "run_check_started", + Self::CheckFinished { .. } => "run_check_finished", + Self::Unresponsive => "run_unresponsive", + Self::Recovered => "run_recovered", + Self::CancelRequested { .. } => "run_cancel_requested", + Self::Finished { .. } => "run_terminated", + } + } + + pub(crate) fn finished_node(&self) -> Option<(&str, NodeOutcome)> { + match self { + Self::NodeFinished { + node_key, outcome, .. + } + | Self::CheckFinished { node_key, outcome } => Some((node_key, *outcome)), + _ => None, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct RunChangeRequest { + pub event_id: Uuid, + pub run_id: Uuid, + pub claim_id: Option, + pub change: RunChange, +} + +/// Constructed after authentication; this authority is never deserialized from a request body. +#[derive(Clone, Copy, Debug)] +pub enum RunAuthority<'a> { + Scheduler(&'a crate::scheduler::models::SchedulerLease), + Worker { run_id: Uuid, claim_id: Uuid }, + Operator(&'a crate::credentials::models::AuditContext), +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct QueueCandidate { + pub(crate) request_json: RunRequest, + pub(crate) queued_at: DateTime, +} + +/// Fairness covers runnable launcher scopes, not queues the installation cannot dispatch. +#[derive(Clone, Copy, Debug)] +pub struct RunDispatch<'a> { + pub scope: Option<(Uuid, &'a str)>, + pub eligible: Option<&'a BTreeSet<(Uuid, String)>>, +} + +impl<'a> From> for RunDispatch<'a> { + fn from(scope: Option<(Uuid, &'a str)>) -> Self { + Self { + scope, + eligible: None, + } + } +} + +impl<'a> From<(Uuid, &'a str)> for RunDispatch<'a> { + fn from(scope: (Uuid, &'a str)) -> Self { + Some(scope).into() + } +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct ChangingRun { + pub(crate) status: RunStatus, + pub(crate) worker_instance: Option, + pub(crate) cancel_requested_at: Option>, + pub(crate) node_counts_json: BTreeMap, +} + +/// Virtual parents and physical children use the same lifecycle evidence during group progress. +#[derive(Debug, Deserialize)] +pub(crate) struct GroupRun { + pub(crate) request_json: RunRequest, + pub(crate) queued_at: DateTime, + #[serde(flatten)] + pub(crate) state: ChangingRun, +} + +/// A queued top-level run and the limits it takes when admitted, in queue order. +#[derive(Debug)] +pub(crate) struct LimitWaiter { + pub(crate) rank: (std::cmp::Reverse, DateTime, Uuid), + pub(crate) run_id: Uuid, + pub(crate) limits: BTreeSet, +} + +#[derive(Debug)] +pub(crate) struct GroupContext { + pub(crate) parent: GroupRun, + pub(crate) children: Vec, + pub(crate) definitions: crate::declarations::models::Definitions, + pub(crate) critical_failed: bool, +} + +/// Differing retries retain conflict evidence while preserving the first output bytes. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct OutputPublication { + pub inserted: u32, + pub duplicates: u32, + pub conflicts: u32, +} + +/// One invocation's records; the server owns its claim namespace, start time and version. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct RecallBatch { + pub invocation_id: String, + pub records: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct RecallRecord { + pub table: sqlbuild_recall::schema::models::Table, + pub values: BTreeMap, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct ClaimMetadata { + pub(crate) claimed_at: DateTime, + pub(crate) sqb_version: String, +} + +/// A physical launcher uses the database clock and current liveness, never an old run event. +#[derive(Debug, Deserialize)] +pub struct RunObservation { + pub status: RunStatus, + pub cancel_requested_at: Option>, + pub claimed_at: Option>, + pub last_heartbeat_at: Option>, + pub observed_at: DateTime, +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/calendar.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/calendar.rs new file mode 100644 index 0000000000..7a27408a66 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/calendar.rs @@ -0,0 +1,31 @@ +//! Map local cron times without using the cron dependency's different fold policy. + +use chrono::{DateTime, LocalResult, NaiveDateTime, TimeZone, Utc}; +use chrono_tz::{GapInfo, Tz}; + +use crate::store::errors::ControlError; + +/// During the first fold, smaller wall times in the second fold may still be in the future. +pub(crate) fn search_start(timezone: Tz, after: DateTime) -> NaiveDateTime { + let local = after.with_timezone(&timezone).naive_local(); + match timezone.from_local_datetime(&local) { + LocalResult::Ambiguous(earlier, later) if after < later.to_utc() => { + local - (later.to_utc() - earlier.to_utc()) + } + _ => local, + } +} + +pub(crate) fn resolve_wall_time( + timezone: Tz, + wall_time: NaiveDateTime, +) -> Result, ControlError> { + match timezone.from_local_datetime(&wall_time) { + LocalResult::Single(time) => Ok(time.to_utc()), + LocalResult::Ambiguous(_, later) => Ok(later.to_utc()), + LocalResult::None => GapInfo::new(&wall_time, &timezone) + .and_then(|gap| gap.end) + .map(|time| time.to_utc()) + .ok_or_else(|| ControlError::Invalid("cron time has no representable gap end".into())), + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs new file mode 100644 index 0000000000..7161888c49 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs @@ -0,0 +1,65 @@ +//! Drain a bounded due-trigger page, renewing the lease between persisted evaluations. + +use std::time::Duration; + +use chrono::{DateTime, Utc}; +use uuid::Uuid; + +use crate::scheduler::_helpers::drain_sensors::drain_sensors; +use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; +use crate::scheduler::main::renew_lease::renew_lease; +use crate::scheduler::models::{SchedulerCycle, SchedulerLease}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn drain_due_triggers( + store: &ControlStore, + mut lease: SchedulerLease, + mut cycle: SchedulerCycle, +) -> Result<(SchedulerLease, SchedulerCycle), ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT s.trigger_id FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id \ + WHERE t.kind='schedule' AND s.status='running' AND s.next_due_at<=$1 AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ + ORDER BY s.next_due_at, s.trigger_id LIMIT 100", vec![Parameter::Time(now)]).await?; + commit(transaction).await?; + for row in rows { + let trigger_id: Uuid = serde_json::from_value(row["trigger_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if let Some(plan) = evaluate_trigger(store, &lease, trigger_id).await? { + cycle.evaluated += 1; + cycle.ticks += plan.actions.len(); + } + lease = renew_lease(store, &lease).await?; + } + let (lease, sensor_evaluations) = drain_sensors(store, lease).await?; + cycle.evaluated += sensor_evaluations; + Ok((lease, cycle)) +} + +/// Shorten the cycle wait to the next running trigger's due time, never below zero. +pub(crate) async fn wait_for_next_due( + store: &ControlStore, + wait_for: Duration, +) -> Result { + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT MIN(s.next_due_at) AS next_due_at FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id \ + WHERE s.status='running' AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL", vec![]).await?; + let next: Option> = serde_json::from_value(rows[0]["next_due_at"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + commit(transaction).await?; + Ok(next.map_or(wait_for, |next| { + (next - now) + .to_std() + .unwrap_or(Duration::ZERO) + .min(wait_for) + })) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs new file mode 100644 index 0000000000..964ee871b6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs @@ -0,0 +1,42 @@ +//! The scheduler drains built-in sensors without importing project code. + +use crate::scheduler::main::renew_lease::renew_lease; +use crate::scheduler::models::SchedulerLease; +use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; +use crate::sensors::main::complete_builtin_sensor::complete_builtin_sensor; +use crate::sensors::main::complete_custom_sensor::complete_custom_sensor; +use crate::sensors::models::SensorCondition; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn drain_sensors( + store: &ControlStore, + mut lease: SchedulerLease, +) -> Result<(SchedulerLease, usize), ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT t.trigger_id FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id WHERE t.kind='sensor' AND s.status='running' \ + AND s.next_due_at<=$1 AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ + ORDER BY s.next_due_at,t.trigger_id LIMIT 100", vec![Parameter::Time(now)]).await?; + commit(transaction).await?; + let mut evaluated = 0; + for row in rows { + let trigger_id = serde_json::from_value(row["trigger_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if let Some(request) = begin_sensor_evaluation(store, &lease, trigger_id).await? { + if matches!(request.definition.when, SensorCondition::Custom { .. }) { + complete_custom_sensor(store, &lease, &request).await?; + } else { + complete_builtin_sensor(store, &lease, &request).await?; + } + evaluated += 1; + } + lease = renew_lease(store, &lease).await?; + } + Ok((lease, evaluated)) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs new file mode 100644 index 0000000000..7df533ee19 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs @@ -0,0 +1,26 @@ +//! Find the newest due occurrence without expanding the history before it. + +use std::ops::Bound::{Included, Unbounded}; + +use chrono::{DateTime, Utc}; + +use crate::declarations::models::{Schedule, TriggerDefinition}; +use crate::scheduler::main::_previous_cron_time::previous_cron_time; +use crate::scheduler::main::parse_cron::parse_cron; +use crate::store::errors::ControlError; + +/// The newest occurrence at or before `now`. +pub(crate) fn latest_occurrence( + trigger: &TriggerDefinition, + now: DateTime, +) -> Result>, ControlError> { + match &trigger.schedule { + Schedule::Cron { + expression, + timezone, + } => previous_cron_time(&parse_cron(expression, timezone)?, now).map(Some), + Schedule::Calendar { times, .. } => { + Ok(times.range((Unbounded, Included(now))).next_back().copied()) + } + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs new file mode 100644 index 0000000000..bee69ea1d3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod calendar; +pub(crate) mod drain_due_triggers; +pub(crate) mod drain_sensors; +pub(crate) mod expansion; +pub(crate) mod persistence; +pub(crate) mod tick_retention; +pub(crate) mod ticks; diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs new file mode 100644 index 0000000000..66d3cead6e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs @@ -0,0 +1,84 @@ +//! Deterministic tick insertion gates all run facts; retries cannot queue a second run. + +use crate::runs::main::queue_run::queue_run; +use crate::runs::models::{RunKind, RunOrigin, RunRequest}; +use crate::scheduler::models::{DueTrigger, SchedulerLease, TickDecision, TickPlan}; +use crate::snapshots::main::job_write_set::job_write_set; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{Change, ControlTransaction, Parameter}; + +pub(crate) async fn persist_ticks( + transaction: ControlTransaction, + (trigger, lease): (&DueTrigger, &SchedulerLease), + plan: &TickPlan, +) -> Result { + let now = transaction.now; + let (mut transaction, write_set) = job_write_set( + transaction, + trigger.snapshot_id, + &trigger.job_json.selection, + ) + .await?; + for action in &plan.actions { + let (outcome, run, covered) = match action.decision { + TickDecision::Requested { run_id } => ("requested", Some(run_id), None), + TickDecision::Covered { covered_by_run_id } => { + ("covered", None, Some(covered_by_run_id)) + } + }; + let inserted; + (transaction, inserted) = query(transaction, + "INSERT INTO sqb_ticks (tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, run_id, covered_by_run_id) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8) ON CONFLICT (trigger_id, scheduled_for) DO NOTHING RETURNING tick_id", + vec![Parameter::Id(action.tick_id), Parameter::Id(trigger.trigger_id), Parameter::Time(action.scheduled_for), Parameter::Time(now), Parameter::Integer(lease.epoch), + Parameter::Text(outcome.into()), Parameter::OptionalId(run), Parameter::OptionalId(covered)]).await?; + if !inserted.is_empty() { + transaction + .changes + .insert(Change::Trigger(trigger.trigger_id)); + } + if let Some(run_id) = run + && !inserted.is_empty() + { + let request = RunRequest { + run_id, + location_id: trigger.location_id, + target: trigger.target.clone(), + snapshot_id: trigger.snapshot_id, + image_id: trigger.image_id.clone(), + kind: trigger.run_kind, + origin: RunOrigin::Schedule, + job_key: Some(trigger.job_key.clone()), + parent_run_id: None, + root_run_id: run_id, + trigger_id: Some(trigger.trigger_id), + tick_id: Some(action.tick_id), + rerun_of_run_id: None, + requested_by: "system:scheduler".into(), + priority: 0, + selection: trigger.job_json.selection.clone(), + write_set: if trigger.run_kind == RunKind::Group { + Default::default() + } else { + write_set.clone() + }, + settings: trigger.job_json.settings.clone(), + window: None, + concurrency_groups: trigger.job_json.concurrency_groups.clone(), + }; + transaction = queue_run(transaction, &request).await?; + } + } + let changed; + (transaction, changed) = query(transaction, + "UPDATE sqb_trigger_state SET last_tick_for=$1, next_due_at=$2, updated_at=$3 WHERE trigger_id=$4 \ + AND (last_tick_for IS DISTINCT FROM $1 OR next_due_at IS DISTINCT FROM $2) RETURNING trigger_id", + vec![Parameter::OptionalTime(plan.last_tick_for), Parameter::OptionalTime(plan.next_due_at), Parameter::Time(now), Parameter::Id(trigger.trigger_id)]).await?; + if !changed.is_empty() { + transaction + .changes + .insert(Change::Trigger(trigger.trigger_id)); + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/tick_retention.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/tick_retention.rs new file mode 100644 index 0000000000..b9091d986b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/tick_retention.rs @@ -0,0 +1,76 @@ +//! Tick expiry is fenced, batched cleanup that keeps each trigger's cursor tick and run links. + +use std::time::{Duration, Instant}; + +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::scheduler::constants::{ + TICK_RETENTION_BATCH, TICK_RETENTION_BATCHES_PER_PASS, TICK_RETENTION_DAYS, + TICK_RETENTION_INTERVAL_SECONDS, +}; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{Change, ControlStore, Parameter, TransactionMode}; + +/// Run the leader's tick cleanup at most once per interval; a capped pass runs again next cycle. +pub(crate) async fn prune_ticks_when_due( + store: &ControlStore, + lease: &SchedulerLease, + pruned_at: Option, +) -> Result, ControlError> { + let interval = Duration::from_secs(TICK_RETENTION_INTERVAL_SECONDS); + match pruned_at { + Some(at) if at.elapsed() < interval => Ok(Some(at)), + _ => { + let deleted = prune_ticks(store, lease).await?; + Ok( + if deleted == TICK_RETENTION_BATCH * TICK_RETENTION_BATCHES_PER_PASS { + pruned_at + } else { + Some(Instant::now()) + }, + ) + } + } +} + +/// One bounded pass; requested ticks stay, as does any tick a run points at. +pub(crate) async fn prune_ticks( + store: &ControlStore, + lease: &SchedulerLease, +) -> Result { + let mut deleted = 0; + for _ in 0..TICK_RETENTION_BATCHES_PER_PASS { + let mut transaction = + check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let cutoff = transaction.now - TimeDelta::days(TICK_RETENTION_DAYS); + let rows; + (transaction, rows) = query( + transaction, + "DELETE FROM sqb_ticks WHERE tick_id IN (SELECT k.tick_id FROM sqb_ticks k \ + WHERE k.outcome='covered' AND k.scheduled_for<$1 \ + AND k.scheduled_for<(SELECT MAX(l.scheduled_for) FROM sqb_ticks l WHERE l.trigger_id=k.trigger_id) \ + AND NOT EXISTS (SELECT 1 FROM sqb_runs r WHERE r.tick_id=k.tick_id) \ + ORDER BY k.scheduled_for, k.tick_id LIMIT $2) RETURNING trigger_id", + vec![ + Parameter::Time(cutoff), + Parameter::Integer(TICK_RETENTION_BATCH as i64), + ], + ) + .await?; + for row in &rows { + let trigger_id: Uuid = serde_json::from_value(row["trigger_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + transaction.changes.insert(Change::Trigger(trigger_id)); + } + commit(check_lease(transaction, lease).await?).await?; + deleted += rows.len(); + if rows.len() < TICK_RETENTION_BATCH { + break; + } + } + Ok(deleted) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs new file mode 100644 index 0000000000..605c3ee155 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs @@ -0,0 +1,38 @@ +//! Stable tick and run IDs, so a retried evaluation reproduces the same decision. + +use chrono::{DateTime, SecondsFormat, Utc}; +use uuid::Uuid; + +use crate::scheduler::models::{TickAction, TickDecision}; + +pub(crate) fn tick_id(trigger_id: Uuid, scheduled_for: DateTime) -> Uuid { + Uuid::new_v5( + &trigger_id, + scheduled_for + .to_rfc3339_opts(SecondsFormat::Nanos, true) + .as_bytes(), + ) +} + +pub(crate) fn requested_run(tick_id: Uuid) -> Uuid { + Uuid::new_v5(&tick_id, b"run") +} + +pub(crate) fn action( + trigger_id: Uuid, + scheduled_for: DateTime, + covering_run: Option, +) -> TickAction { + let tick_id = tick_id(trigger_id, scheduled_for); + let decision = match covering_run { + Some(covered_by_run_id) => TickDecision::Covered { covered_by_run_id }, + None => TickDecision::Requested { + run_id: requested_run(tick_id), + }, + }; + TickAction { + tick_id, + scheduled_for, + decision, + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/constants.rs b/crates/sqlbuild-orchestration/src/scheduler/constants.rs new file mode 100644 index 0000000000..c905cbad1c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/constants.rs @@ -0,0 +1,14 @@ +//! Leadership timing is shared by embedded and standalone schedulers. + +/// Renew well before another process may take over a stopped scheduler. +pub const RENEW_INTERVAL_SECONDS: u64 = 5; +/// A stopped holder cannot delay failover for more than this database-clock interval. +pub const LEASE_TTL_SECONDS: i64 = 30; +/// Bound a calendar declaration, including due times carried over a republish. +pub const MAX_TICKS_PER_BATCH: usize = 10_000; +/// Covered ticks older than this are deleted. +pub(crate) const TICK_RETENTION_DAYS: i64 = 30; +pub(crate) const TICK_RETENTION_BATCH: usize = 100; +/// Batches per cleanup pass; a longer backlog resumes next cycle, after the lease renews. +pub(crate) const TICK_RETENTION_BATCHES_PER_PASS: usize = 10; +pub(crate) const TICK_RETENTION_INTERVAL_SECONDS: u64 = 60; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_acquire_lease.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_acquire_lease.rs new file mode 100644 index 0000000000..20b4286e1b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_acquire_lease.rs @@ -0,0 +1,37 @@ +//! Elect one scheduler under the control store's serialized writer lock. + +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::scheduler::constants::LEASE_TTL_SECONDS; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// An existing live lease is never stolen, including by a duplicate caller with the same holder. +pub(crate) async fn acquire_lease( + store: &ControlStore, + holder: Uuid, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let expires_at = now + TimeDelta::seconds(LEASE_TTL_SECONDS); + let (transaction, rows) = query(transaction, + "INSERT INTO sqb_scheduler_lease (name, holder, epoch, expires_at) VALUES ('scheduler',$1,1,$2) \ + ON CONFLICT (name) DO UPDATE SET holder=excluded.holder, epoch=sqb_scheduler_lease.epoch+1, expires_at=excluded.expires_at \ + WHERE sqb_scheduler_lease.expires_at<=$3 RETURNING epoch", + vec![Parameter::Id(holder), Parameter::Time(expires_at), Parameter::Time(now)]).await?; + let lease = rows + .first() + .and_then(|row| row["epoch"].as_i64()) + .map(|epoch| SchedulerLease { + holder, + epoch, + expires_at, + }); + commit(transaction).await?; + Ok(lease) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs new file mode 100644 index 0000000000..b1a8a114be --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs @@ -0,0 +1,76 @@ +//! Fence, decide and publish using one serialized transaction and its database clock. + +use uuid::Uuid; + +use crate::declarations::models::TriggerStatus; +use crate::scheduler::_helpers::persistence::persist_ticks; +use crate::scheduler::main::_plan_ticks::plan_ticks; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::{DueTrigger, SchedulerLease, TickPlan, TickState}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// A removed or paused trigger is inert; a lost fence rolls back every write, including Recall. +pub(crate) async fn evaluate_trigger( + store: &ControlStore, + lease: &SchedulerLease, + trigger_id: Uuid, +) -> Result, ControlError> { + let transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let (transaction, rows) = query(transaction, + "SELECT t.trigger_id, t.location_id, t.target, t.job_key, t.snapshot_id, s.image_id, t.definition_json, j.definition_json AS job_json, j.kind AS run_kind, \ + state.started_at, state.last_tick_for, state.status FROM sqb_triggers t \ + JOIN sqb_trigger_state state ON state.trigger_id=t.trigger_id JOIN sqb_snapshots s ON s.snapshot_id=t.snapshot_id \ + JOIN sqb_jobs j ON j.location_id=t.location_id AND j.target=t.target AND j.job_key=t.job_key \ + WHERE t.trigger_id=$1 AND t.kind='schedule' AND t.removed_in_snapshot_id IS NULL", + vec![Parameter::Id(trigger_id)]).await?; + let Some(trigger) = rows + .into_iter() + .next() + .map(decode_row::) + .transpose()? + else { + return Ok(None); + }; + if trigger.status != TriggerStatus::Running { + return Ok(None); + } + let now = transaction.now; + let (transaction, rows) = query( + transaction, + "SELECT r.run_id FROM sqb_runs r WHERE r.location_id=$1 AND r.target=$2 AND r.job_key=$3 \ + AND r.status='queued' AND r.cancel_requested_at IS NULL \ + AND r.worker_instance IS NULL ORDER BY r.queued_at, r.run_id LIMIT 1", + vec![ + Parameter::Id(trigger.location_id), + Parameter::Text(trigger.target.clone()), + Parameter::Text(trigger.job_key.clone()), + ], + ) + .await?; + let waiting_run = rows + .first() + .and_then(|row| row.get("run_id")) + .and_then(serde_json::Value::as_str) + .map(Uuid::parse_str) + .transpose() + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let plan = plan_ticks( + &trigger.definition_json, + &TickState { + trigger_id, + started_at: trigger.started_at, + last_tick_for: trigger.last_tick_for, + waiting_run, + }, + now, + )?; + let transaction = persist_ticks(transaction, (&trigger, lease), &plan).await?; + let transaction = check_lease(transaction, lease).await?; + commit(transaction).await?; + Ok(Some(plan)) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_next_cron_time.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_next_cron_time.rs new file mode 100644 index 0000000000..b06ca823e1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_next_cron_time.rs @@ -0,0 +1,31 @@ +//! Calculate the first distinct UTC tick after a persisted cursor. + +use chrono::{DateTime, Timelike, Utc}; + +use crate::scheduler::_helpers::calendar::{resolve_wall_time, search_start}; +use crate::scheduler::models::CronSchedule; +use crate::store::errors::ControlError; + +/// Later fold for ambiguities, IANA gap end for missing times, searching from the cursor's whole second. +pub(in crate::scheduler) fn next_cron_time( + schedule: &CronSchedule, + after: DateTime, +) -> Result, ControlError> { + let mut cursor = search_start(schedule.timezone, after) + .with_nanosecond(0) + .ok_or_else(|| ControlError::Invalid("cron cursor is out of range".into()))? + .and_utc(); + loop { + let candidate = schedule + .expression + .find_next_occurrence(&cursor, false) + .map_err(|error| { + ControlError::Invalid(format!("cron has no next occurrence: {error}")) + })?; + let time = resolve_wall_time(schedule.timezone, candidate.naive_utc())?; + if time > after { + return Ok(time); + } + cursor = candidate; + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_plan_ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_plan_ticks.rs new file mode 100644 index 0000000000..9a10a16ce3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_plan_ticks.rs @@ -0,0 +1,34 @@ +//! Pure scheduling decisions: crashes and clock jumps cannot change a tick's identity. + +use chrono::{DateTime, TimeDelta, Utc}; + +use crate::declarations::models::TriggerDefinition; +use crate::scheduler::_helpers::expansion::latest_occurrence; +use crate::scheduler::_helpers::ticks::action; +use crate::scheduler::main::next_schedule_time::next_schedule_time; +use crate::scheduler::models::{TickPlan, TickState}; +use crate::store::errors::ControlError; + +/// Only the latest missed occurrence ticks; a waiting unclaimed run covers it instead of queueing another. +pub(in crate::scheduler) fn plan_ticks( + trigger: &TriggerDefinition, + state: &TickState, + now: DateTime, +) -> Result { + let after = match state.last_tick_for { + Some(last) => last, + None => state + .started_at + .checked_sub_signed(TimeDelta::microseconds(1)) + .ok_or_else(|| ControlError::Invalid("trigger start is out of range".into()))?, + }; + let latest = latest_occurrence(trigger, now)?.filter(|time| *time > after); + Ok(TickPlan { + actions: latest + .map(|time| action(state.trigger_id, time, state.waiting_run)) + .into_iter() + .collect(), + last_tick_for: latest.or(state.last_tick_for), + next_due_at: next_schedule_time(&trigger.schedule, latest.unwrap_or(after))?, + }) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_previous_cron_time.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_previous_cron_time.rs new file mode 100644 index 0000000000..fdc29f8c28 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_previous_cron_time.rs @@ -0,0 +1,35 @@ +//! Locate the latest due tick without expanding an unbounded catch-up backlog. + +use chrono::{DateTime, Timelike, Utc}; + +use crate::scheduler::_helpers::calendar::resolve_wall_time; +use crate::scheduler::models::CronSchedule; +use crate::store::errors::ControlError; + +/// Forward expansion's fold and gap mapping, searching from `at`'s whole second so results stay on occurrences. +pub(in crate::scheduler) fn previous_cron_time( + schedule: &CronSchedule, + at: DateTime, +) -> Result, ControlError> { + let mut cursor = at + .with_timezone(&schedule.timezone) + .naive_local() + .with_nanosecond(0) + .ok_or_else(|| ControlError::Invalid("cron search time is out of range".into()))? + .and_utc(); + let mut inclusive = true; + loop { + let candidate = schedule + .expression + .find_previous_occurrence(&cursor, inclusive) + .map_err(|error| { + ControlError::Invalid(format!("cron has no previous occurrence: {error}")) + })?; + let time = resolve_wall_time(schedule.timezone, candidate.naive_utc())?; + if time <= at { + return Ok(time); + } + cursor = candidate; + inclusive = false; + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs new file mode 100644 index 0000000000..c6267cfc84 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs @@ -0,0 +1,48 @@ +//! A cycle renews leadership and drains a bounded set of persisted due triggers. + +use std::time::Duration; + +use crate::runs::constants::MONITOR_BATCH_SIZE; +use crate::runs::main::monitor_runs::monitor_runs; +use crate::scheduler::_helpers::drain_due_triggers::{drain_due_triggers, wait_for_next_due}; +use crate::scheduler::_helpers::tick_retention::prune_ticks_when_due; +use crate::scheduler::constants::RENEW_INTERVAL_SECONDS; +use crate::scheduler::main::_acquire_lease::acquire_lease; +use crate::scheduler::main::renew_lease::renew_lease; +use crate::scheduler::models::{SchedulerCycle, SchedulerState}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +pub(crate) async fn scheduler_cycle( + store: &ControlStore, + mut state: SchedulerState, +) -> Result<(SchedulerState, SchedulerCycle), ControlError> { + let renewal = match state.lease.as_ref() { + Some(lease) => renew_lease(store, lease).await.map(Some), + None => acquire_lease(store, state.holder).await, + }; + state.lease = match renewal { + Ok(lease) => lease, + Err(ControlError::Fenced) => None, + Err(error) => return Err(error), + }; + let cycle = SchedulerCycle { + leader: state.lease.is_some(), + evaluated: 0, + ticks: 0, + unresponsive: 0, + wait_for: Duration::from_secs(RENEW_INTERVAL_SECONDS), + }; + let Some(lease) = state.lease.clone() else { + return Ok((state, cycle)); + }; + let (lease, mut cycle) = drain_due_triggers(store, lease, cycle).await?; + cycle.unresponsive = monitor_runs(store, &lease).await?; + if cycle.unresponsive == MONITOR_BATCH_SIZE { + cycle.wait_for = Duration::ZERO; + } + state.pruned_at = prune_ticks_when_due(store, &lease, state.pruned_at).await?; + state.lease = Some(lease); + cycle.wait_for = wait_for_next_due(store, cycle.wait_for).await?; + Ok((state, cycle)) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/check_lease.rs b/crates/sqlbuild-orchestration/src/scheduler/main/check_lease.rs new file mode 100644 index 0000000000..4c9d7a9247 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/check_lease.rs @@ -0,0 +1,22 @@ +//! Shared fence validation using a fresh database-clock reading in the writer transaction. + +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::main::refresh_clock::refresh_clock; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn check_lease( + transaction: ControlTransaction, + lease: &SchedulerLease, +) -> Result { + let transaction = refresh_clock(transaction).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT epoch FROM sqb_scheduler_lease WHERE name='scheduler' AND holder=$1 AND epoch=$2 AND expires_at>$3", + vec![Parameter::Id(lease.holder), Parameter::Integer(lease.epoch), Parameter::Time(now)]).await?; + if rows.is_empty() { + return Err(ControlError::Fenced); + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/list_ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/main/list_ticks.rs new file mode 100644 index 0000000000..391f5149fa --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/list_ticks.rs @@ -0,0 +1,29 @@ +//! Tick history contains immutable evaluations, including covered schedule times. + +use uuid::Uuid; + +use crate::scheduler::models::TickRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn list_ticks( + store: &ControlStore, + trigger_id: Uuid, + limit: u32, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, run_id, covered_by_run_id \ + FROM sqb_ticks WHERE trigger_id=$1 ORDER BY scheduled_for DESC LIMIT $2", + vec![Parameter::Id(trigger_id), Parameter::Integer(i64::from(limit.min(1000)))]).await?; + let ticks = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(ticks) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs new file mode 100644 index 0000000000..f91c3d1180 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs @@ -0,0 +1,12 @@ +pub(in crate::scheduler) mod _acquire_lease; +pub(in crate::scheduler) mod _evaluate_trigger; +pub(in crate::scheduler) mod _next_cron_time; +pub(in crate::scheduler) mod _plan_ticks; +pub(in crate::scheduler) mod _previous_cron_time; +pub(in crate::scheduler) mod _scheduler_cycle; +pub(crate) mod check_lease; +pub mod list_ticks; +pub(crate) mod next_schedule_time; +pub mod parse_cron; +pub mod renew_lease; +pub mod run_scheduler; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs b/crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs new file mode 100644 index 0000000000..880e4ad7c9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs @@ -0,0 +1,28 @@ +//! Find the next due time for a resolved cron or calendar declaration. + +use std::ops::Bound::{Excluded, Unbounded}; + +use chrono::{DateTime, Utc}; + +use crate::declarations::models::Schedule; +use crate::scheduler::main::_next_cron_time::next_cron_time; +use crate::scheduler::main::parse_cron::parse_cron; +use crate::store::errors::ControlError; + +pub(crate) fn next_schedule_time( + schedule: &Schedule, + after: DateTime, +) -> Result>, ControlError> { + match schedule { + Schedule::Cron { + expression, + timezone, + } => Ok(Some(next_cron_time( + &parse_cron(expression, timezone)?, + after, + )?)), + Schedule::Calendar { times, .. } => { + Ok(times.range((Excluded(after), Unbounded)).next().copied()) + } + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/parse_cron.rs b/crates/sqlbuild-orchestration/src/scheduler/main/parse_cron.rs new file mode 100644 index 0000000000..50dbeee4b6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/parse_cron.rs @@ -0,0 +1,24 @@ +//! Validate POSIX minute-resolution schedules and IANA time zones before publication. + +use croner::parser::{CronParser, Seconds, Year}; + +use crate::scheduler::models::CronSchedule; +use crate::store::errors::ControlError; + +/// Day-of-month and day-of-week use POSIX OR, not Quartz AND semantics. +pub fn parse_cron(expression: &str, timezone: &str) -> Result { + let expression = CronParser::builder() + .seconds(Seconds::Disallowed) + .year(Year::Disallowed) + .dom_and_dow(false) + .build() + .parse(expression) + .map_err(|error| ControlError::Invalid(format!("invalid cron expression: {error}")))?; + let timezone = timezone + .parse() + .map_err(|error| ControlError::Invalid(format!("invalid IANA time zone: {error}")))?; + Ok(CronSchedule { + expression, + timezone, + }) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/renew_lease.rs b/crates/sqlbuild-orchestration/src/scheduler/main/renew_lease.rs new file mode 100644 index 0000000000..5e9811a5a0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/renew_lease.rs @@ -0,0 +1,30 @@ +//! A paused or superseded scheduler cannot renew an expired fence. + +use chrono::TimeDelta; + +use crate::scheduler::constants::LEASE_TTL_SECONDS; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Renewal preserves the epoch; only takeover allocates a new epoch. +pub async fn renew_lease( + store: &ControlStore, + lease: &SchedulerLease, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let transaction = check_lease(transaction, lease).await?; + let expires_at = transaction.now + TimeDelta::seconds(LEASE_TTL_SECONDS); + let (transaction, _) = query(transaction, + "UPDATE sqb_scheduler_lease SET expires_at=$1 WHERE name='scheduler' AND holder=$2 AND epoch=$3", + vec![Parameter::Time(expires_at), Parameter::Id(lease.holder), Parameter::Integer(lease.epoch)]).await?; + commit(transaction).await?; + Ok(SchedulerLease { + expires_at, + ..lease.clone() + }) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs new file mode 100644 index 0000000000..4233635a0d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs @@ -0,0 +1,35 @@ +//! Embedded and standalone roles share the same interruptible, durable scheduling loop. + +use tokio::sync::watch; + +use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; +use crate::scheduler::models::SchedulerState; +use crate::scheduler::types::RunLauncher; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +/// Durable database state drives work; shutdown interrupts the wait. +pub async fn run_scheduler( + store: &ControlStore, + mut state: SchedulerState, + mut shutdown: watch::Receiver, + mut launcher: impl RunLauncher, +) -> Result<(), ControlError> { + let mut last_lease = None; + let result = async { + while !*shutdown.borrow() { + let (next_state, cycle) = scheduler_cycle(store, state).await?; + state = next_state; + last_lease = state.lease.clone(); + launcher.poll(store, state.lease.as_ref()).await?; + tokio::select! { + _ = tokio::time::sleep(cycle.wait_for.min(std::time::Duration::from_secs(1))) => {}, + result = shutdown.changed() => { if result.is_err() { break; } }, + } + } + Ok(()) + } + .await; + let stopped = launcher.shutdown(store, last_lease.as_ref()).await; + result.and(stopped) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/mod.rs new file mode 100644 index 0000000000..f5b66b8aca --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/mod.rs @@ -0,0 +1,10 @@ +//! Fenced scheduler leadership and deterministic decisions over durable due times. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/scheduler/models.rs b/crates/sqlbuild-orchestration/src/scheduler/models.rs new file mode 100644 index 0000000000..adc5f713d5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/models.rs @@ -0,0 +1,116 @@ +//! Values owned by the scheduler, independent of Python and runtime timers. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::declarations::models::{JobDefinition, TriggerDefinition, TriggerStatus}; + +/// A holder must present the exact epoch for every scheduler write. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SchedulerLease { + pub holder: Uuid, + pub epoch: i64, + pub expires_at: DateTime, +} + +/// Process identity survives cycles; the database remains authoritative for leadership. +#[derive(Clone, Debug)] +pub struct SchedulerState { + pub(crate) holder: Uuid, + pub(crate) lease: Option, + pub(crate) pruned_at: Option, +} + +impl SchedulerState { + #[must_use] + pub const fn new(holder: Uuid) -> Self { + Self { + holder, + lease: None, + pruned_at: None, + } + } +} + +#[derive(Clone, Debug)] +pub struct SchedulerCycle { + pub leader: bool, + pub evaluated: usize, + pub ticks: usize, + pub unresponsive: usize, + pub wait_for: std::time::Duration, +} + +/// Validated at the native boundary and evaluated as wall-clock times with SQLBuild's DST policy. +#[derive(Clone, Debug)] +pub struct CronSchedule { + pub(crate) expression: croner::Cron, + pub(crate) timezone: chrono_tz::Tz, +} + +/// The durable cursor and waiting work an evaluation decides from; due times come from the declaration. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TickState { + pub trigger_id: Uuid, + pub started_at: DateTime, + pub last_tick_for: Option>, + /// Only unclaimed waiting work can cover a later tick; executing work cannot cover future data. + pub waiting_run: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub enum TickDecision { + Requested { run_id: Uuid }, + Covered { covered_by_run_id: Uuid }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TickAction { + pub tick_id: Uuid, + pub scheduled_for: DateTime, + pub decision: TickDecision, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TickPlan { + pub actions: Vec, + pub last_tick_for: Option>, + pub next_due_at: Option>, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct DueTrigger { + pub(crate) trigger_id: Uuid, + pub(crate) location_id: Uuid, + pub(crate) target: String, + pub(crate) job_key: String, + pub(crate) snapshot_id: Uuid, + pub(crate) image_id: Option, + pub(crate) definition_json: TriggerDefinition, + pub(crate) job_json: JobDefinition, + pub(crate) run_kind: crate::runs::models::RunKind, + pub(crate) started_at: DateTime, + pub(crate) last_tick_for: Option>, + pub(crate) status: TriggerStatus, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum TickOutcome { + Requested, + Covered, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct TickRecord { + pub tick_id: Uuid, + pub trigger_id: Uuid, + pub scheduled_for: DateTime, + pub evaluated_at: DateTime, + pub epoch: i64, + pub outcome: TickOutcome, + pub run_id: Option, + pub covered_by_run_id: Option, +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests.rs b/crates/sqlbuild-orchestration/src/scheduler/tests.rs new file mode 100644 index 0000000000..6f30f95476 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests.rs @@ -0,0 +1,14 @@ +#[path = "../../tests/scheduler/test_contracts.rs"] +pub(crate) mod contracts; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_cron.rs"] +mod test_cron; +#[path = "../../tests/scheduler/test_sensor_fairness.rs"] +mod test_sensor_fairness; +#[path = "tests/test_simulation.rs"] +mod test_simulation; +#[path = "tests/test_ticks.rs"] +mod test_ticks; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs new file mode 100644 index 0000000000..f1a299164c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs @@ -0,0 +1,235 @@ +//! Clock-independent schedule fixtures. + +use std::collections::{BTreeMap, BTreeSet}; + +use chrono::{DateTime, TimeDelta, Utc}; +use proptest::prelude::any; +use proptest::test_runner::{Config, TestCaseError, TestCaseResult, TestRunner}; +use proptest::{prop_assert, prop_assert_eq}; +use uuid::Uuid; + +use crate::declarations::models::{Schedule, TriggerDefinition, TriggerStatus}; +use crate::scheduler::main::_plan_ticks::plan_ticks; +use crate::scheduler::models::{TickAction, TickDecision, TickPlan, TickState}; +use crate::scheduler::tests::test_types::TickTestCase; + +pub(super) fn time(second: i64) -> DateTime { + DateTime::from_timestamp(second, 0).expect("synthetic schedule time is representable") +} + +pub(super) fn state(test_case: &TickTestCase) -> (TriggerDefinition, TickState) { + ( + calendar(test_case.due_seconds.iter().copied()), + TickState { + trigger_id: Uuid::from_u128(19), + started_at: time(0), + last_tick_for: test_case.last_second.map(time), + waiting_run: test_case.active.then_some(Uuid::from_u128(31)), + }, + ) +} + +fn calendar(seconds: impl Iterator) -> TriggerDefinition { + TriggerDefinition { + description: "orders calendar".into(), + job_key: "orders".into(), + starts_at: Some(time(0)), + default_status: TriggerStatus::Running, + schedule: Schedule::Calendar { + times: seconds.map(time).collect(), + refresh_at: time(2000), + }, + } +} + +pub(super) fn request_seconds(plan: &TickPlan) -> Vec { + plan.actions + .iter() + .filter(|action| matches!(action.decision, TickDecision::Requested { .. })) + .map(|action| action.scheduled_for.timestamp()) + .collect() +} + +pub(super) fn covered_seconds(plan: &TickPlan) -> Vec { + plan.actions + .iter() + .filter(|action| matches!(action.decision, TickDecision::Covered { .. })) + .map(|action| action.scheduled_for.timestamp()) + .collect() +} + +/// Under crashes and clock jumps a tick is never early, repeated or invented, and the latest due time always ticks. +fn exercise_history(due: Vec, history: Vec<(i64, bool)>) -> TestCaseResult { + let expected: BTreeSet = due.iter().copied().collect(); + let trigger = calendar(due.into_iter()); + let mut state = TickState { + trigger_id: Uuid::from_u128(19), + started_at: time(0), + last_tick_for: None, + waiting_run: None, + }; + let mut committed: BTreeMap = BTreeMap::new(); + let final_step = std::iter::once((1001, true)); + for (clock, commit_succeeds) in history.into_iter().chain(final_step) { + let plan = plan_ticks(&trigger, &state, time(clock)) + .map_err(|error| TestCaseError::fail(error.to_string()))?; + let Some(plan) = commit_succeeds.then_some(plan) else { + continue; + }; + for action in plan.actions { + prop_assert!(action.scheduled_for.timestamp() <= clock); + prop_assert!( + committed + .insert(action.scheduled_for.timestamp(), action) + .is_none(), + "a committed tick was emitted twice" + ); + } + state.last_tick_for = plan.last_tick_for; + } + let ticked: BTreeSet = committed.keys().copied().collect(); + prop_assert!(ticked.is_subset(&expected), "a tick was invented"); + prop_assert_eq!( + ticked.last(), + expected.last(), + "the latest due time was lost" + ); + prop_assert_eq!( + committed + .values() + .map(|action| action.tick_id) + .collect::>() + .len(), + committed.len() + ); + Ok(()) +} + +pub(super) fn simulate() -> Result<(), String> { + let strategy = ( + proptest::collection::vec(0_i64..1000, 0..128), + proptest::collection::vec((-100_i64..1100, any::()), 0..128), + ); + let mut property_runner = TestRunner::new(Config { + cases: 512, + failure_persistence: None, + ..Config::default() + }); + property_runner + .run(&strategy, |(due, history)| exercise_history(due, history)) + .map_err(|error| error.to_string()) +} + +/// Reference occurrences found day by day from local wall times, independent of cursor search. +fn reference_occurrences( + timezone: chrono_tz::Tz, + wall_times: &[(u32, u32)], + (start, end): (DateTime, DateTime), +) -> BTreeSet> { + use chrono::TimeZone; + let first_date = start.with_timezone(&timezone).date_naive() - TimeDelta::days(1); + let days = (end.with_timezone(&timezone).date_naive() - first_date).num_days(); + (0..=days) + .flat_map(|day| { + wall_times.iter().map(move |(hour, minute)| { + let wall = (first_date + TimeDelta::days(day)) + .and_hms_opt(*hour, *minute, 0) + .expect("wall time"); + // The later fold for repeated times; the first existing minute after a gap. + (0..=180) + .find_map(|offset| { + timezone + .from_local_datetime(&(wall + TimeDelta::minutes(offset))) + .latest() + }) + .expect("a local time exists within three hours") + .to_utc() + }) + }) + .filter(|instant| *instant > start && *instant <= end) + .collect() +} + +/// Real cron planning from a start time; each committed tick is due, distinct and later than the last. +fn cron_ticks( + trigger: &TriggerDefinition, + (start, end): (DateTime, DateTime), + steps: impl Iterator, +) -> Result>, TestCaseError> { + let mut state = TickState { + trigger_id: Uuid::from_u128(23), + started_at: start, + last_tick_for: None, + waiting_run: None, + }; + let mut committed: BTreeSet> = BTreeSet::new(); + let mut clock = start; + for (jump, commit) in steps.chain(std::iter::once((end - start, true))) { + clock = (clock + jump).clamp(start, end); + let plan = plan_ticks(trigger, &state, clock) + .map_err(|error| TestCaseError::fail(error.to_string()))?; + for action in plan.actions.iter().filter(|_| commit) { + prop_assert!( + action.scheduled_for <= clock, + "a tick ran before it was due" + ); + prop_assert!( + committed + .last() + .is_none_or(|last| action.scheduled_for > *last), + "a committed tick was emitted twice or out of order" + ); + committed.insert(action.scheduled_for); + } + state.last_tick_for = [state.last_tick_for, plan.last_tick_for][usize::from(commit)]; + } + Ok(committed) +} + +/// Months spanning both 2026 New York transitions; clocks jump up to a day back or four days on. +pub(super) fn simulate_cron(expression: &str, wall_times: &[(u32, u32)]) -> Result<(), String> { + let start = DateTime::parse_from_rfc3339("2026-02-25T00:00:00.25Z") + .expect("start") + .to_utc(); + let end = DateTime::parse_from_rfc3339("2026-11-10T00:00:00Z") + .expect("end") + .to_utc(); + let trigger = TriggerDefinition { + description: "Simulated orders".into(), + job_key: "orders".into(), + schedule: Schedule::Cron { + expression: expression.into(), + timezone: "America/New_York".into(), + }, + starts_at: Some(start), + default_status: TriggerStatus::default(), + }; + let strategy = proptest::collection::vec((-1_440_i64..5_760, any::()), 0..256); + let mut property_runner = TestRunner::new(Config { + cases: 64, + failure_persistence: None, + ..Config::default() + }); + let reference = reference_occurrences(chrono_tz::America::New_York, wall_times, (start, end)); + let every_step = std::iter::repeat_n((TimeDelta::minutes(10), true), 37_500); + let stepped = + cron_ticks(&trigger, (start, end), every_step).map_err(|error| error.to_string())?; + (stepped == reference) + .then_some(()) + .ok_or("ten-minute evaluation lost, repeated or invented an occurrence")?; + property_runner + .run(&strategy, |history| { + let steps = history + .into_iter() + .map(|(minutes, commit)| (TimeDelta::minutes(minutes), commit)); + let ticked = cron_ticks(&trigger, (start, end), steps)?; + prop_assert!(ticked.is_subset(&reference), "a tick was invented"); + prop_assert_eq!( + ticked.last(), + reference.last(), + "the latest due time was lost" + ); + Ok(()) + }) + .map_err(|error| error.to_string()) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs new file mode 100644 index 0000000000..c7d258cc3d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs @@ -0,0 +1,179 @@ +use chrono::{DateTime, SecondsFormat}; + +use crate::scheduler::main::_next_cron_time::next_cron_time; +use crate::scheduler::main::_previous_cron_time::previous_cron_time; +use crate::scheduler::main::parse_cron::parse_cron; +use crate::scheduler::tests::test_types::{CronTestCase, PreviousCronTestCase}; + +#[test] +fn given_cron_and_iana_time_zone_when_crossing_dst_then_utc_ticks_follow_the_explicit_policy() { + let test_cases = [ + CronTestCase { + description: "spring gap ends at 03:00, not 03:30", + expression: "30 2 * * *", + timezone: "America/New_York", + after: "2026-03-08T00:00:00Z", + expected_next: "2026-03-08T07:00:00Z", + }, + CronTestCase { + description: "ambiguous daily time uses the later fold", + expression: "30 1 * * *", + timezone: "America/New_York", + after: "2026-11-01T00:00:00Z", + expected_next: "2026-11-01T06:30:00Z", + }, + CronTestCase { + description: "restarting in the first fold does not skip a smaller future wall time", + expression: "30 1 * * *", + timezone: "America/New_York", + after: "2026-11-01T05:45:00Z", + expected_next: "2026-11-01T06:30:00Z", + }, + CronTestCase { + description: "a persisted later-fold cursor does not repeat the tick", + expression: "30 1 * * *", + timezone: "America/New_York", + after: "2026-11-01T06:30:00Z", + expected_next: "2026-11-02T06:30:00Z", + }, + CronTestCase { + description: "wildcard schedules also select only the later fold", + expression: "* * * * *", + timezone: "America/New_York", + after: "2026-11-01T05:45:00Z", + expected_next: "2026-11-01T06:00:00Z", + }, + CronTestCase { + description: "half-hour spring gap moves to the actual gap end", + expression: "15 2 * * *", + timezone: "Australia/Lord_Howe", + after: "2026-10-03T13:00:00Z", + expected_next: "2026-10-03T15:30:00Z", + }, + CronTestCase { + description: "half-hour autumn fold uses the later 01:45", + expression: "45 1 * * *", + timezone: "Australia/Lord_Howe", + after: "2026-04-04T13:00:00Z", + expected_next: "2026-04-04T15:15:00Z", + }, + CronTestCase { + description: "an entire skipped date moves to the first instant of the next date", + expression: "0 12 * * *", + timezone: "Pacific/Apia", + after: "2011-12-30T00:00:00Z", + expected_next: "2011-12-30T10:00:00Z", + }, + CronTestCase { + description: "POSIX day-of-month and weekday are alternatives", + expression: "0 9 13 * MON", + timezone: "UTC", + after: "2026-10-10T12:00:00Z", + expected_next: "2026-10-12T09:00:00Z", + }, + CronTestCase { + description: "UTC schedules do not introduce local-zone shifts", + expression: "17 6 * * *", + timezone: "UTC", + after: "2026-03-08T06:17:00Z", + expected_next: "2026-03-09T06:17:00Z", + }, + CronTestCase { + description: "a first tick after a sub-second trigger start lands on the whole second", + expression: "0 0 * * *", + timezone: "UTC", + after: "2026-07-26T23:59:59.999999Z", + expected_next: "2026-07-27T00:00:00Z", + }, + CronTestCase { + description: "a sub-second cursor past its occurrence does not repeat that occurrence", + expression: "0 0 * * *", + timezone: "UTC", + after: "2026-07-27T00:00:00.999999Z", + expected_next: "2026-07-28T00:00:00Z", + }, + ]; + for test_case in test_cases { + let schedule = + parse_cron(test_case.expression, test_case.timezone).expect("valid synthetic cron"); + let after = DateTime::parse_from_rfc3339(test_case.after) + .expect("valid start timestamp") + .to_utc(); + let observed = next_cron_time(&schedule, after).expect("next cron time exists"); + assert_eq!( + observed.to_rfc3339_opts(SecondsFormat::AutoSi, true), + test_case.expected_next, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_latest_tick_search_when_crossing_folds_and_gaps_then_it_agrees_with_forward_tick_policy() +{ + let test_cases = [ + PreviousCronTestCase { + description: "first fold excludes every ambiguous minute", + expression: "* * * * *", + timezone: "America/New_York", + at: "2026-11-01T05:45:00Z", + expected_previous: "2026-11-01T04:59:00Z", + }, + PreviousCronTestCase { + description: "second fold includes the later ambiguous instant", + expression: "* * * * *", + timezone: "America/New_York", + at: "2026-11-01T06:15:00Z", + expected_previous: "2026-11-01T06:15:00Z", + }, + PreviousCronTestCase { + description: "daily missing time fires exactly at the gap end", + expression: "30 2 * * *", + timezone: "America/New_York", + at: "2026-03-08T07:00:00Z", + expected_previous: "2026-03-08T07:00:00Z", + }, + PreviousCronTestCase { + description: "before gap end uses the preceding day", + expression: "30 2 * * *", + timezone: "America/New_York", + at: "2026-03-08T06:59:59Z", + expected_previous: "2026-03-07T07:30:00Z", + }, + PreviousCronTestCase { + description: "half-hour gap resolves to its actual end", + expression: "15 2 * * *", + timezone: "Australia/Lord_Howe", + at: "2026-10-03T15:30:00Z", + expected_previous: "2026-10-03T15:30:00Z", + }, + PreviousCronTestCase { + description: "skipped-date cron fires at the first representable instant", + expression: "0 12 * * *", + timezone: "Pacific/Apia", + at: "2011-12-30T10:00:00Z", + expected_previous: "2011-12-30T10:00:00Z", + }, + PreviousCronTestCase { + description: "a sub-second search time finds the whole-second occurrence", + expression: "0 0 * * *", + timezone: "UTC", + at: "2026-07-27T00:00:00.5Z", + expected_previous: "2026-07-27T00:00:00Z", + }, + ]; + for test_case in test_cases { + let cron = parse_cron(test_case.expression, test_case.timezone).expect("valid cron"); + let at = DateTime::parse_from_rfc3339(test_case.at) + .expect("search time") + .to_utc(); + let previous = previous_cron_time(&cron, at).expect("previous scheduled instant"); + assert_eq!( + previous.to_rfc3339_opts(SecondsFormat::AutoSi, true), + test_case.expected_previous, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_simulation.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_simulation.rs new file mode 100644 index 0000000000..ac881ce02e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_simulation.rs @@ -0,0 +1,53 @@ +use crate::scheduler::tests::helpers::{simulate, simulate_cron}; +use crate::scheduler::tests::test_types::{CronSimulationTestCase, SimulationTestCase}; + +#[test] +fn given_random_crashes_and_clock_jumps_when_restarting_then_every_due_tick_commits_once() { + let test_cases = [SimulationTestCase { + description: "latest catch-up keeps covered tick history through crashes", + expected_success: true, + }]; + for test_case in test_cases { + let observed = simulate(); + assert_eq!( + observed.is_ok(), + test_case.expected_success, + "{}: {observed:?}", + test_case.description + ); + } +} + +#[test] +fn given_dst_transitions_crashes_and_clock_jumps_when_expanding_real_cron_then_every_occurrence_ticks_once() + { + let test_cases = [ + CronSimulationTestCase { + description: "a daily time in the autumn fold ticks once, at the later instant", + expression: "30 1 * * *", + wall_times: &[(1, 30)], + expected_success: true, + }, + CronSimulationTestCase { + description: "a daily time in the spring gap ticks once, at the gap end", + expression: "30 2 * * *", + wall_times: &[(2, 30)], + expected_success: true, + }, + CronSimulationTestCase { + description: "several times inside the fold and gap keep their distinct instants", + expression: "*/30 1-2 * * *", + wall_times: &[(1, 0), (1, 30), (2, 0), (2, 30)], + expected_success: true, + }, + ]; + for test_case in test_cases { + let observed = simulate_cron(test_case.expression, test_case.wall_times); + assert_eq!( + observed.is_ok(), + test_case.expected_success, + "{}: {observed:?}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs new file mode 100644 index 0000000000..0dfbe8087b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs @@ -0,0 +1,90 @@ +use crate::scheduler::main::_plan_ticks::plan_ticks; +use crate::scheduler::tests::helpers::{covered_seconds, request_seconds, state, time}; +use crate::scheduler::tests::test_types::TickTestCase; + +#[test] +fn given_durable_schedule_state_when_time_advances_then_decisions_preserve_due_ticks() { + let test_cases = [ + TickTestCase { + description: "catch-up requests only the latest missed time and records nothing earlier", + due_seconds: vec![30, 10, 20, 10, 40], + last_second: None, + now_second: 30, + active: false, + expected_requests: vec![30], + expected_covered: vec![], + expected_last: Some(30), + expected_next: Some(40), + }, + TickTestCase { + description: "the latest missed time is covered by the already waiting run", + due_seconds: vec![10, 20, 30, 40], + last_second: Some(10), + now_second: 30, + active: true, + expected_requests: vec![], + expected_covered: vec![30], + expected_last: Some(30), + expected_next: Some(40), + }, + TickTestCase { + description: "a backward clock jump cannot move the durable cursor backward", + due_seconds: vec![10, 20, 30, 40], + last_second: Some(30), + now_second: 20, + active: false, + expected_requests: vec![], + expected_covered: vec![], + expected_last: Some(30), + expected_next: Some(40), + }, + ]; + for test_case in test_cases { + let (trigger, input) = state(&test_case); + let plan = + plan_ticks(&trigger, &input, time(test_case.now_second)).expect("plan valid schedule"); + assert_eq!( + request_seconds(&plan), + test_case.expected_requests, + "{}", + test_case.description + ); + assert_eq!( + covered_seconds(&plan), + test_case.expected_covered, + "{}", + test_case.description + ); + assert_eq!( + plan.last_tick_for.map(|time| time.timestamp()), + test_case.expected_last + ); + assert_eq!( + plan.next_due_at.map(|time| time.timestamp()), + test_case.expected_next + ); + let retried = + plan_ticks(&trigger, &input, time(test_case.now_second)).expect("retry schedule"); + assert_eq!( + plan, retried, + "a retry has exactly the same deterministic tick and run IDs" + ); + let tick_ids: std::collections::BTreeSet<_> = + plan.actions.iter().map(|action| action.tick_id).collect(); + assert_eq!( + tick_ids.len(), + test_case.expected_requests.len() + test_case.expected_covered.len() + ); + let advanced = crate::scheduler::models::TickState { + last_tick_for: plan.last_tick_for, + ..input + }; + let second = plan_ticks(&trigger, &advanced, time(test_case.now_second)) + .expect("plan from committed cursor"); + assert!(second.actions.iter().all(|next| { + plan.actions + .iter() + .all(|before| before.tick_id != next.tick_id) + })); + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs new file mode 100644 index 0000000000..c58abe4898 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs @@ -0,0 +1,55 @@ +//! Inputs make cursor, catch-up, and coalescing decisions observably different. + +use crate::store::tests::store_test_types::Backend; + +pub(super) struct SensorFairnessTestCase { + pub(super) description: &'static str, + pub(super) backend: Backend, + pub(super) pending: usize, + pub(super) builtin_status: &'static str, + pub(super) expected_completed: usize, +} + +#[derive(Debug)] +pub(super) struct TickTestCase { + pub(super) description: &'static str, + pub(super) due_seconds: Vec, + pub(super) last_second: Option, + pub(super) now_second: i64, + pub(super) active: bool, + pub(super) expected_requests: Vec, + pub(super) expected_covered: Vec, + pub(super) expected_last: Option, + pub(super) expected_next: Option, +} + +#[derive(Debug)] +pub(super) struct CronTestCase { + pub(super) description: &'static str, + pub(super) expression: &'static str, + pub(super) timezone: &'static str, + pub(super) after: &'static str, + pub(super) expected_next: &'static str, +} + +#[derive(Debug)] +pub(super) struct PreviousCronTestCase { + pub(super) description: &'static str, + pub(super) expression: &'static str, + pub(super) timezone: &'static str, + pub(super) at: &'static str, + pub(super) expected_previous: &'static str, +} + +#[derive(Debug)] +pub(super) struct SimulationTestCase { + pub(super) description: &'static str, + pub(super) expected_success: bool, +} + +pub(super) struct CronSimulationTestCase { + pub(super) description: &'static str, + pub(super) expression: &'static str, + pub(super) wall_times: &'static [(u32, u32)], + pub(super) expected_success: bool, +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/types.rs b/crates/sqlbuild-orchestration/src/scheduler/types.rs new file mode 100644 index 0000000000..c32501956f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/types.rs @@ -0,0 +1,20 @@ +//! Compound scheduler return values. + +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +/// Physical process ownership survives leadership loss, but durable writes require a live fence. +pub trait RunLauncher { + fn poll( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> impl Future> + Send; + + fn shutdown( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> impl Future> + Send; +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs new file mode 100644 index 0000000000..7266843ef7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs @@ -0,0 +1,67 @@ +//! Indexed reads recover attempts and schedule projections without mutable phase transitions. + +use chrono::{DateTime, Utc}; +use uuid::Uuid; + +use crate::sensors::models::{DueSensor, SensorEvaluationFact}; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_sensor( + transaction: ControlTransaction, + trigger_id: Uuid, +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT t.trigger_id,t.location_id,t.target,t.snapshot_id,t.definition_json,s.status \ + FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id WHERE t.trigger_id=$1 AND t.kind='sensor' \ + AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL", + vec![Parameter::Id(trigger_id)]).await?; + Ok(( + transaction, + rows.into_iter().next().map(decode_row).transpose()?, + )) +} + +pub(crate) async fn last_attempt( + transaction: ControlTransaction, + trigger_id: Uuid, +) -> Result<(ControlTransaction, Option<(SensorEvaluationFact, bool)>), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT e.fact_json, CASE WHEN EXISTS (SELECT 1 FROM sqb_sensor_evaluation_events terminal \ + WHERE terminal.evaluation_id=e.evaluation_id AND terminal.phase<>'requested') THEN 1 ELSE 0 END AS terminal \ + FROM sqb_sensor_evaluation_events e WHERE e.trigger_id=$1 AND e.phase='requested' \ + ORDER BY e.occurred_at DESC, e.evaluation_id DESC LIMIT 1", + vec![Parameter::Id(trigger_id)]).await?; + let last = rows + .first() + .map(|row| { + let fact = serde_json::from_value(row["fact_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + Ok::<_, ControlError>((fact, row["terminal"].as_i64() == Some(1))) + }) + .transpose()?; + Ok((transaction, last)) +} + +pub(crate) async fn defer_sensor( + transaction: ControlTransaction, + trigger_id: Uuid, + next: DateTime, +) -> Result { + let now = transaction.now; + let (transaction, _) = query( + transaction, + "UPDATE sqb_trigger_state SET next_due_at=$1,updated_at=$2 WHERE trigger_id=$3 \ + AND next_due_at IS DISTINCT FROM $1", + vec![ + Parameter::Time(next), + Parameter::Time(now), + Parameter::Id(trigger_id), + ], + ) + .await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/completion.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/completion.rs new file mode 100644 index 0000000000..4e919e3129 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/completion.rs @@ -0,0 +1,71 @@ +//! Every evaluation result uses the same lease, snapshot, cursor and terminal boundary. + +use chrono::TimeDelta; + +use crate::declarations::models::TriggerStatus; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::sensors::_helpers::attempts::{defer_sensor, last_attempt, read_sensor}; +use crate::sensors::_helpers::facts::{append_evaluation, commit_cursor}; +use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; +use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, ControlTransaction, TransactionMode}; + +pub(crate) async fn pending_evaluation( + store: &ControlStore, + lease: &SchedulerLease, + evaluation: &SensorEvaluationRequest, +) -> Result, ControlError> { + let transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + if evaluation.epoch != lease.epoch { + return Err(ControlError::Fenced); + } + let (transaction, previous) = last_attempt(transaction, evaluation.trigger_id).await?; + let Some((previous, terminal)) = previous else { + return Err(ControlError::Fenced); + }; + if previous.request != *evaluation { + return Err(ControlError::Fenced); + } + if terminal { + commit(transaction).await?; + return Ok(None); + } + let (transaction, sensor) = read_sensor(transaction, evaluation.trigger_id).await?; + let (transaction, cursor) = read_sensor_cursor(transaction, evaluation.trigger_id).await?; + if sensor.is_none_or(|sensor| { + sensor.status != TriggerStatus::Running || sensor.snapshot_id != evaluation.snapshot_id + }) || cursor != evaluation.cursor + { + let transaction = + append_evaluation(transaction, evaluation, SensorEvaluationOutcome::Abandoned).await?; + commit(transaction).await?; + return Ok(None); + } + Ok(Some(transaction)) +} + +pub(crate) async fn finish_evaluation( + transaction: ControlTransaction, + lease: &SchedulerLease, + evaluation: &SensorEvaluationRequest, + outcome: SensorEvaluationOutcome, +) -> Result<(), ControlError> { + let cursor = match &outcome { + SensorEvaluationOutcome::Completed { cursor, .. } => Some(cursor.clone()), + _ => None, + }; + let mut transaction = append_evaluation(transaction, evaluation, outcome).await?; + if let Some(cursor) = cursor { + transaction = commit_cursor(transaction, evaluation.trigger_id, &cursor).await?; + } + let next = evaluation.requested_at + + TimeDelta::seconds(i64::from( + evaluation.definition.minimum_interval_seconds.get(), + )); + let transaction = defer_sensor(transaction, evaluation.trigger_id, next).await?; + commit(check_lease(transaction, lease).await?).await +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs new file mode 100644 index 0000000000..6b5ba9f245 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs @@ -0,0 +1,49 @@ +//! Each native condition consumes its bounded page inside the evaluation transaction. + +use uuid::Uuid; + +use crate::sensors::_helpers::nodes::{ + consume_node_events, consume_source_events, consume_stale_sources, +}; +use crate::sensors::_helpers::platform::consume_platform_events; +use crate::sensors::_helpers::runs::consume_run_events; +use crate::sensors::models::{SensorCondition, SensorCursor, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn consume_condition( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, +) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { + match &evaluation.definition.when { + SensorCondition::Custom { .. } => Err(ControlError::Invalid( + "custom sensors require a location result".into(), + )), + SensorCondition::PlatformEvent { kinds } => { + consume_platform_events(transaction, evaluation, kinds).await + } + SensorCondition::NodeEvent { selection, kinds } => { + consume_node_events(transaction, evaluation, selection, kinds).await + } + SensorCondition::SourceNewData { selection } => { + consume_source_events(transaction, evaluation, selection).await + } + SensorCondition::SourceStale { + selection, + after_seconds, + } => consume_stale_sources(transaction, evaluation, selection, *after_seconds).await, + SensorCondition::RunEvent { + job_key, + kinds, + executed_successfully, + } => { + consume_run_events( + transaction, + evaluation, + (job_key, kinds), + executed_successfully.as_ref(), + ) + .await + } + } +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs new file mode 100644 index 0000000000..c2e65bb427 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs @@ -0,0 +1,104 @@ +//! Custom callbacks cross only the existing location-scoped outbound-work boundary. + +use std::collections::BTreeSet; + +use chrono::TimeDelta; +use serde_json::json; +use uuid::Uuid; + +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::models::{LocationWork, LocationWorkOutcome, LocationWorkRequest}; +use crate::sensors::_helpers::requests::queue_sensor_run; +use crate::sensors::models::{ + CustomSensorOutcome, SensorCondition, SensorEvaluationOutcome, SensorEvaluationRequest, +}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn request_evaluation_work( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, +) -> Result { + if !matches!(evaluation.definition.when, SensorCondition::Custom { .. }) { + return Ok(transaction); + } + let request = LocationWorkRequest { + request_id: evaluation.evaluation_id, + location_id: evaluation.location_id, + target: evaluation.target.clone(), + reason: "Evaluate custom sensor".into(), + work: LocationWork::EvaluateSensor { + evaluation: Box::new(evaluation.clone()), + }, + }; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_location_work \ + (request_id,location_id,target,snapshot_id,generation,request_json,requested_by,requested_at,expires_at) \ + SELECT $1,$2,$3,snapshot_id,generation,$4,'system:sensor',$5,$6 FROM sqb_snapshots WHERE snapshot_id=$7", + vec![Parameter::Id(request.request_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), + Parameter::Json(json!(request)), Parameter::Time(evaluation.requested_at), + Parameter::Time(evaluation.requested_at + TimeDelta::minutes(5)), Parameter::Id(evaluation.snapshot_id)]).await?; + Ok(transaction) +} + +pub(crate) async fn read_evaluation_result( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, work) = read_location_work(transaction, evaluation.evaluation_id).await?; + let work = work.ok_or_else(|| ControlError::Invalid("sensor has no location work".into()))?; + if work.record.result_json.is_none() { + let expired = work.record.expires_at <= transaction.now; + return Ok(( + transaction, + expired.then_some(SensorEvaluationOutcome::Abandoned), + )); + } + match work.record.result_json.map(|result| result.outcome) { + Some(LocationWorkOutcome::SensorEvaluation { outcome }) => { + apply_result(transaction, evaluation, outcome).await + } + Some(LocationWorkOutcome::Superseded {}) => { + Ok((transaction, Some(SensorEvaluationOutcome::Abandoned))) + } + _ => Err(ControlError::Invalid( + "sensor has an invalid location result".into(), + )), + } +} + +async fn apply_result( + mut transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + outcome: CustomSensorOutcome, +) -> Result<(ControlTransaction, Option), ControlError> { + let (custom, run_keys) = match outcome { + CustomSensorOutcome::Completed { cursor, run_keys } => (cursor, run_keys), + CustomSensorOutcome::Skipped { cursor, .. } => (cursor, Vec::new()), + CustomSensorOutcome::Failed => { + return Ok(( + transaction, + Some(SensorEvaluationOutcome::Failed { + code: "custom_sensor_failed".into(), + }), + )); + } + CustomSensorOutcome::TimedOut => { + return Ok((transaction, Some(SensorEvaluationOutcome::TimedOut))); + } + }; + let mut cursor = evaluation.cursor.clone(); + cursor.custom = custom; + let mut run_ids: Vec = Vec::new(); + let run_keys: BTreeSet = run_keys.into_iter().collect(); + for key in run_keys { + let run_id; + (transaction, run_id) = queue_sensor_run(transaction, evaluation, &key).await?; + run_ids.push(run_id); + } + Ok(( + transaction, + Some(SensorEvaluationOutcome::Completed { cursor, run_ids }), + )) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs new file mode 100644 index 0000000000..232ecf451a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs @@ -0,0 +1,53 @@ +//! Sensor evaluation phases are recorded once; the latest cursor lives on the trigger's state. + +use serde_json::json; +use uuid::Uuid; + +use crate::sensors::models::{ + SensorCursor, SensorEvaluationFact, SensorEvaluationOutcome, SensorEvaluationRequest, +}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{Change, ControlTransaction, Parameter}; + +pub(crate) async fn append_evaluation( + transaction: ControlTransaction, + request: &SensorEvaluationRequest, + outcome: SensorEvaluationOutcome, +) -> Result { + let phase = outcome.phase(); + let event_id = Uuid::new_v5(&request.evaluation_id, phase.as_bytes()); + let occurred_at = transaction.now; + let fact = SensorEvaluationFact { + request: request.clone(), + outcome, + }; + let (mut transaction, _) = query(transaction, + "INSERT INTO sqb_sensor_evaluation_events (event_id,evaluation_id,trigger_id,phase,occurred_at,fact_json) \ + VALUES ($1,$2,$3,$4,$5,$6)", + vec![Parameter::Id(event_id), Parameter::Id(request.evaluation_id), Parameter::Id(request.trigger_id), + Parameter::Text(phase.into()), Parameter::Time(occurred_at), Parameter::Json(json!(fact))]).await?; + transaction + .changes + .insert(Change::Trigger(request.trigger_id)); + Ok(transaction) +} + +pub(crate) async fn commit_cursor( + transaction: ControlTransaction, + trigger_id: Uuid, + cursor: &SensorCursor, +) -> Result { + let now = transaction.now; + let (transaction, _) = query( + transaction, + "UPDATE sqb_trigger_state SET cursor_json=$1,updated_at=$2 WHERE trigger_id=$3", + vec![ + Parameter::Json(json!(cursor)), + Parameter::Time(now), + Parameter::Id(trigger_id), + ], + ) + .await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs new file mode 100644 index 0000000000..08edfd3e68 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod attempts; +pub(crate) mod completion; +pub(crate) mod conditions; +pub(crate) mod custom; +pub(crate) mod facts; +pub(crate) mod nodes; +pub(crate) mod platform; +pub(crate) mod requests; +pub(crate) mod runs; +pub(crate) mod transitions; diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs new file mode 100644 index 0000000000..32e4853032 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs @@ -0,0 +1,272 @@ +//! Node conditions consume executed run-node outcomes or the latest warehouse source observations. + +use std::collections::BTreeSet; +use std::num::NonZeroU32; + +use chrono::{DateTime, NaiveDateTime, TimeDelta, Utc}; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::sensors::_helpers::requests::queue_sensor_run; +use crate::sensors::_helpers::transitions::matches_transition; +use crate::sensors::constants::TIMESTAMP_VALUE_KIND; +use crate::sensors::models::{ + NodeEventKind, NodeSensorEvent, SensorCursor, SensorEvaluationRequest, +}; +use crate::sensors::types::SourcePage; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; +use crate::warehouse_metadata::models::WarehouseFreshnessRecord; + +pub(crate) async fn consume_node_events( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + selection: &BTreeSet, + kinds: &BTreeSet, +) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { + let (mut transaction, rows) = query( + transaction, + "SELECT n.run_id,n.node_key,n.status,n.finished_storage_id FROM sqb_run_nodes n \ + JOIN sqb_runs r ON r.run_id=n.run_id \ + WHERE n.finished_storage_id>$1 AND n.status IN ('success','failed') \ + AND n.started_at IS NOT NULL AND r.location_id=$2 AND r.target=$3 AND r.kind<>'group' \ + ORDER BY n.finished_storage_id LIMIT 100", + vec![ + Parameter::Integer(evaluation.cursor.event_storage_id), + Parameter::Id(evaluation.location_id), + Parameter::Text(evaluation.target.clone()), + ], + ) + .await?; + let mut cursor = evaluation.cursor.clone(); + let mut run_ids: Vec = Vec::new(); + for row in rows { + let node: NodeSensorEvent = decode_row(row)?; + cursor.event_storage_id = node.finished_storage_id; + if !selection.contains(&node.node_key) { + continue; + } + let matches; + (transaction, matches) = matches_transition(transaction, evaluation, &node, kinds).await?; + if matches { + let run_id; + (transaction, run_id) = queue_sensor_run( + transaction, + evaluation, + &format!("{}/{}", node.run_id, node.node_key), + ) + .await?; + run_ids.push(run_id); + } + } + Ok((transaction, (cursor, run_ids))) +} + +/// A source's newest version queues one run per streak, keyed on the streak's first row so pruning never reuses a key. +pub(crate) async fn consume_source_events( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + selection: &BTreeSet, +) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { + let (mut transaction, (nodes, next)) = source_page(transaction, evaluation, selection).await?; + let mut run_ids: Vec = Vec::new(); + for (key, definition) in nodes { + let (parameters, predicate) = source_scope(evaluation, &key, &definition); + let rows; + (transaction, rows) = query(transaction, &format!( + "SELECT latest.record_json, (SELECT streak.event_id FROM sqb_warehouse_freshness streak \ + WHERE streak.location_id=$1 AND streak.target=$2 AND streak.record_json ->> 'source_name'=$3{streak} \ + AND streak.record_json ->> 'data_version' IS NOT NULL \ + AND streak.record_json ->> 'data_version_hash'=latest.record_json ->> 'data_version_hash' \ + AND NOT EXISTS (SELECT 1 FROM sqb_warehouse_freshness other \ + WHERE other.location_id=$1 AND other.target=$2 AND other.record_json ->> 'source_name'=$3{other} \ + AND other.record_json ->> 'data_version' IS NOT NULL \ + AND other.record_json ->> 'data_version_hash'<>latest.record_json ->> 'data_version_hash' \ + AND (other.received_at,other.observed_at,other.event_id)>(streak.received_at,streak.observed_at,streak.event_id)) \ + ORDER BY streak.received_at,streak.observed_at,streak.event_id LIMIT 1) AS streak_start \ + FROM sqb_warehouse_freshness latest WHERE latest.location_id=$1 AND latest.target=$2 \ + AND latest.record_json ->> 'source_name'=$3{latest} AND latest.record_json ->> 'data_version' IS NOT NULL \ + ORDER BY latest.received_at DESC,latest.observed_at DESC,latest.event_id DESC LIMIT 1", + streak = predicate.replace("record_json", "streak.record_json"), + other = predicate.replace("record_json", "other.record_json"), + latest = predicate.replace("record_json", "latest.record_json"), + ), parameters).await?; + let Some(row) = rows.first() else { continue }; + let record: WarehouseFreshnessRecord = + serde_json::from_value(row["record_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let identity = serde_json::to_vec(&( + &key, + &record.target_database, + &record.target_schema, + &record.target_name, + &record.data_version_hash, + &row["streak_start"], + )) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let run_key = sha256_hex(&identity); + let requested; + (transaction, requested) = query( + transaction, + "SELECT 1 AS requested FROM sqb_run_requests WHERE trigger_id=$1 AND run_key=$2", + vec![ + Parameter::Id(evaluation.trigger_id), + Parameter::Text(run_key.clone()), + ], + ) + .await?; + if !requested.is_empty() { + continue; + } + let run_id; + (transaction, run_id) = queue_sensor_run(transaction, evaluation, &run_key).await?; + run_ids.push(run_id); + } + let mut cursor = evaluation.cursor.clone(); + cursor.custom = next; + Ok((transaction, (cursor, run_ids))) +} + +/// One page of selected source definitions after the cursor's last key, and the next page's start. +async fn source_page( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + selection: &BTreeSet, +) -> Result { + let mut remaining = selection.iter().filter(|key| { + evaluation + .cursor + .custom + .as_ref() + .is_none_or(|after| *key > after) + }); + let page: BTreeSet = remaining.by_ref().take(100).cloned().collect(); + let next = remaining + .next() + .is_some() + .then(|| page.last().cloned()) + .flatten(); + let (transaction, nodes) = + read_snapshot_nodes(transaction, evaluation.snapshot_id, Some(&page)).await?; + Ok(( + transaction, + ( + nodes + .into_iter() + .map(|(key, node)| (key, node.definition)) + .collect(), + next, + ), + )) +} + +/// Parameters $1-$3 and a predicate matching the declared source's physical relation. +fn source_scope( + evaluation: &SensorEvaluationRequest, + key: &str, + definition: &serde_json::Value, +) -> (Vec, String) { + let mut parameters = vec![ + Parameter::Id(evaluation.location_id), + Parameter::Text(evaluation.target.clone()), + Parameter::Text(key.replacen("source:", "", 1)), + ]; + let mut predicate = String::new(); + for (column, value) in [ + ( + "target_database", + definition["namespace"] + .get(0) + .unwrap_or(&definition["database"]), + ), + ( + "target_schema", + definition["namespace"] + .get(1) + .unwrap_or(&definition["schema"]), + ), + ("target_name", &definition["table"]), + ] { + if let Some(value) = value.as_str() { + parameters.push(Parameter::Text(value.into())); + predicate.push_str(&format!( + " AND record_json ->> '{column}'=${}", + parameters.len() + )); + } else { + predicate.push_str(&format!(" AND record_json ->> '{column}' IS NULL")); + } + } + (parameters, predicate) +} + +fn stale_version( + record: &WarehouseFreshnessRecord, + now: DateTime, + after_seconds: NonZeroU32, +) -> Result { + let Some(version) = record + .data_version + .as_deref() + .filter(|_| record.value_kind == TIMESTAMP_VALUE_KIND) + else { + return Ok(false); + }; + let instant = DateTime::parse_from_rfc3339(version) + .map(|value| value.with_timezone(&Utc)) + .or_else(|_| { + NaiveDateTime::parse_from_str(version, "%Y-%m-%dT%H:%M:%S%.f") + .map(|value| value.and_utc()) + }) + .map_err(|_| { + ControlError::Invalid( + "timestamp source freshness version is not an ISO datetime".into(), + ) + })?; + Ok(now.signed_duration_since(instant) > TimeDelta::seconds(i64::from(after_seconds.get()))) +} + +/// Reevaluate current timestamp age; only page position and idempotent requests are stored. +pub(crate) async fn consume_stale_sources( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + selection: &BTreeSet, + after_seconds: NonZeroU32, +) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { + let (mut transaction, (nodes, next)) = source_page(transaction, evaluation, selection).await?; + let mut run_ids: Vec = Vec::new(); + for (key, definition) in nodes { + let (parameters, predicate) = source_scope(evaluation, &key, &definition); + let rows; + (transaction, rows) = query(transaction, &format!( + "SELECT record_json FROM sqb_warehouse_freshness WHERE location_id=$1 AND target=$2 \ + AND record_json ->> 'source_name'=$3{predicate} ORDER BY observed_at DESC,event_id DESC LIMIT 1" + ), parameters).await?; + let Some(row) = rows.first() else { continue }; + let record: WarehouseFreshnessRecord = + serde_json::from_value(row["record_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if !stale_version(&record, evaluation.requested_at, after_seconds)? { + continue; + } + let identity = serde_json::to_vec(&( + &key, + &record.target_database, + &record.target_schema, + &record.target_name, + &record.data_version_hash, + after_seconds, + )) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let run_id; + (transaction, run_id) = + queue_sensor_run(transaction, evaluation, &sha256_hex(&identity)).await?; + run_ids.push(run_id); + } + let mut cursor = evaluation.cursor.clone(); + cursor.custom = next; + Ok((transaction, (cursor, run_ids))) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs new file mode 100644 index 0000000000..6ce610bf03 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs @@ -0,0 +1,36 @@ +//! One bounded platform page yields matched run requests and a storage-order cursor. + +use uuid::Uuid; + +use crate::platform::main::read_platform_events::read_platform_events; +use crate::platform::models::PlatformEventKind; +use crate::sensors::_helpers::requests::queue_sensor_run; +use crate::sensors::models::{SensorCursor, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn consume_platform_events( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + kinds: &std::collections::BTreeSet, +) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { + let (mut transaction, events) = + read_platform_events(transaction, evaluation.cursor.event_storage_id).await?; + let cursor = SensorCursor { + event_storage_id: events + .last() + .map_or(evaluation.cursor.event_storage_id, |event| event.storage_id), + ..evaluation.cursor.clone() + }; + let mut run_ids: Vec = Vec::new(); + for event in events + .iter() + .filter(|event| kinds.contains(&event.payload.platform.kind())) + { + let run_id; + (transaction, run_id) = + queue_sensor_run(transaction, evaluation, &event.event_id.to_string()).await?; + run_ids.push(run_id); + } + Ok((transaction, (cursor, run_ids))) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs new file mode 100644 index 0000000000..237959031b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs @@ -0,0 +1,62 @@ +//! A sensor run key gates queue publication within the cursor/outcome transaction. + +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::runs::main::frozen_job_request::frozen_job_request; +use crate::runs::main::queue_run::queue_run; +use crate::runs::models::{JobRunRequest, RunOrigin}; +use crate::sensors::models::SensorEvaluationRequest; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn queue_sensor_run( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + run_key: &str, +) -> Result<(ControlTransaction, Uuid), ControlError> { + let (transaction, rows) = query( + transaction, + "SELECT run_id FROM sqb_run_requests WHERE trigger_id=$1 AND run_key=$2", + vec![ + Parameter::Id(evaluation.trigger_id), + Parameter::Text(run_key.into()), + ], + ) + .await?; + if let Some(row) = rows.into_iter().next() { + return Ok(( + transaction, + serde_json::from_value(row["run_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + )); + } + let run_id = Uuid::new_v5(&evaluation.trigger_id, run_key.as_bytes()); + let context = AuditContext { + actor: "system:sensor".into(), + granting_role: "system:scheduler".into(), + request_id: evaluation.evaluation_id, + ip_address: None, + }; + let authored = JobRunRequest { + run_id, + location_id: evaluation.location_id, + target: evaluation.target.clone(), + job_key: evaluation.definition.job_key.clone(), + reason: "Sensor condition matched".into(), + priority: 0, + window: None, + }; + let (transaction, mut frozen) = frozen_job_request(transaction, &context, &authored).await?; + if frozen.snapshot_id != evaluation.snapshot_id { + return Err(ControlError::Fenced); + } + frozen.origin = RunOrigin::Sensor; + frozen.trigger_id = Some(evaluation.trigger_id); + let transaction = queue_run(transaction, &frozen).await?; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_run_requests (trigger_id,run_key,evaluation_id,run_id) VALUES ($1,$2,$3,$4)", + vec![Parameter::Id(evaluation.trigger_id), Parameter::Text(run_key.into()), Parameter::Id(evaluation.evaluation_id), Parameter::Id(run_id)]).await?; + Ok((transaction, run_id)) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs new file mode 100644 index 0000000000..d164157ad3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs @@ -0,0 +1,95 @@ +//! Terminal projections carry the durable finish cursor; late observations cannot move it. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::sensors::_helpers::requests::queue_sensor_run; +use crate::sensors::constants::EXECUTION_NODE_BATCH_SIZE; +use crate::sensors::models::{ + RunEventKind, SensorCursor, SensorEvaluationRequest, TerminalSensorRun, +}; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn consume_run_events( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + (job_key, kinds): (&str, &BTreeSet), + executed_successfully: Option<&BTreeSet>, +) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { + let (mut transaction, rows) = query( + transaction, + "SELECT run_id,status,last_event_storage_id FROM sqb_runs \ + WHERE location_id=$1 AND target=$2 AND job_key=$3 AND last_event_storage_id>$4 \ + AND status IN ('success','warn','partial','failed','cancelled') \ + ORDER BY last_event_storage_id LIMIT 100", + vec![ + Parameter::Id(evaluation.location_id), + Parameter::Text(evaluation.target.clone()), + Parameter::Text(job_key.into()), + Parameter::Integer(evaluation.cursor.event_storage_id), + ], + ) + .await?; + let runs: Vec = + rows.into_iter().map(decode_row).collect::>()?; + let mut eligible: BTreeSet = runs.iter().map(|run| run.run_id).collect(); + if let Some(nodes) = executed_successfully { + (transaction, eligible) = matching_executions(transaction, eligible, nodes).await?; + } + let mut cursor = evaluation.cursor.clone(); + let mut run_ids: Vec = Vec::new(); + for run in runs { + cursor.event_storage_id = run.last_event_storage_id; + if eligible.contains(&run.run_id) && kinds.iter().any(|kind| kind.matches(run.status)) { + let run_id; + (transaction, run_id) = + queue_sensor_run(transaction, evaluation, &run.run_id.to_string()).await?; + run_ids.push(run_id); + } + } + Ok((transaction, (cursor, run_ids))) +} + +/// Each chunk intersects the previous matches: every selected node must qualify. +async fn matching_executions( + mut transaction: ControlTransaction, + mut eligible: BTreeSet, + nodes: &BTreeSet, +) -> Result<(ControlTransaction, BTreeSet), ControlError> { + let nodes: Vec<&String> = nodes.iter().collect(); + for chunk in nodes.chunks(EXECUTION_NODE_BATCH_SIZE) { + if eligible.is_empty() { + break; + } + let mut parameters: Vec = eligible.iter().copied().map(Parameter::Id).collect(); + let run_parameters = (1..=parameters.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + let node_parameters = (parameters.len() + 1..=parameters.len() + chunk.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + parameters.extend(chunk.iter().map(|key| Parameter::Text((*key).to_string()))); + let rows; + (transaction, rows) = query(transaction, &format!( + "SELECT r.run_id,r.status,r.last_event_storage_id FROM sqb_runs r \ + WHERE r.run_id IN ({run_parameters}) AND \ + (SELECT COUNT(*) FROM sqb_run_nodes n WHERE n.run_id=r.run_id AND n.kind='node' \ + AND n.node_key IN ({node_parameters}) AND n.status='success' AND n.ended_at IS NOT NULL \ + AND ((r.kind<>'group' AND n.started_at IS NOT NULL) OR \ + (r.kind='group' AND EXISTS(SELECT 1 FROM sqb_runs c \ + JOIN sqb_run_nodes cn ON cn.run_id=c.run_id AND cn.node_key=n.node_key AND cn.kind='node' \ + WHERE c.parent_run_id=r.run_id AND c.location_id=r.location_id AND c.target=r.target \ + AND cn.status='success' AND cn.started_at IS NOT NULL AND cn.ended_at IS NOT NULL))))={}", + chunk.len()), parameters).await?; + let matches: Vec = + rows.into_iter().map(decode_row).collect::>()?; + eligible = matches.into_iter().map(|run| run.run_id).collect(); + } + Ok((transaction, eligible)) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/transitions.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/transitions.rs new file mode 100644 index 0000000000..8a33de2970 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/transitions.rs @@ -0,0 +1,47 @@ +//! Failure health is derived from decisive executed facts, never a mutable health flag. + +use std::collections::BTreeSet; + +use crate::runs::models::NodeOutcome; +use crate::sensors::models::{NodeEventKind, NodeSensorEvent, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn matches_transition( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + node: &NodeSensorEvent, + kinds: &BTreeSet, +) -> Result<(ControlTransaction, bool), ControlError> { + let outcome = node.status; + if outcome == NodeOutcome::Success && kinds.contains(&NodeEventKind::Success) { + return Ok((transaction, true)); + } + let kind = match outcome { + NodeOutcome::Success => NodeEventKind::Recovered, + NodeOutcome::Failed => NodeEventKind::Failed, + _ => return Ok((transaction, false)), + }; + if !kinds.contains(&kind) { + return Ok((transaction, false)); + } + let (transaction, rows) = query( + transaction, + "SELECT n.status FROM sqb_run_nodes n JOIN sqb_runs r ON r.run_id=n.run_id \ + WHERE n.node_key=$1 AND n.finished_storage_id<$2 AND n.status IN ('success','failed') \ + AND n.started_at IS NOT NULL AND r.location_id=$3 AND r.target=$4 AND r.kind<>'group' \ + ORDER BY n.finished_storage_id DESC LIMIT 1", + vec![ + Parameter::Text(node.node_key.clone()), + Parameter::Integer(node.finished_storage_id), + Parameter::Id(evaluation.location_id), + Parameter::Text(evaluation.target.clone()), + ], + ) + .await?; + let failed = rows + .first() + .is_some_and(|row| row["status"] == NodeOutcome::Failed.as_str()); + Ok((transaction, (outcome == NodeOutcome::Success) == failed)) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/constants.rs b/crates/sqlbuild-orchestration/src/sensors/constants.rs new file mode 100644 index 0000000000..06201978b9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/constants.rs @@ -0,0 +1,7 @@ +//! Bound query parameters independently of the size of an authored selection. + +pub(crate) const EXECUTION_NODE_BATCH_SIZE: usize = 500; +pub(crate) const CUSTOM_SENSOR_RECHECK_SECONDS: i64 = 1; +pub(crate) const SOURCE_NODE_KIND: &str = "source"; +pub(crate) const TIMESTAMP_VALUE_KIND: &str = "timestamp"; +pub const MAX_SENSOR_RUN_REQUESTS: usize = 100; diff --git a/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs b/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs new file mode 100644 index 0000000000..30c6503463 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs @@ -0,0 +1,82 @@ +//! Commit the requested phase before evaluation, so crashes count as attempts. + +use chrono::TimeDelta; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::declarations::models::TriggerStatus; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::sensors::_helpers::attempts::{defer_sensor, last_attempt, read_sensor}; +use crate::sensors::_helpers::custom::request_evaluation_work; +use crate::sensors::_helpers::facts::append_evaluation; +use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; +use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn begin_sensor_evaluation( + store: &ControlStore, + lease: &SchedulerLease, + trigger_id: Uuid, +) -> Result, ControlError> { + let transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let (transaction, sensor) = read_sensor(transaction, trigger_id).await?; + let Some(sensor) = sensor else { + return Ok(None); + }; + if sensor.status != TriggerStatus::Running { + return Ok(None); + } + let now = transaction.now; + let (transaction, cursor) = read_sensor_cursor(transaction, trigger_id).await?; + let (mut transaction, previous) = last_attempt(transaction, trigger_id).await?; + if let Some((previous, terminal)) = previous { + if !terminal { + if previous.request.epoch == lease.epoch + && previous.request.snapshot_id == sensor.snapshot_id + && previous.request.cursor == cursor + { + commit(transaction).await?; + return Ok(Some(previous.request)); + } + transaction = append_evaluation( + transaction, + &previous.request, + SensorEvaluationOutcome::Abandoned, + ) + .await?; + } + let next = previous.request.requested_at + + TimeDelta::seconds(i64::from( + sensor.definition_json.minimum_interval_seconds.get(), + )); + if next > now { + commit(defer_sensor(transaction, trigger_id, next).await?).await?; + return Ok(None); + } + } + let definition_fingerprint = sha256_hex( + &serde_json::to_vec(&sensor.definition_json) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + let request = SensorEvaluationRequest { + evaluation_id: Uuid::now_v7(), + trigger_id: sensor.trigger_id, + location_id: sensor.location_id, + target: sensor.target, + snapshot_id: sensor.snapshot_id, + definition: sensor.definition_json, + definition_fingerprint, + cursor, + epoch: lease.epoch, + requested_at: now, + }; + let transaction = + append_evaluation(transaction, &request, SensorEvaluationOutcome::Requested).await?; + let transaction = request_evaluation_work(transaction, &request).await?; + commit(check_lease(transaction, lease).await?).await?; + Ok(Some(request)) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs new file mode 100644 index 0000000000..5fbd35834d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs @@ -0,0 +1,28 @@ +//! Cursor, outcome, deduplicated requests and queued runs commit together. + +use crate::scheduler::models::SchedulerLease; +use crate::sensors::_helpers::completion::{finish_evaluation, pending_evaluation}; +use crate::sensors::_helpers::conditions::consume_condition; +use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +pub async fn complete_builtin_sensor( + store: &ControlStore, + lease: &SchedulerLease, + evaluation: &SensorEvaluationRequest, +) -> Result { + let Some(transaction) = pending_evaluation(store, lease, evaluation).await? else { + return Ok(0); + }; + let (transaction, (cursor, run_ids)) = consume_condition(transaction, evaluation).await?; + let count = run_ids.len(); + finish_evaluation( + transaction, + lease, + evaluation, + SensorEvaluationOutcome::Completed { cursor, run_ids }, + ) + .await?; + Ok(count) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/complete_custom_sensor.rs b/crates/sqlbuild-orchestration/src/sensors/main/complete_custom_sensor.rs new file mode 100644 index 0000000000..98d23dd900 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/complete_custom_sensor.rs @@ -0,0 +1,42 @@ +//! A location result cannot queue runs or advance a cursor until a fenced scheduler commits it. + +use chrono::TimeDelta; + +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::sensors::_helpers::attempts::defer_sensor; +use crate::sensors::_helpers::completion::{finish_evaluation, pending_evaluation}; +use crate::sensors::_helpers::custom::read_evaluation_result; +use crate::sensors::constants::CUSTOM_SENSOR_RECHECK_SECONDS; +use crate::sensors::models::{SensorCondition, SensorEvaluationOutcome, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::main::commit::commit; +use crate::store::models::ControlStore; + +pub async fn complete_custom_sensor( + store: &ControlStore, + lease: &SchedulerLease, + evaluation: &SensorEvaluationRequest, +) -> Result { + if !matches!(evaluation.definition.when, SensorCondition::Custom { .. }) { + return Err(ControlError::Invalid( + "custom sensor evaluation required".into(), + )); + } + let Some(transaction) = pending_evaluation(store, lease, evaluation).await? else { + return Ok(0); + }; + let (transaction, outcome) = read_evaluation_result(transaction, evaluation).await?; + let Some(outcome) = outcome else { + let next = transaction.now + TimeDelta::seconds(CUSTOM_SENSOR_RECHECK_SECONDS); + let transaction = defer_sensor(transaction, evaluation.trigger_id, next).await?; + commit(check_lease(transaction, lease).await?).await?; + return Ok(0); + }; + let count = match &outcome { + SensorEvaluationOutcome::Completed { run_ids, .. } => run_ids.len(), + _ => 0, + }; + finish_evaluation(transaction, lease, evaluation, outcome).await?; + Ok(count) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/mod.rs b/crates/sqlbuild-orchestration/src/sensors/main/mod.rs new file mode 100644 index 0000000000..e819eb77e1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/mod.rs @@ -0,0 +1,7 @@ +pub mod begin_sensor_evaluation; +pub mod complete_builtin_sensor; +pub mod complete_custom_sensor; +pub(crate) mod publish_sensors; +pub(crate) mod read_sensor_cursor; +pub(crate) mod reset_sensor_cursor; +pub(crate) mod validate_sensors; diff --git a/crates/sqlbuild-orchestration/src/sensors/main/publish_sensors.rs b/crates/sqlbuild-orchestration/src/sensors/main/publish_sensors.rs new file mode 100644 index 0000000000..d4e4d13f96 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/publish_sensors.rs @@ -0,0 +1,37 @@ +//! Sensor publication keeps operator overrides and immutable cursor history across deployments. + +use serde_json::json; +use uuid::Uuid; + +use crate::declarations::models::Definitions; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn publish_sensors( + mut transaction: ControlTransaction, + snapshot: &SnapshotRecord, + definitions: &Definitions, +) -> Result { + for (name, sensor) in &definitions.sensors { + let trigger_id = Uuid::new_v5( + &snapshot.location_id, + &serde_json::to_vec(&(&snapshot.target, "sensor", name)) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + (transaction, _) = query(transaction, + "INSERT INTO sqb_triggers (trigger_id,location_id,target,kind,name,job_key,definition_json,snapshot_id) \ + VALUES ($1,$2,$3,'sensor',$4,$5,$6,$7) ON CONFLICT (trigger_id) DO UPDATE \ + SET job_key=excluded.job_key,definition_json=excluded.definition_json,snapshot_id=excluded.snapshot_id,removed_in_snapshot_id=NULL", + vec![Parameter::Id(trigger_id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), + Parameter::Text(name.clone()), Parameter::Text(sensor.job_key.clone()), Parameter::Json(json!(sensor)), Parameter::Id(snapshot.snapshot_id)]).await?; + let status = sensor.default_status; + (transaction, _) = query(transaction, + "INSERT INTO sqb_trigger_state (trigger_id,status,started_at,next_due_at,updated_at) VALUES ($1,$2,$3,$3,$3) \ + ON CONFLICT (trigger_id) DO UPDATE SET status=COALESCE(sqb_trigger_state.status_override,excluded.status), \ + next_due_at=excluded.next_due_at,updated_at=excluded.updated_at", + vec![Parameter::Id(trigger_id), Parameter::Text(status.as_str().into()), Parameter::Time(snapshot.published_at)]).await?; + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/read_sensor_cursor.rs b/crates/sqlbuild-orchestration/src/sensors/main/read_sensor_cursor.rs new file mode 100644 index 0000000000..fd18c1a5b6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/read_sensor_cursor.rs @@ -0,0 +1,27 @@ +//! Operator reads and evaluations share the cursor on the trigger state. + +use uuid::Uuid; + +use crate::sensors::models::SensorCursor; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_sensor_cursor( + transaction: ControlTransaction, + trigger_id: Uuid, +) -> Result<(ControlTransaction, SensorCursor), ControlError> { + let (transaction, rows) = query( + transaction, + "SELECT cursor_json FROM sqb_trigger_state WHERE trigger_id=$1 AND cursor_json IS NOT NULL", + vec![Parameter::Id(trigger_id)], + ) + .await?; + let cursor = rows + .first() + .map(|row| serde_json::from_value(row["cursor_json"].clone())) + .transpose() + .map_err(|error| ControlError::Invalid(error.to_string()))? + .unwrap_or_default(); + Ok((transaction, cursor)) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/reset_sensor_cursor.rs b/crates/sqlbuild-orchestration/src/sensors/main/reset_sensor_cursor.rs new file mode 100644 index 0000000000..896384084c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/reset_sensor_cursor.rs @@ -0,0 +1,15 @@ +//! Operator reset returns the sensor to its initial cursor; the trigger change audit records it. + +use uuid::Uuid; + +use crate::sensors::_helpers::facts::commit_cursor; +use crate::sensors::models::SensorCursor; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn reset_sensor_cursor( + transaction: ControlTransaction, + trigger_id: Uuid, +) -> Result { + commit_cursor(transaction, trigger_id, &SensorCursor::default()).await +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs new file mode 100644 index 0000000000..7ba89f5f56 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs @@ -0,0 +1,64 @@ +//! Native publication validates every sensor before any snapshot pointer moves. + +use crate::declarations::models::Definitions; +use crate::sensors::constants::SOURCE_NODE_KIND; +use crate::sensors::models::SensorCondition; +use crate::snapshots::models::SnapshotInput; +use crate::store::errors::ControlError; + +pub(crate) fn validate_sensors( + definitions: &Definitions, + input: &SnapshotInput, +) -> Result<(), ControlError> { + for (name, sensor) in &definitions.sensors { + let valid_condition = match &sensor.when { + SensorCondition::Custom { + function, + code_hash, + } => !function.trim().is_empty() && !code_hash.is_empty(), + SensorCondition::PlatformEvent { kinds } => !kinds.is_empty(), + SensorCondition::NodeEvent { selection, kinds } => { + !kinds.is_empty() + && !selection.is_empty() + && selection.iter().all(|key| input.nodes.contains_key(key)) + } + SensorCondition::SourceNewData { selection } + | SensorCondition::SourceStale { selection, .. } => { + !selection.is_empty() + && selection.iter().all(|key| { + input + .nodes + .get(key) + .is_some_and(|node| node.kind == SOURCE_NODE_KIND) + }) + } + SensorCondition::RunEvent { + job_key, + kinds, + executed_successfully, + } => { + !kinds.is_empty() + && definitions + .jobs + .get(job_key) + .or_else(|| definitions.groups.get(job_key).map(|group| &group.job)) + .is_some_and(|source| { + executed_successfully.as_ref().is_none_or(|nodes| { + !nodes.is_empty() && nodes.is_subset(&source.selection) + }) + }) + } + }; + if name.trim().is_empty() + || sensor.description.trim().is_empty() + || !valid_condition + || (!definitions.jobs.contains_key(&sensor.job_key) + && !definitions.groups.contains_key(&sensor.job_key)) + { + return Err(ControlError::Invalid(format!( + "sensor {name} needs a description, a known job and nonempty event kinds; executed_successfully must select nonempty source job/group nodes; node_event must select nonempty snapshot nodes; source_new_data and source_stale must select nonempty snapshot sources" + ))); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/mod.rs b/crates/sqlbuild-orchestration/src/sensors/mod.rs new file mode 100644 index 0000000000..0b504ff3a3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/mod.rs @@ -0,0 +1,11 @@ +//! Sensor attempts are append-only facts; the latest cursor lives on the trigger's state. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub(crate) mod types; + +#[cfg(test)] +#[path = "../../tests/sensors/test_contracts.rs"] +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/sensors/models.rs b/crates/sqlbuild-orchestration/src/sensors/models.rs new file mode 100644 index 0000000000..e0cfc25b6c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/models.rs @@ -0,0 +1,192 @@ +//! Published conditions and immutable attempts share one sensor abstraction. + +use std::collections::BTreeSet; +use std::num::NonZeroU32; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::declarations::models::TriggerStatus; +use crate::platform::models::PlatformEventKind; +use crate::runs::models::{NodeOutcome, RunOutcome}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum SensorCondition { + PlatformEvent { + kinds: BTreeSet, + }, + RunEvent { + job_key: String, + kinds: BTreeSet, + #[serde(default, skip_serializing_if = "Option::is_none")] + executed_successfully: Option>, + }, + NodeEvent { + selection: BTreeSet, + kinds: BTreeSet, + }, + SourceNewData { + selection: BTreeSet, + }, + SourceStale { + selection: BTreeSet, + #[schema(value_type = u32, minimum = 1)] + after_seconds: NonZeroU32, + }, + Custom { + function: String, + code_hash: String, + }, +} + +#[derive( + Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum NodeEventKind { + Success, + Failed, + Recovered, +} + +#[derive( + Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum RunEventKind { + Completed, + Success, + Warn, + Partial, + Failed, + Cancelled, +} + +impl RunEventKind { + #[must_use] + pub const fn matches(self, outcome: RunOutcome) -> bool { + matches!( + (self, outcome), + (Self::Completed, _) + | (Self::Success, RunOutcome::Success) + | (Self::Warn, RunOutcome::Warn) + | (Self::Partial, RunOutcome::Partial) + | (Self::Failed, RunOutcome::Failed) + | (Self::Cancelled, RunOutcome::Cancelled) + ) + } +} + +#[derive(Debug, Deserialize)] +pub(crate) struct TerminalSensorRun { + pub(crate) run_id: Uuid, + pub(crate) status: RunOutcome, + pub(crate) last_event_storage_id: i64, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct NodeSensorEvent { + pub(crate) run_id: Uuid, + pub(crate) node_key: String, + pub(crate) status: NodeOutcome, + pub(crate) finished_storage_id: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct SensorDefinition { + pub description: String, + pub job_key: String, + pub when: SensorCondition, + #[schema(value_type = u32, minimum = 1)] + pub minimum_interval_seconds: NonZeroU32, + #[serde(default)] + pub default_status: TriggerStatus, +} + +/// Storage order is authoritative even when a fact's occurrence time predates earlier pages. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SensorCursor { + pub event_storage_id: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub custom: Option, +} + +/// Location-owned Python returns data; only the fenced scheduler may apply it. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)] +pub enum CustomSensorOutcome { + Completed { + cursor: Option, + run_keys: Vec, + }, + Skipped { + cursor: Option, + reason: String, + }, + Failed, + TimedOut, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SensorEvaluationRequest { + pub evaluation_id: Uuid, + pub trigger_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub snapshot_id: Uuid, + pub definition: SensorDefinition, + pub definition_fingerprint: String, + pub cursor: SensorCursor, + pub epoch: i64, + pub requested_at: DateTime, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "phase", rename_all = "snake_case", deny_unknown_fields)] +pub enum SensorEvaluationOutcome { + Requested, + Completed { + cursor: SensorCursor, + run_ids: Vec, + }, + Failed { + code: String, + }, + TimedOut, + Abandoned, +} + +impl SensorEvaluationOutcome { + #[must_use] + pub const fn phase(&self) -> &'static str { + match self { + Self::Requested => "requested", + Self::Completed { .. } => "completed", + Self::Failed { .. } => "failed", + Self::TimedOut => "timed_out", + Self::Abandoned => "abandoned", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SensorEvaluationFact { + pub request: SensorEvaluationRequest, + pub outcome: SensorEvaluationOutcome, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct DueSensor { + pub(crate) trigger_id: Uuid, + pub(crate) location_id: Uuid, + pub(crate) target: String, + pub(crate) snapshot_id: Uuid, + pub(crate) definition_json: SensorDefinition, + pub(crate) status: TriggerStatus, +} diff --git a/crates/sqlbuild-orchestration/src/sensors/types.rs b/crates/sqlbuild-orchestration/src/sensors/types.rs new file mode 100644 index 0000000000..e721dd5748 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/types.rs @@ -0,0 +1,9 @@ +//! Compound sensor return values. + +use crate::store::models::ControlTransaction; + +/// Selected source definitions by node key, and the key the next page starts after. +pub(crate) type SourcePage = ( + ControlTransaction, + (Vec<(String, serde_json::Value)>, Option), +); diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs new file mode 100644 index 0000000000..e744c7dc8f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs @@ -0,0 +1,67 @@ +//! Canonical content addresses and compressed immutable bodies. + +use std::collections::BTreeMap; + +use serde_json::Value; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::snapshots::models::{NodeContent, PreparedNode, SnapshotNode}; +use crate::store::errors::ControlError; +use crate::store::main::compress::compress; + +fn canonical(value: &Value) -> Value { + match value { + Value::Object(values) => { + let sorted: BTreeMap<_, _> = values + .iter() + .map(|(key, value)| (key.clone(), canonical(value))) + .collect(); + Value::Object(sorted.into_iter().collect()) + } + Value::Array(values) => Value::Array(values.iter().map(canonical).collect()), + _ => value.clone(), + } +} + +pub(crate) fn json_bytes(value: &Value) -> Result, ControlError> { + serde_json::to_vec(&canonical(value)).map_err(|error| ControlError::Invalid(error.to_string())) +} + +pub(crate) fn node_content(node: &SnapshotNode) -> Result, ControlError> { + let mut sources: BTreeMap<&'static str, (String, &str)> = BTreeMap::new(); + for (kind, text) in [ + ("raw_sql", &node.raw_sql), + ("compiled_sql", &node.compiled_sql), + ("python_source", &node.python_source), + ] { + if let Some(text) = text { + sources.insert(kind, (sha256_hex(text.as_bytes()), text)); + } + } + let source_hashes: BTreeMap<_, _> = sources + .iter() + .map(|(kind, (hash, _))| (*kind, hash)) + .collect(); + let definition = json_bytes( + &serde_json::json!({"kind": node.kind, "definition": node.definition, "sources": source_hashes}), + )?; + Ok(NodeContent { + definition, + sources, + }) +} + +pub(crate) fn prepare_node(node: &SnapshotNode) -> Result { + let content = node_content(node)?; + let sources = content + .sources + .into_iter() + .map(|(kind, (hash, text))| Ok((kind, (hash, compress(text.as_bytes())?)))) + .collect::>()?; + Ok(PreparedNode { + kind: node.kind.clone(), + hash: sha256_hex(&content.definition), + definition: compress(&content.definition)?, + sources, + }) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs new file mode 100644 index 0000000000..8853fb96b6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod content; +pub(crate) mod preparation; +pub(crate) mod publication; +pub(crate) mod reading; diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs new file mode 100644 index 0000000000..a697f16dd4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs @@ -0,0 +1,75 @@ +//! Validate and content-address a successful compile before holding a database write lock. + +use std::collections::BTreeMap; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::snapshots::_helpers::content::{json_bytes, prepare_node}; +use crate::snapshots::models::{PreparedNode, PreparedSnapshot, SnapshotInput}; +use crate::store::errors::ControlError; +use crate::store::main::compress::compress; + +pub(crate) fn prepare_snapshot(input: &SnapshotInput) -> Result { + if input.target.trim().is_empty() + || input.environment_hash.is_empty() + || input.generation <= 0 + || input + .nodes + .iter() + .any(|(key, node)| key.is_empty() || node.kind.is_empty()) + { + return Err(ControlError::Invalid( + "target, environment, positive generation, and named nodes are required".into(), + )); + } + let mut nodes: BTreeMap = BTreeMap::new(); + for (key, node) in &input.nodes { + nodes.insert(key.clone(), prepare_node(node)?); + } + let hashes: BTreeMap = nodes + .iter() + .map(|(key, node)| (key.clone(), node.hash.clone())) + .collect(); + let documents = prepare_documents(input)?; + let document_hashes: BTreeMap<_, _> = documents + .iter() + .map(|(kind, (hash, _))| (kind, hash)) + .collect(); + let manifest_digest = sha256_hex(&json_bytes(&serde_json::json!({ + "nodes": hashes, "documents": document_hashes, "environment_hash": input.environment_hash, + "image_id": input.image_id, "target": input.target, "sqb_version": input.sqb_version, + "git_revision": input.git_revision, "git_dirty": input.git_dirty, "git_author": input.git_author, + }))?); + Ok(PreparedSnapshot { + nodes, + hashes, + documents, + manifest_digest, + }) +} + +fn prepare_documents( + input: &SnapshotInput, +) -> Result)>, ControlError> { + let mut documents: BTreeMap)> = BTreeMap::new(); + for (kind, value) in &input.documents { + if ![ + "jobs", + "groups", + "triggers", + "roles", + "lineage", + "columns", + "providers", + ] + .contains(&kind.as_str()) + { + return Err(ControlError::Invalid(format!( + "unknown snapshot document {kind}" + ))); + } + let bytes = json_bytes(value)?; + documents.insert(kind.clone(), (sha256_hex(&bytes), compress(&bytes)?)); + } + Ok(documents) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs new file mode 100644 index 0000000000..ec1ce071f5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs @@ -0,0 +1,68 @@ +//! Immutable body inserts shared by snapshot publication. + +use std::collections::{BTreeMap, BTreeSet}; + +use uuid::Uuid; + +use crate::snapshots::models::{PreparedNode, SnapshotChange}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::main::store_source_text::store_source_text; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn store_nodes( + mut transaction: ControlTransaction, + snapshot_id: Uuid, + nodes: BTreeMap, +) -> Result { + for (key, node) in nodes { + (transaction, _) = query(transaction, + "INSERT INTO sqb_node_definitions (node_hash, body) VALUES ($1, $2) ON CONFLICT DO NOTHING", + vec![Parameter::Text(node.hash.clone()), Parameter::Bytes(node.definition)]).await?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshot_nodes (snapshot_id, node_key, node_kind, node_hash) VALUES ($1, $2, $3, $4)", + vec![Parameter::Id(snapshot_id), Parameter::Text(key.clone()), Parameter::Text(node.kind), Parameter::Text(node.hash)]).await?; + for (kind, (hash, body)) in node.sources { + transaction = store_source_text(transaction, (hash.clone(), body)).await?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshot_sources (snapshot_id, node_key, kind, text_hash) VALUES ($1, $2, $3, $4)", + vec![Parameter::Id(snapshot_id), Parameter::Text(key.clone()), Parameter::Text(kind.into()), Parameter::Text(hash)]).await?; + } + } + Ok(transaction) +} + +pub(crate) async fn store_changes( + mut transaction: ControlTransaction, + snapshot_id: Uuid, + previous: &BTreeMap, + hashes: &BTreeMap, +) -> Result { + let keys: BTreeSet<&String> = previous.keys().chain(hashes.keys()).collect(); + let mut sequence = 0; + for key in keys { + let before = previous.get(key); + let after = hashes.get(key); + if before == after { + continue; + } + let change = SnapshotChange { + change_kind: match (before, after) { + (None, _) => "added", + (_, None) => "removed", + _ => "changed", + } + .into(), + subject_key: key.clone(), + before_hash: before.cloned(), + after_hash: after.cloned(), + }; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshot_changes (snapshot_id, change_seq, change_kind, subject_key, before_hash, after_hash) \ + VALUES ($1,$2,$3,$4,$5,$6)", vec![Parameter::Id(snapshot_id), Parameter::Integer(sequence), + Parameter::Text(change.change_kind), Parameter::Text(change.subject_key), + Parameter::OptionalText(change.before_hash), Parameter::OptionalText(change.after_hash)]).await?; + sequence += 1; + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs new file mode 100644 index 0000000000..e96bb1fec3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs @@ -0,0 +1,56 @@ +//! Decode stored bodies and attach source text without evaluating it. + +use std::collections::BTreeMap; + +use crate::snapshots::models::SnapshotNode; +use crate::store::errors::ControlError; +use crate::store::main::decompress::decompress; +use crate::store::types::Row; +use serde::de::DeserializeOwned; +use serde_json::Value; + +fn decode_body(value: &Value) -> Result { + serde_json::from_slice(&decompress(value)?) + .map_err(|error| ControlError::Invalid(error.to_string())) +} + +pub(crate) fn read_nodes(rows: Vec) -> Result, ControlError> { + read_bodies(rows, "node_key") +} + +fn read_bodies( + rows: Vec, + key: &str, +) -> Result, ControlError> { + rows.into_iter() + .map(|row| { + let body: T = decode_body(&row["body"])?; + Ok((row[key].as_str().unwrap_or_default().to_owned(), body)) + }) + .collect() +} + +pub(crate) fn attach_sources( + mut nodes: BTreeMap, + rows: Vec, +) -> Result, ControlError> { + for row in rows { + let key = row["node_key"].as_str().unwrap_or_default(); + let node = nodes + .get_mut(key) + .ok_or_else(|| ControlError::Invalid("snapshot source has no node".into()))?; + let text = String::from_utf8(decompress(&row["body"])?) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + match row["kind"].as_str() { + Some("raw_sql") => node.raw_sql = Some(text), + Some("compiled_sql") => node.compiled_sql = Some(text), + Some("python_source") => node.python_source = Some(text), + _ => return Err(ControlError::Invalid("unknown stored source kind".into())), + } + } + Ok(nodes) +} + +pub(crate) fn read_documents(rows: Vec) -> Result, ControlError> { + read_bodies(rows, "kind") +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/constants.rs b/crates/sqlbuild-orchestration/src/snapshots/constants.rs new file mode 100644 index 0000000000..a995563fc0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/constants.rs @@ -0,0 +1,37 @@ +//! Snapshot retention keeps recent publications per location and target, collected in small batches. + +pub(crate) const DEFAULT_SNAPSHOT_RETENTION_COUNT: u16 = 10; +pub(crate) const SNAPSHOT_RETENTION_BATCH: usize = 10; +pub(crate) const CONTENT_HASH_CHUNK: usize = 500; + +/// Beyond the newest generations, any retained reference keeps a snapshot. +pub(crate) const EXPIRED_SNAPSHOTS: &str = "SELECT s.snapshot_id FROM sqb_snapshots s \ + WHERE (SELECT COUNT(*) FROM sqb_snapshots newer WHERE newer.location_id=s.location_id \ + AND newer.target=s.target AND newer.generation>s.generation)>=$1 \ + AND NOT EXISTS (SELECT 1 FROM sqb_current_snapshots c WHERE c.snapshot_id=s.snapshot_id) \ + AND NOT EXISTS (SELECT 1 FROM sqb_runs r WHERE r.snapshot_id=s.snapshot_id) \ + AND NOT EXISTS (SELECT 1 FROM sqb_jobs j WHERE j.snapshot_id=s.snapshot_id) \ + AND NOT EXISTS (SELECT 1 FROM sqb_triggers t WHERE t.snapshot_id=s.snapshot_id \ + OR t.removed_in_snapshot_id=s.snapshot_id) \ + AND NOT EXISTS (SELECT 1 FROM sqb_location_work w WHERE w.snapshot_id=s.snapshot_id) \ + ORDER BY s.published_at, s.snapshot_id LIMIT $2"; + +/// Content-addressed bodies of the expiring snapshots, removed once nothing else references them. +pub(crate) const CONTENT: [((&str, &str), &str, &str); 3] = [ + ( + ("sqb_snapshot_nodes", "node_hash"), + "sqb_node_definitions", + "", + ), + ( + ("sqb_snapshot_sources", "text_hash"), + "sqb_source_texts", + " AND NOT EXISTS (SELECT 1 FROM sqb_warehouse_fingerprints f \ + WHERE f.definition_text_hash=sqb_source_texts.text_hash)", + ), + ( + ("sqb_snapshot_documents", "document_hash"), + "sqb_documents", + "", + ), +]; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/_publish_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/_publish_snapshot.rs new file mode 100644 index 0000000000..7829a8a40e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/_publish_snapshot.rs @@ -0,0 +1,97 @@ +//! Publish a complete generation and its current pointer in one transaction. + +use std::collections::BTreeMap; + +use serde_json::Value; +use uuid::Uuid; + +use crate::declarations::main::publish_definitions::publish_definitions; +use crate::declarations::main::snapshot_definitions::snapshot_definitions; +use crate::snapshots::_helpers::preparation::prepare_snapshot; +use crate::snapshots::_helpers::publication::{store_changes, store_nodes}; +use crate::snapshots::models::{PreparedSnapshot, SnapshotInput, SnapshotRecord}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{Change, ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn publish_snapshot_with_defaults( + store: &ControlStore, + input: &SnapshotInput, + pause_new_triggers: bool, +) -> Result { + let definitions = snapshot_definitions(input)?; + let PreparedSnapshot { + nodes, + hashes, + documents, + manifest_digest, + } = prepare_snapshot(input)?; + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, previous) = query(transaction, + "SELECT s.generation FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id \ + WHERE c.location_id=$1 AND c.target=$2", + vec![Parameter::Id(input.location_id), Parameter::Text(input.target.clone())]).await?; + if previous + .first() + .and_then(|row| row.get("generation")) + .and_then(Value::as_i64) + .is_some_and(|generation| generation >= input.generation) + { + return Err(ControlError::StaleGeneration); + } + let (mut transaction, previous) = query(transaction, + "SELECT n.node_key, n.node_hash FROM sqb_snapshot_nodes n JOIN sqb_current_snapshots c ON c.snapshot_id=n.snapshot_id \ + WHERE c.location_id=$1 AND c.target=$2", + vec![Parameter::Id(input.location_id), Parameter::Text(input.target.clone())]).await?; + let previous: BTreeMap = previous + .into_iter() + .map(|row| { + ( + row["node_key"].as_str().unwrap_or_default().into(), + row["node_hash"].as_str().unwrap_or_default().into(), + ) + }) + .collect(); + let record = SnapshotRecord { + snapshot_id: Uuid::now_v7(), + location_id: input.location_id, + target: input.target.clone(), + generation: input.generation, + manifest_digest, + published_at: transaction.now, + }; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshots (snapshot_id, location_id, target, generation, git_revision, git_dirty, git_author, image_id, \ + environment_hash, sqb_version, snapshot_schema_version, manifest_digest, published_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,1,$11,$12)", vec![ + Parameter::Id(record.snapshot_id), Parameter::Id(input.location_id), Parameter::Text(input.target.clone()), + Parameter::Integer(input.generation), Parameter::OptionalText(input.git_revision.clone()), Parameter::Boolean(input.git_dirty), + Parameter::OptionalText(input.git_author.clone()), Parameter::OptionalText(input.image_id.clone()), + Parameter::Text(input.environment_hash.clone()), Parameter::Text(input.sqb_version.clone()), + Parameter::Text(record.manifest_digest.clone()), Parameter::Time(record.published_at), + ]).await?; + transaction = store_nodes(transaction, record.snapshot_id, nodes).await?; + for (kind, (hash, body)) in documents { + (transaction, _) = query(transaction, + "INSERT INTO sqb_documents (document_hash, body) VALUES ($1, $2) ON CONFLICT DO NOTHING", + vec![Parameter::Text(hash.clone()), Parameter::Bytes(body)]).await?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshot_documents (snapshot_id, kind, document_hash) VALUES ($1, $2, $3)", + vec![Parameter::Id(record.snapshot_id), Parameter::Text(kind), Parameter::Text(hash)]).await?; + } + transaction = store_changes(transaction, record.snapshot_id, &previous, &hashes).await?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_current_snapshots (location_id, target, snapshot_id, since) VALUES ($1,$2,$3,$4) \ + ON CONFLICT (location_id, target) DO UPDATE SET snapshot_id=excluded.snapshot_id, since=excluded.since", + vec![Parameter::Id(input.location_id), Parameter::Text(input.target.clone()), + Parameter::Id(record.snapshot_id), Parameter::Time(record.published_at)]).await?; + transaction = + publish_definitions(transaction, &record, definitions, pause_new_triggers).await?; + transaction + .changes + .insert(Change::Snapshot(input.location_id, input.target.clone())); + commit(transaction).await?; + Ok(record) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs new file mode 100644 index 0000000000..ddffa096f8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs @@ -0,0 +1,50 @@ +//! Read one consistent generation without locking out snapshot publishers. + +use uuid::Uuid; + +use crate::snapshots::_helpers::reading::attach_sources; +use crate::snapshots::main::read_current_record::read_current_record; +use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::snapshots::models::{CurrentSnapshot, SnapshotChange}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The last successfully published generation, or none before the first successful compile. +pub async fn current_snapshot( + store: &ControlStore, + location_id: Uuid, + target: &str, +) -> Result, ControlError> { + let (transaction, record) = read_current_record( + begin(store, TransactionMode::Read).await?, + (location_id, target), + ) + .await?; + let Some(record) = record else { + return Ok(None); + }; + let (transaction, nodes) = read_snapshot_nodes(transaction, record.snapshot_id, None).await?; + let (transaction, rows) = query(transaction, + "SELECT s.node_key, s.kind, t.body FROM sqb_snapshot_sources s JOIN sqb_source_texts t ON t.text_hash=s.text_hash WHERE s.snapshot_id=$1", + vec![Parameter::Id(record.snapshot_id)]).await?; + let nodes = attach_sources(nodes, rows)?; + let (transaction, documents) = + read_snapshot_documents(transaction, record.snapshot_id, None).await?; + let (transaction, rows) = query(transaction, + "SELECT change_kind, subject_key, before_hash, after_hash FROM sqb_snapshot_changes WHERE snapshot_id=$1 ORDER BY change_seq", + vec![Parameter::Id(record.snapshot_id)]).await?; + let changes: Vec = + rows.into_iter().map(decode_row).collect::>()?; + commit(transaction).await?; + Ok(Some(CurrentSnapshot { + record, + nodes, + documents, + changes, + })) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs new file mode 100644 index 0000000000..940c0a3804 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs @@ -0,0 +1,23 @@ +//! Reconcile publication identity without reading node definitions, source bodies or documents. + +use uuid::Uuid; + +use crate::snapshots::main::read_current_record::read_current_record; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn current_snapshot_record( + store: &ControlStore, + location_id: Uuid, + target: &str, +) -> Result, ControlError> { + let (transaction, record) = read_current_record( + begin(store, TransactionMode::Read).await?, + (location_id, target), + ) + .await?; + commit(transaction).await?; + Ok(record) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs b/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs new file mode 100644 index 0000000000..3060d1b261 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs @@ -0,0 +1,30 @@ +//! Resolve a job's physical writers from its pinned snapshot. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn job_write_set( + transaction: ControlTransaction, + snapshot_id: Uuid, + selection: &BTreeSet, +) -> Result<(ControlTransaction, BTreeSet), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT node_key FROM sqb_snapshot_nodes WHERE snapshot_id=$1 AND node_kind IN ('model','seed','udf','table_fn')", + vec![Parameter::Id(snapshot_id)]).await?; + Ok(( + transaction, + rows.into_iter() + .filter_map(|row| { + row.get("node_key") + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + }) + .filter(|key| selection.contains(key)) + .collect(), + )) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs new file mode 100644 index 0000000000..342c47fd5a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -0,0 +1,14 @@ +pub(super) mod _publish_snapshot; +pub mod current_snapshot; +pub mod current_snapshot_record; +pub(crate) mod job_write_set; +pub mod next_remote_generation; +pub(crate) mod prune_snapshots; +pub(crate) mod publish_development_snapshot; +pub mod publish_remote_snapshot; +pub mod publish_snapshot; +pub(crate) mod read_current_record; +pub mod read_person_node_sql; +pub(crate) mod read_snapshot_documents; +pub(crate) mod read_snapshot_nodes; +pub mod verify_execution; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs b/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs new file mode 100644 index 0000000000..3c4f974037 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs @@ -0,0 +1,36 @@ +//! Reconcile publication generations from the authenticated server, including lost acknowledgements. + +use uuid::Uuid; + +use crate::api::errors::ClientError; +use crate::api::main::client_response::client_response; +use crate::api::main::connect_client::connect_client; +use crate::snapshots::models::SnapshotRecord; + +pub async fn next_remote_generation( + server_url: &str, + bearer: &str, + location_id: Uuid, + target: &str, +) -> Result { + let connection = connect_client(server_url, bearer)?; + let url = connection.endpoint(&[ + "api", + "locations", + &location_id.to_string(), + "targets", + target, + "snapshots", + "metadata", + ])?; + match client_response::(connection.client.get(url)).await { + Ok(current) => current + .generation + .checked_add(1) + .ok_or(ClientError::Configuration( + "snapshot generation is exhausted", + )), + Err(ClientError::Rejected(status)) if status == reqwest::StatusCode::NOT_FOUND => Ok(1), + Err(error) => Err(error), + } +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/prune_snapshots.rs b/crates/sqlbuild-orchestration/src/snapshots/main/prune_snapshots.rs new file mode 100644 index 0000000000..6b42331385 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/prune_snapshots.rs @@ -0,0 +1,105 @@ +//! Snapshot expiry is by reference: anything a retained row points at stays, and content follows. + +use uuid::Uuid; + +use crate::snapshots::constants::{ + CONTENT, CONTENT_HASH_CHUNK, EXPIRED_SNAPSHOTS, SNAPSHOT_RETENTION_BATCH, +}; +use crate::snapshots::models::SnapshotRetention; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, ControlTransaction, Parameter, TransactionMode}; + +/// Batches commit independently; a repeated or interrupted pass finds only what is still expired. +pub(crate) async fn prune_snapshots( + store: &ControlStore, + retention: SnapshotRetention, +) -> Result { + let mut deleted = 0; + loop { + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, rows) = query( + transaction, + EXPIRED_SNAPSHOTS, + vec![ + Parameter::Integer(retention.kept()), + Parameter::Integer(SNAPSHOT_RETENTION_BATCH as i64), + ], + ) + .await?; + let mut expired: Vec = Vec::with_capacity(rows.len()); + for row in &rows { + expired.push( + serde_json::from_value(row["snapshot_id"].clone()) + .map_err(|error| ControlError::Schema(error.to_string()))?, + ); + } + let transaction = delete_snapshots(transaction, &expired).await?; + commit(transaction).await?; + deleted += expired.len(); + if expired.len() < SNAPSHOT_RETENTION_BATCH { + return Ok(deleted); + } + } +} + +async fn delete_snapshots( + mut transaction: ControlTransaction, + expired: &[Uuid], +) -> Result { + if expired.is_empty() { + return Ok(transaction); + } + let placeholders = (1..=expired.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + let ids: Vec = expired.iter().copied().map(Parameter::Id).collect(); + let mut hashes: Vec> = Vec::with_capacity(CONTENT.len()); + for ((table, column), _, _) in CONTENT { + let rows; + (transaction, rows) = query( + transaction, + &format!("SELECT DISTINCT {column} FROM {table} WHERE snapshot_id IN ({placeholders})"), + ids.clone(), + ) + .await?; + hashes.push( + rows.iter() + .filter_map(|row| row[column].as_str().map(str::to_owned)) + .collect(), + ); + } + for table in [ + "sqb_snapshot_sources", + "sqb_snapshot_nodes", + "sqb_snapshot_documents", + "sqb_snapshot_changes", + "sqb_snapshots", + ] { + (transaction, _) = query( + transaction, + &format!("DELETE FROM {table} WHERE snapshot_id IN ({placeholders})"), + ids.clone(), + ) + .await?; + } + for (((table, column), content, elsewhere), hashes) in CONTENT.iter().zip(hashes) { + for chunk in hashes.chunks(CONTENT_HASH_CHUNK) { + let placeholders = (1..=chunk.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + (transaction, _) = query( + transaction, + &format!( + "DELETE FROM {content} WHERE {column} IN ({placeholders}) AND NOT EXISTS \ + (SELECT 1 FROM {table} n WHERE n.{column}={content}.{column}){elsewhere}" + ), + chunk.iter().cloned().map(Parameter::Text).collect(), + ) + .await?; + } + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/publish_development_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publish_development_snapshot.rs new file mode 100644 index 0000000000..9d3065c812 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/publish_development_snapshot.rs @@ -0,0 +1,13 @@ +//! Publication keeps development safety defaults inside the snapshot transaction. + +use crate::snapshots::main::_publish_snapshot::publish_snapshot_with_defaults; +use crate::snapshots::models::{SnapshotInput, SnapshotRecord}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +pub(crate) async fn publish_development_snapshot( + store: &ControlStore, + input: &SnapshotInput, +) -> Result { + publish_snapshot_with_defaults(store, input, true).await +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/publish_remote_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publish_remote_snapshot.rs new file mode 100644 index 0000000000..681cc0f61e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/publish_remote_snapshot.rs @@ -0,0 +1,23 @@ +//! A code location sends retained compiled data over HTTP and never opens the control store. + +use crate::api::errors::ClientError; +use crate::api::main::client_response::client_response; +use crate::api::main::connect_client::connect_client; +use crate::snapshots::models::{SnapshotInput, SnapshotRecord}; + +pub async fn publish_remote_snapshot( + server_url: &str, + bearer: &str, + input: &SnapshotInput, +) -> Result { + let connection = connect_client(server_url, bearer)?; + let url = connection.endpoint(&[ + "api", + "locations", + &input.location_id.to_string(), + "targets", + &input.target, + "snapshots", + ])?; + client_response(connection.client.post(url).json(input)).await +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs new file mode 100644 index 0000000000..471a2bebe6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs @@ -0,0 +1,14 @@ +//! Publish a generation using ordinary production trigger defaults. + +use crate::snapshots::main::_publish_snapshot::publish_snapshot_with_defaults; +use crate::snapshots::models::{SnapshotInput, SnapshotRecord}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +/// Reject stale generations without changing the last good snapshot. +pub async fn publish_snapshot( + store: &ControlStore, + input: &SnapshotInput, +) -> Result { + publish_snapshot_with_defaults(store, input, false).await +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_current_record.rs b/crates/sqlbuild-orchestration/src/snapshots/main/read_current_record.rs new file mode 100644 index 0000000000..a5be66e12f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/read_current_record.rs @@ -0,0 +1,22 @@ +//! Read publication identity inside the caller's existing authorization transaction. + +use uuid::Uuid; + +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_current_record( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT s.snapshot_id, s.location_id, s.target, s.generation, s.manifest_digest, s.published_at \ + FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id WHERE c.location_id=$1 AND c.target=$2", + vec![Parameter::Id(location_id), Parameter::Text(target.into())]).await?; + Ok(( + transaction, + rows.into_iter().next().map(decode_row).transpose()?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs b/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs new file mode 100644 index 0000000000..02b144d4cb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs @@ -0,0 +1,66 @@ +//! Live SQL permission, current publication and sensitive-read audit share one writer fence. + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::main::authorize_person::authorize_person; +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::snapshots::_helpers::reading::attach_sources; +use crate::snapshots::main::read_current_record::read_current_record; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::snapshots::models::NodeSql; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn read_person_node_sql( + store: &ControlStore, + credentials: (&AuditContext, Uuid), + (location_id, target, node_key): (Uuid, &str, &str), +) -> Result, ControlError> { + let scope = PermissionScope { + action: Action::ViewSql, + location_id: Some(location_id), + target: Some(target.into()), + nodes: [node_key.into()].into(), + }; + let (transaction, context) = authorize_person( + begin(store, TransactionMode::Write).await?, + credentials, + &scope, + DefaultRole::None, + ) + .await?; + let (mut transaction, record) = read_current_record(transaction, (location_id, target)).await?; + let mut result = None; + if let Some(record) = record { + let nodes; + (transaction, nodes) = + read_snapshot_nodes(transaction, record.snapshot_id, Some(&scope.nodes)).await?; + if !nodes.is_empty() { + let rows; + (transaction, rows) = query(transaction, + "SELECT s.node_key, s.kind, t.body FROM sqb_snapshot_sources s JOIN sqb_source_texts t ON t.text_hash=s.text_hash \ + WHERE s.snapshot_id=$1 AND s.node_key=$2 AND s.kind IN ('raw_sql','compiled_sql')", + vec![Parameter::Id(record.snapshot_id), Parameter::Text(node_key.into())]).await?; + if let Some(node) = attach_sources(nodes, rows)?.remove(node_key) { + result = Some(NodeSql { + snapshot_id: record.snapshot_id, + node_key: node_key.into(), + raw_sql: node.raw_sql, + compiled_sql: node.compiled_sql, + }); + } + } + } + let transaction = append_audit( + transaction, + &context, + (Uuid::now_v7(), "sql.read"), + (json!(scope), json!(null), json!({"snapshot_id":result.as_ref().map(|node| node.snapshot_id),"found":result.is_some()})), + ) + .await?; + commit(transaction).await?; + Ok(result) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs b/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs new file mode 100644 index 0000000000..45a03182f8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs @@ -0,0 +1,36 @@ +//! Decode immutable documents within the caller's consistent snapshot transaction. + +use std::collections::BTreeMap; + +use serde_json::Value; +use uuid::Uuid; + +use crate::snapshots::_helpers::reading::read_documents; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_snapshot_documents( + transaction: ControlTransaction, + snapshot_id: Uuid, + kinds: Option<&[&str]>, +) -> Result<(ControlTransaction, BTreeMap), ControlError> { + if kinds.is_some_and(<[&str]>::is_empty) { + return Ok((transaction, BTreeMap::new())); + } + let mut parameters = vec![Parameter::Id(snapshot_id)]; + let filter = kinds.map_or(String::new(), |keys| { + parameters.extend(keys.iter().map(|key| Parameter::Text((*key).into()))); + format!( + " AND d.kind IN ({})", + (2..parameters.len() + 1) + .map(|index| format!("${index}")) + .collect::>() + .join(",") + ) + }); + let (transaction, rows) = query(transaction, + &format!("SELECT d.kind, b.body FROM sqb_snapshot_documents d JOIN sqb_documents b ON b.document_hash=d.document_hash WHERE d.snapshot_id=$1{filter}"), + parameters).await?; + Ok((transaction, read_documents(rows)?)) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs b/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs new file mode 100644 index 0000000000..33240705d0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs @@ -0,0 +1,36 @@ +//! Read retained node definitions without expanding their display-only source bodies. + +use std::collections::{BTreeMap, BTreeSet}; + +use uuid::Uuid; + +use crate::snapshots::_helpers::reading::read_nodes; +use crate::snapshots::models::SnapshotNode; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_snapshot_nodes( + transaction: ControlTransaction, + snapshot_id: Uuid, + selection: Option<&BTreeSet>, +) -> Result<(ControlTransaction, BTreeMap), ControlError> { + if selection.is_some_and(BTreeSet::is_empty) { + return Ok((transaction, BTreeMap::new())); + } + let mut parameters = vec![Parameter::Id(snapshot_id)]; + let filter = selection.map_or(String::new(), |keys| { + parameters.extend(keys.iter().cloned().map(Parameter::Text)); + format!( + " AND n.node_key IN ({})", + (2..parameters.len() + 1) + .map(|index| format!("${index}")) + .collect::>() + .join(",") + ) + }); + let (transaction, rows) = query(transaction, + &format!("SELECT n.node_key, d.body FROM sqb_snapshot_nodes n JOIN sqb_node_definitions d ON d.node_hash=n.node_hash WHERE n.snapshot_id=$1{filter} ORDER BY n.node_key"), + parameters).await?; + Ok((transaction, read_nodes(rows)?)) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs b/crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs new file mode 100644 index 0000000000..d4d2c00236 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs @@ -0,0 +1,50 @@ +//! Refuse execution when a worker's local compilation differs from the pinned publication. + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::runs::models::RunExecution; +use crate::snapshots::_helpers::content::node_content; +use crate::snapshots::models::SnapshotInput; +use crate::store::errors::ControlError; + +/// Display documents and publication generations are not execution inputs. +pub fn verify_execution( + input: &SnapshotInput, + expected: &RunExecution, +) -> Result<(), ControlError> { + let request = &expected.claim.request; + if input.location_id != request.location_id || input.target != request.target { + return Err(ControlError::Invalid( + "worker location or target differs from the pinned run".into(), + )); + } + if input.image_id != request.image_id { + return Err(ControlError::Invalid( + "worker image differs from the pinned run".into(), + )); + } + if input.environment_hash != expected.environment_hash { + return Err(ControlError::Invalid( + "worker environment differs from the pinned snapshot".into(), + )); + } + if input.sqb_version != expected.sqb_version { + return Err(ControlError::Invalid( + "worker SQLBuild version differs from the pinned snapshot".into(), + )); + } + if input.nodes.len() != expected.node_hashes.len() { + return Err(ControlError::Invalid( + "worker node set differs from the pinned snapshot".into(), + )); + } + for (key, node) in &input.nodes { + let hash = sha256_hex(&node_content(node)?.definition); + if expected.node_hashes.get(key) != Some(&hash) { + return Err(ControlError::Invalid( + "worker node content differs from the pinned snapshot".into(), + )); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/mod.rs new file mode 100644 index 0000000000..e833f67102 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +#[path = "../../tests/snapshots/test_contracts.rs"] +mod tests; diff --git a/crates/sqlbuild-orchestration/src/snapshots/models.rs b/crates/sqlbuild-orchestration/src/snapshots/models.rs new file mode 100644 index 0000000000..4fef7aee74 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/models.rs @@ -0,0 +1,124 @@ +//! Snapshot inputs are data only: no executable user objects cross into the control plane. + +use std::collections::BTreeMap; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use uuid::Uuid; + +/// A compiled node and its display-only source. SQL and Python are never run by the server. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct SnapshotNode { + pub kind: String, + pub definition: Value, + pub raw_sql: Option, + pub compiled_sql: Option, + pub python_source: Option, +} + +/// SQL inspection never returns Python source, execution artifacts or unrelated node metadata. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct NodeSql { + pub snapshot_id: Uuid, + pub node_key: String, + pub raw_sql: Option, + pub compiled_sql: Option, +} + +/// A location's successful compile for one target. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct SnapshotInput { + pub location_id: Uuid, + pub target: String, + pub generation: i64, + pub git_revision: Option, + pub git_dirty: bool, + pub git_author: Option, + pub image_id: Option, + pub environment_hash: String, + pub sqb_version: String, + pub nodes: BTreeMap, + pub documents: BTreeMap, +} + +/// Immutable publication metadata. Generations increase independently per location and target. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct SnapshotRecord { + pub snapshot_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub generation: i64, + pub manifest_digest: String, + pub published_at: DateTime, +} + +/// Node-level changes do not duplicate unchanged definitions or source text. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct SnapshotChange { + pub change_kind: String, + pub subject_key: String, + pub before_hash: Option, + pub after_hash: Option, +} + +/// The current successful snapshot, read with a consistent generation and change set. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct CurrentSnapshot { + pub record: SnapshotRecord, + pub nodes: BTreeMap, + pub documents: BTreeMap, + pub changes: Vec, +} + +/// Canonical identity is shared by publication and worker verification; only publication compresses. +#[derive(Debug)] +pub(crate) struct NodeContent<'a> { + pub(crate) definition: Vec, + pub(crate) sources: BTreeMap<&'static str, (String, &'a str)>, +} + +#[derive(Debug)] +pub(crate) struct PreparedNode { + pub(crate) kind: String, + pub(crate) hash: String, + pub(crate) definition: Vec, + pub(crate) sources: BTreeMap<&'static str, (String, Vec)>, +} + +#[derive(Debug)] +pub(crate) struct PreparedSnapshot { + pub(crate) nodes: BTreeMap, + pub(crate) hashes: BTreeMap, + pub(crate) documents: BTreeMap)>, + pub(crate) manifest_digest: String, +} + +/// Installation configuration: how many recent publications to keep per location and target. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SnapshotRetention { + count: u16, +} + +impl SnapshotRetention { + pub fn count(count: u16) -> Result { + if count == 0 { + return Err(crate::store::errors::ControlError::Invalid( + "snapshot retention must keep at least one snapshot".into(), + )); + } + Ok(Self { count }) + } + + pub(crate) fn kept(self) -> i64 { + i64::from(self.count) + } +} + +impl Default for SnapshotRetention { + fn default() -> Self { + Self { + count: crate::snapshots::constants::DEFAULT_SNAPSHOT_RETENTION_COUNT, + } + } +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/changes.rs b/crates/sqlbuild-orchestration/src/store/_helpers/changes.rs new file mode 100644 index 0000000000..cd23dd0052 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/changes.rs @@ -0,0 +1,87 @@ +//! Commit-scoped invalidations; authoritative records remain in their existing stores. + +use std::collections::{BTreeSet, HashMap}; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, OnceLock}; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use tokio::sync::broadcast; + +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ + Change, ChangeMessage, ChangeTopic, ChangedScope, ControlTransaction, Parameter, +}; + +/// Weak entries do not keep deleted test databases or idle subscribers alive. +pub(crate) fn sqlite_changes( + path: &Path, +) -> Result, ControlError> { + static CHANNELS: OnceLock>>> = + OnceLock::new(); + let path = path.canonicalize()?; + let mut channels = CHANNELS + .get_or_init(Mutex::default) + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + channels.retain(|_, sender| sender.strong_count() > 0); + if let Some(sender) = channels.get(&path).and_then(broadcast::WeakSender::upgrade) { + return Ok(sender); + } + let (sender, _) = broadcast::channel(256); + channels.insert(path, sender.downgrade()); + Ok(sender) +} + +pub(crate) async fn resolve_changes( + mut transaction: ControlTransaction, +) -> Result<(ControlTransaction, BTreeSet), ControlError> { + let mut messages: BTreeSet = BTreeSet::new(); + for change in std::mem::take(&mut transaction.changes) { + if change == Change::Credentials { + messages.insert(ChangeMessage::CredentialsChanged); + continue; + } + let scope = match &change { + Change::Snapshot(location_id, target) | Change::NodeStatus(location_id, target) => { + ChangedScope { + location_id: *location_id, + target: target.clone(), + } + } + Change::Run(id) | Change::Trigger(id) => { + let sql = if matches!(change, Change::Run(_)) { + "SELECT location_id, target FROM sqb_runs WHERE run_id=$1" + } else { + "SELECT location_id, target FROM sqb_triggers WHERE trigger_id=$1" + }; + let rows; + (transaction, rows) = query(transaction, sql, vec![Parameter::Id(*id)]).await?; + let Some(row) = rows.into_iter().next() else { + continue; + }; + decode_row(row)? + } + Change::Credentials => continue, + }; + let topics = match change { + Change::Run(id) => vec![ + ChangeTopic::Runs, + ChangeTopic::Run(id), + ChangeTopic::NodeStatus, + ], + Change::Trigger(_) => vec![ChangeTopic::Ticks], + Change::Snapshot(..) => vec![ChangeTopic::Snapshot, ChangeTopic::NodeStatus], + Change::NodeStatus(..) => vec![ChangeTopic::NodeStatus], + Change::Credentials => continue, + }; + for topic in topics { + messages.insert(ChangeMessage::Changed { + location_id: scope.location_id, + target_hash: sha256_hex(scope.target.as_bytes()), + topic, + }); + } + } + Ok((transaction, messages)) +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/database.rs b/crates/sqlbuild-orchestration/src/store/_helpers/database.rs new file mode 100644 index 0000000000..433dd5b495 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/database.rs @@ -0,0 +1,154 @@ +//! Typed binding and row decoding for the two control-store engines. + +use chrono::{DateTime, SecondsFormat, Utc}; +use rusqlite::types::{Value as SqliteValue, ValueRef}; +use serde_json::Value; +use sqlbuild_recall::schema::models::ColumnType; +use sqlx_core::{column::Column, row::Row as _, type_info::TypeInfo, value::ValueRef as _}; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::Parameter; +use crate::store::types::Row; + +pub(crate) fn sqlite_value(parameter: &Parameter) -> SqliteValue { + match parameter { + Parameter::Null(_) => SqliteValue::Null, + Parameter::Text(value) => SqliteValue::Text(value.clone()), + Parameter::OptionalText(value) => { + value.clone().map_or(SqliteValue::Null, SqliteValue::Text) + } + Parameter::Bytes(value) => SqliteValue::Blob(value.clone()), + Parameter::Integer(value) => SqliteValue::Integer(*value), + Parameter::Real(value) => SqliteValue::Real(*value), + Parameter::Json(value) => SqliteValue::Text(value.to_string()), + Parameter::Boolean(value) => SqliteValue::Integer(i64::from(*value)), + Parameter::Id(value) => SqliteValue::Text(value.to_string()), + Parameter::OptionalId(value) => value.map_or(SqliteValue::Null, |value| { + SqliteValue::Text(value.to_string()) + }), + Parameter::Time(value) => { + SqliteValue::Text(value.to_rfc3339_opts(SecondsFormat::Micros, true)) + } + Parameter::OptionalTime(value) => value.map_or(SqliteValue::Null, |value| { + SqliteValue::Text(value.to_rfc3339_opts(SecondsFormat::Micros, true)) + }), + } +} + +pub(crate) fn sqlite_query( + connection: &rusqlite::Connection, + sql: &str, + parameters: &[Parameter], +) -> Result, ControlError> { + let values: Vec = parameters.iter().map(sqlite_value).collect(); + let mut statement = connection.prepare(sql)?; + let columns: Vec = statement + .column_names() + .into_iter() + .map(str::to_owned) + .collect(); + let declared_types: Vec> = statement + .columns() + .into_iter() + .map(|column| column.decl_type().map(str::to_ascii_uppercase)) + .collect(); + for (index, value) in values.into_iter().enumerate() { + let name = format!("${}", index + 1); + let position = statement + .parameter_index(&name)? + .ok_or_else(|| ControlError::Invalid(format!("missing SQL parameter {name}")))?; + statement.raw_bind_parameter(position, value)?; + } + let mut query = statement.raw_query(); + let mut result: Vec = Vec::new(); + while let Some(row) = query.next()? { + let mut mapped = Row::new(); + for (index, name) in columns.iter().enumerate() { + let value = match row.get_ref(index)? { + ValueRef::Null => Value::Null, + ValueRef::Integer(value) if declared_types[index].as_deref() == Some("BOOLEAN") => { + Value::Bool(value != 0) + } + ValueRef::Integer(value) => Value::from(value), + ValueRef::Real(value) => Value::from(value), + ValueRef::Text(value) => Value::from(String::from_utf8_lossy(value).into_owned()), + ValueRef::Blob(value) => Value::from(value.to_vec()), + }; + let value = if declared_types[index].as_deref() == Some("JSON") { + value.as_str().map_or_else( + || Ok(value.clone()), + |text| { + serde_json::from_str(text) + .map_err(|error| ControlError::Invalid(error.to_string())) + }, + )? + } else { + value + }; + mapped.insert(name.clone(), value); + } + result.push(mapped); + } + Ok(result) +} + +pub(crate) async fn postgres_query( + transaction: &mut sqlx_core::transaction::Transaction<'static, sqlx_postgres::Postgres>, + sql: &str, + parameters: Vec, +) -> Result, ControlError> { + let mut query = sqlx_core::query::query(sql); + for parameter in parameters { + query = match parameter { + Parameter::Null(kind) => match kind { + ColumnType::Text => query.bind(None::), + ColumnType::Timestamp => query.bind(None::>), + ColumnType::Json => query.bind(None::), + ColumnType::Integer => query.bind(None::), + ColumnType::Boolean => query.bind(None::), + ColumnType::Real => query.bind(None::), + ColumnType::Blob => query.bind(None::>), + }, + Parameter::Text(value) => query.bind(value), + Parameter::OptionalText(value) => query.bind(value), + Parameter::Bytes(value) => query.bind(value), + Parameter::Integer(value) => query.bind(value), + Parameter::Real(value) => query.bind(value), + Parameter::Json(value) => query.bind(value), + Parameter::Boolean(value) => query.bind(value), + Parameter::Id(value) => query.bind(value), + Parameter::OptionalId(value) => query.bind(value), + Parameter::Time(value) => query.bind(value), + Parameter::OptionalTime(value) => query.bind(value), + }; + } + let mut result: Vec = Vec::new(); + for row in query.fetch_all(&mut **transaction).await? { + let mut mapped = Row::new(); + for (index, column) in row.columns().iter().enumerate() { + let raw = row.try_get_raw(index)?; + let value = if raw.is_null() { + Value::Null + } else { + match column.type_info().name() { + "BOOL" => Value::from(row.try_get::(index)?), + "INT8" => Value::from(row.try_get::(index)?), + "INT4" => Value::from(row.try_get::(index)?), + "FLOAT8" => Value::from(row.try_get::(index)?), + "UUID" => Value::from(row.try_get::(index)?.to_string()), + "TIMESTAMPTZ" => Value::from( + row.try_get::, _>(index)? + .to_rfc3339_opts(SecondsFormat::Micros, true), + ), + "JSONB" | "JSON" => row.try_get::(index)?, + "BYTEA" => Value::from(row.try_get::, _>(index)?), + _ => Value::from(row.try_get::(index)?), + } + }; + mapped.insert(column.name().to_owned(), value); + } + result.push(mapped); + } + Ok(result) +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs new file mode 100644 index 0000000000..f35f06a534 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -0,0 +1,35 @@ +//! One schema template, rendered only at the database type boundary. + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::store::_helpers::recall_schema::recall_schema; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; +use crate::store::types::Row; + +pub(crate) const SCHEMA: &str = include_str!("../schema.sql"); + +pub(crate) fn schema_digest(store: &ControlStore) -> String { + sha256_hex(format!("{}\n{SCHEMA}", recall_schema(store).join(";\n")).as_bytes()) +} + +/// Accept only the exact schema this server creates. +pub(crate) fn verify_schema(rows: &[Row], digest: &str) -> Result<(), ControlError> { + if rows + .iter() + .any(|row| row.get("checksum").and_then(|value| value.as_str()) != Some(digest)) + { + return Err(ControlError::Schema( + "control store schema checksum mismatch".into(), + )); + } + Ok(()) +} + +pub(crate) fn render(store: &ControlStore, sql: &str) -> String { + let postgres = matches!(store, ControlStore::Postgres(_)); + sql.replace("{uuid}", if postgres { "UUID" } else { "TEXT" }) + .replace("{time}", if postgres { "TIMESTAMPTZ" } else { "TEXT" }) + .replace("{binary}", if postgres { "BYTEA" } else { "BLOB" }) + .replace("{json}", if postgres { "JSONB" } else { "JSON" }) +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs new file mode 100644 index 0000000000..1c71cbb1a9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod changes; +pub(crate) mod database; +pub(crate) mod migration; +pub(crate) mod open_sqlite; +pub(crate) mod recall_rows; +pub(crate) mod recall_schema; diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs b/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs new file mode 100644 index 0000000000..b4ca936711 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs @@ -0,0 +1,51 @@ +//! Configure a file-backed SQLite control store without interpreting filenames as URIs. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +pub(crate) fn sqlite_path(url: &str) -> Result { + let path = Path::new(url.strip_prefix("sqlite://").unwrap_or(url)).to_path_buf(); + if path.as_os_str().is_empty() + || path == Path::new(":memory:") + || url.contains("://") && !url.starts_with("sqlite://") + { + return Err(ControlError::Invalid( + "control-store URL must name PostgreSQL or a SQLite file".into(), + )); + } + Ok(path) +} + +pub(crate) fn open_sqlite(path: PathBuf) -> Result { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + std::fs::create_dir_all(parent)?; + } + let absolute = std::env::current_dir()?.join(path); + let path = + if absolute.exists() { + absolute.canonicalize()? + } else { + absolute + .parent() + .ok_or_else(|| ControlError::Invalid("control store requires a filename".into()))? + .canonicalize()? + .join(absolute.file_name().ok_or_else(|| { + ControlError::Invalid("control store requires a filename".into()) + })?) + }; + let connection = rusqlite::Connection::open_with_flags( + &path, + rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE + | rusqlite::OpenFlags::SQLITE_OPEN_CREATE + | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, + )?; + connection.busy_timeout(Duration::from_secs(30))?; + connection.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL")?; + Ok(ControlStore::Sqlite(path)) +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/recall_rows.rs b/crates/sqlbuild-orchestration/src/store/_helpers/recall_rows.rs new file mode 100644 index 0000000000..2d92e03869 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/recall_rows.rs @@ -0,0 +1,100 @@ +//! Insert native Recall rows and identify retry conflicts within a control transaction. + +use std::collections::HashSet; + +use chrono::{DateTime, SecondsFormat}; +use sqlbuild_recall::records::models::{RecallRow, RecallValue}; +use sqlbuild_recall::schema::main::table_columns::table_columns; +use sqlbuild_recall::schema::models::{ColumnType, Table}; +use sqlbuild_recall::store::main::conflict_evidence::conflict_evidence; +use sqlbuild_recall::store::main::relational_insert::relational_insert; +use sqlbuild_recall::store::main::relational_lookup::relational_lookup; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{Connection, ControlTransaction, Parameter, RecallAppendOutcome}; + +fn parameters(values: &[(&RecallValue, ColumnType)]) -> Result, ControlError> { + values + .iter() + .map(|(value, kind)| { + Ok(match (value, kind) { + (RecallValue::Null, kind) => Parameter::Null(*kind), + (RecallValue::Integer(value), ColumnType::Boolean) => { + Parameter::Boolean(*value != 0) + } + (RecallValue::Integer(value), _) => Parameter::Integer(*value), + (RecallValue::Real(value), _) => Parameter::Real(*value), + (RecallValue::Text(value), ColumnType::Json) => Parameter::Json( + serde_json::from_str(value) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ), + (RecallValue::Text(value), ColumnType::Timestamp) => Parameter::Time( + DateTime::parse_from_rfc3339(value) + .map_err(|error| ControlError::Invalid(error.to_string()))? + .to_utc(), + ), + (RecallValue::Text(value), _) => Parameter::Text(value.clone()), + (RecallValue::Blob(value), _) => Parameter::Bytes(value.clone()), + }) + }) + .collect() +} + +pub(crate) fn text<'a>(row: &'a RecallRow, name: &str) -> Result<&'a str, ControlError> { + match row.value(name) { + Some(RecallValue::Text(value)) => Ok(value), + _ => Err(ControlError::Invalid(format!("Recall {name} must be text"))), + } +} + +pub(crate) async fn write_row( + transaction: ControlTransaction, + row: &RecallRow, +) -> Result<(ControlTransaction, RecallAppendOutcome), ControlError> { + let postgres = matches!(transaction.connection, Some(Connection::Postgres(_))); + let ingested = RecallValue::Text(transaction.now.to_rfc3339_opts(SecondsFormat::Micros, true)); + let columns: HashSet = table_columns(row.table) + .into_iter() + .map(|column| column.name.to_owned()) + .collect(); + let insert = relational_insert((row, &columns), &ingested, postgres); + let returning = if row.table == Table::Events { + "storage_id" + } else { + "1 AS inserted" + }; + let (transaction, rows) = query( + transaction, + &format!("{} RETURNING {returning}", insert.sql), + parameters(&insert.values)?, + ) + .await?; + if let Some(found) = rows.first() { + return Ok(( + transaction, + RecallAppendOutcome { + inserted: true, + storage_id: found.get("storage_id").and_then(serde_json::Value::as_i64), + conflict: None, + }, + )); + } + let lookup = relational_lookup(row, postgres, true); + let (transaction, rows) = query(transaction, &lookup.sql, parameters(&lookup.values)?).await?; + let found = rows.first().ok_or_else(|| { + ControlError::Invalid("Recall duplicate has no matching stored key".into()) + })?; + let existing = found + .get("content_hash") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| ControlError::Invalid("stored Recall row has no content hash".into()))?; + Ok(( + transaction, + RecallAppendOutcome { + inserted: false, + storage_id: found.get("storage_id").and_then(serde_json::Value::as_i64), + conflict: conflict_evidence(row, existing), + }, + )) +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/recall_schema.rs b/crates/sqlbuild-orchestration/src/store/_helpers/recall_schema.rs new file mode 100644 index 0000000000..d63c9b5681 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/recall_schema.rs @@ -0,0 +1,13 @@ +//! Embed Recall in the control database so run facts and projections share one transaction. + +use sqlbuild_recall::schema::main::control_table_statements::control_table_statements; +use sqlbuild_recall::schema::main::control_view_statements::control_view_statements; + +use crate::store::models::ControlStore; + +pub(crate) fn recall_schema(store: &ControlStore) -> Vec { + let postgres = matches!(store, ControlStore::Postgres(_)); + let mut statements = control_table_statements(postgres); + statements.extend(control_view_statements(postgres)); + statements +} diff --git a/crates/sqlbuild-orchestration/src/store/errors.rs b/crates/sqlbuild-orchestration/src/store/errors.rs new file mode 100644 index 0000000000..8d552fd243 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/errors.rs @@ -0,0 +1,43 @@ +//! Control-store failures; callers must not publish a successful state after one. + +/// A rejected control-plane operation or failed durable write. +#[derive(Debug, thiserror::Error)] +pub enum ControlError { + #[error("control store: {0}")] + Database(#[from] sqlx_core::Error), + #[error("control store: {0}")] + Sqlite(#[from] rusqlite::Error), + #[error("control-store worker failed: {0}")] + Worker(#[from] tokio::task::JoinError), + #[error("control-store filesystem: {0}")] + Io(#[from] std::io::Error), + #[error("invalid control-plane data: {0}")] + Invalid(String), + #[error("control-plane operation denied: {reason}; decision {decision_id}")] + Denied { + reason: String, + decision_id: uuid::Uuid, + }, + #[error("control-store schema is unsupported: {0}")] + Schema(String), + #[error("snapshot generation is not newer than the current generation")] + StaleGeneration, + #[error("scheduler lease expired or was fenced by another holder")] + Fenced, +} + +impl ControlError { + pub(crate) fn transient_database(error: &sqlx_core::Error) -> bool { + match error { + sqlx_core::Error::Io(_) + | sqlx_core::Error::PoolTimedOut + | sqlx_core::Error::WorkerCrashed => true, + sqlx_core::Error::Database(error) => error.code().is_some_and(|code| { + code.starts_with("08") + || code.starts_with("53") + || matches!(code.as_ref(), "57P01" | "57P02" | "57P03") + }), + _ => false, + } + } +} diff --git a/crates/sqlbuild-orchestration/src/store/main/append_control_event.rs b/crates/sqlbuild-orchestration/src/store/main/append_control_event.rs new file mode 100644 index 0000000000..3143159449 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/append_control_event.rs @@ -0,0 +1,52 @@ +//! Sequence scheduler and platform facts inside the caller's serialized transaction. + +use chrono::SecondsFormat; +use serde_json::{Map, Value, json}; +use sqlbuild_recall::records::main::event_object_row::event_object_row; +use sqlbuild_recall::records::models::RowEnvelope; + +use crate::store::errors::ControlError; +use crate::store::main::append_recall::append_recall; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlTransaction, Parameter, RecallAppendOutcome}; + +pub(crate) async fn append_control_event( + transaction: ControlTransaction, + event: ControlEvent<'_>, +) -> Result<(ControlTransaction, RecallAppendOutcome), ControlError> { + let timestamp = event + .occurred_at + .to_rfc3339_opts(SecondsFormat::Micros, true); + let (transaction, rows) = query(transaction, + "SELECT invocation_sequence AS last_sequence, invocation_started_at AS started_at FROM recall_events \ + WHERE invocation_id=$1 ORDER BY invocation_sequence DESC LIMIT 1", + vec![Parameter::Text(event.invocation_id.clone())]).await?; + let previous = rows.first(); + let sequence = previous + .and_then(|row| row.get("last_sequence")) + .and_then(Value::as_i64) + .map_or(Some(0), |value| value.checked_add(1)) + .ok_or_else(|| ControlError::Invalid("run event sequence is exhausted".into()))?; + let envelope = RowEnvelope { + invocation_id: event.invocation_id, + invocation_started_at: previous + .and_then(|row| row.get("started_at")) + .and_then(Value::as_str) + .unwrap_or(×tamp) + .into(), + producer_version: env!("CARGO_PKG_VERSION").into(), + }; + let object = Map::from_iter([ + ("event_id".into(), json!(event.event_id)), + ("event_type".into(), json!(event.event_type)), + ("schema_version".into(), json!(1)), + ("occurred_at".into(), json!(timestamp)), + ("producer".into(), json!(event.producer)), + ("invocation_sequence".into(), json!(sequence)), + ("run_id".into(), json!(event.run_id)), + ("payload".into(), event.payload), + ]); + let row = event_object_row(&envelope, &object) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + append_recall(transaction, &row).await +} diff --git a/crates/sqlbuild-orchestration/src/store/main/append_recall.rs b/crates/sqlbuild-orchestration/src/store/main/append_recall.rs new file mode 100644 index 0000000000..349fe86d1a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/append_recall.rs @@ -0,0 +1,36 @@ +//! Append Recall facts and retry-conflict evidence to the caller's control transaction. + +use chrono::SecondsFormat; +use sqlbuild_recall::records::main::conflict_row::conflict_row; +use sqlbuild_recall::records::models::{RecallRow, RowEnvelope}; +use sqlbuild_recall::schema::main::table_spec::table_spec; +use sqlbuild_recall::schema::models::Table; + +use crate::store::_helpers::recall_rows::{text, write_row}; +use crate::store::errors::ControlError; +use crate::store::models::{ControlTransaction, RecallAppendOutcome}; + +pub(crate) async fn append_recall( + transaction: ControlTransaction, + row: &RecallRow, +) -> Result<(ControlTransaction, RecallAppendOutcome), ControlError> { + let (mut transaction, outcome) = write_row(transaction, row).await?; + if let Some(evidence) = &outcome.conflict + && row.table != Table::Conflicts + { + let envelope = RowEnvelope { + invocation_id: text(row, "invocation_id")?.into(), + invocation_started_at: text(row, "invocation_started_at")?.into(), + producer_version: text(row, "producer_version")?.into(), + }; + let detected_at = transaction.now.to_rfc3339_opts(SecondsFormat::Micros, true); + let conflict = conflict_row( + &envelope, + (table_spec(row.table).name, &evidence.record_key), + (&evidence.existing_sha256, &evidence.rejected_sha256), + &detected_at, + ); + (transaction, _) = write_row(transaction, &conflict).await?; + } + Ok((transaction, outcome)) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/begin.rs b/crates/sqlbuild-orchestration/src/store/main/begin.rs new file mode 100644 index 0000000000..94af142705 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/begin.rs @@ -0,0 +1,65 @@ +//! Serialize durable appends before allocating their storage IDs. + +use std::time::Duration; + +use chrono::Utc; +use rusqlite::OpenFlags; + +use crate::store::_helpers::changes::sqlite_changes; +use crate::store::errors::ControlError; +use crate::store::main::refresh_clock::refresh_clock; +use crate::store::models::{Connection, ControlStore, ControlTransaction, TransactionMode}; + +pub(crate) async fn begin( + store: &ControlStore, + mode: TransactionMode, +) -> Result { + let (connection, notifications) = match store { + ControlStore::Sqlite(path) => { + let path = path.clone(); + tokio::task::spawn_blocking(move || { + let connection = rusqlite::Connection::open_with_flags( + &path, + OpenFlags::SQLITE_OPEN_READ_WRITE + | OpenFlags::SQLITE_OPEN_CREATE + | OpenFlags::SQLITE_OPEN_NO_MUTEX, + )?; + connection.busy_timeout(Duration::from_secs(30))?; + connection.execute_batch(if mode == TransactionMode::Write { + "PRAGMA foreign_keys=ON; BEGIN IMMEDIATE" + } else { + "PRAGMA foreign_keys=ON; PRAGMA query_only=ON; BEGIN" + })?; + let notifications = if mode == TransactionMode::Write { + Some(sqlite_changes(&path)?) + } else { + None + }; + Ok::<_, ControlError>((Connection::Sqlite(connection), notifications)) + }) + .await?? + } + ControlStore::Postgres(pool) => { + let mut transaction = pool.begin().await?; + if mode == TransactionMode::Write { + sqlx_core::query::query("SELECT pg_advisory_xact_lock(1936810572, 1)") + .execute(&mut *transaction) + .await?; + } else { + sqlx_core::query::query( + "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY", + ) + .execute(&mut *transaction) + .await?; + } + (Connection::Postgres(transaction), None) + } + }; + let transaction = ControlTransaction { + connection: Some(connection), + now: Utc::now(), + changes: Default::default(), + notifications, + }; + refresh_clock(transaction).await +} diff --git a/crates/sqlbuild-orchestration/src/store/main/commit.rs b/crates/sqlbuild-orchestration/src/store/main/commit.rs new file mode 100644 index 0000000000..cc72f730ed --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/commit.rs @@ -0,0 +1,41 @@ +//! Commit facts and their projections together. + +use crate::store::_helpers::changes::resolve_changes; +use crate::store::errors::ControlError; +use crate::store::models::{Connection, ControlTransaction}; + +pub(crate) async fn commit(transaction: ControlTransaction) -> Result<(), ControlError> { + let (mut transaction, messages) = resolve_changes(transaction).await?; + match transaction + .connection + .take() + .ok_or_else(|| ControlError::Invalid("transaction has already finished".into()))? + { + Connection::Sqlite(connection) => { + tokio::task::spawn_blocking(move || { + connection.execute_batch("COMMIT")?; + if let Some(sender) = transaction.notifications { + for message in messages { + let _ = sender.send(message); + } + } + Ok::<_, rusqlite::Error>(()) + }) + .await??; + } + Connection::Postgres(mut transaction) => { + for message in messages { + let payload = serde_json::to_string(&message) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + sqlx_core::query::query( + "SELECT pg_notify('sqb_changes_' || md5(current_schema()), $1)", + ) + .bind(payload) + .execute(&mut *transaction) + .await?; + } + transaction.commit().await?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/compress.rs b/crates/sqlbuild-orchestration/src/store/main/compress.rs new file mode 100644 index 0000000000..d29c898e55 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/compress.rs @@ -0,0 +1,13 @@ +//! Shared encoding for immutable, content-addressed control-store bodies. + +use std::io::Write; + +use flate2::{Compression, write::GzEncoder}; + +use crate::store::errors::ControlError; + +pub(crate) fn compress(bytes: &[u8]) -> Result, ControlError> { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(bytes)?; + Ok(encoder.finish()?) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/decode_row.rs b/crates/sqlbuild-orchestration/src/store/main/decode_row.rs new file mode 100644 index 0000000000..bc02200fb1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/decode_row.rs @@ -0,0 +1,12 @@ +//! Decode driver-neutral typed records through the store's shared row contract. + +use serde::de::DeserializeOwned; +use serde_json::Value; + +use crate::store::errors::ControlError; +use crate::store::types::Row; + +pub(crate) fn decode_row(row: Row) -> Result { + serde_json::from_value(Value::Object(row.into_iter().collect())) + .map_err(|error| ControlError::Invalid(error.to_string())) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/decompress.rs b/crates/sqlbuild-orchestration/src/store/main/decompress.rs new file mode 100644 index 0000000000..520cd4e7d7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/decompress.rs @@ -0,0 +1,16 @@ +//! Decode immutable compressed control-store bodies. + +use std::io::Read; + +use flate2::read::GzDecoder; +use serde_json::Value; + +use crate::store::errors::ControlError; + +pub(crate) fn decompress(value: &Value) -> Result, ControlError> { + let bytes: Vec = serde_json::from_value(value.clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let mut output: Vec = Vec::new(); + GzDecoder::new(bytes.as_slice()).read_to_end(&mut output)?; + Ok(output) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/list_recall_invocations.rs b/crates/sqlbuild-orchestration/src/store/main/list_recall_invocations.rs new file mode 100644 index 0000000000..ced2c62b2c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/list_recall_invocations.rs @@ -0,0 +1,62 @@ +//! Discover invocation namespaces from retained Recall facts, including output-only attempts. + +use sqlbuild_recall::schema::constants::INVOCATION_TABLES; +use sqlbuild_recall::schema::main::table_spec::table_spec; + +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::types::Row; + +/// The caller authorizes exact namespace roots; prefix matching escapes LIKE patterns and includes the slash boundary. +pub async fn list_recall_invocations( + store: &ControlStore, + namespaces: &[String], + limit: u32, +) -> Result, ControlError> { + if namespaces.is_empty() || limit == 0 { + return Ok(Vec::new()); + } + let mut parameters: Vec = Vec::new(); + let predicates = namespaces + .iter() + .map(|namespace| { + let exact = parameters.len() + 1; + parameters.push(Parameter::Text(namespace.clone())); + let pattern = namespace + .replace('!', "!!") + .replace('%', "!%") + .replace('_', "!_"); + parameters.push(Parameter::Text(format!("{pattern}/%"))); + format!( + "invocation_id=${exact} OR invocation_id LIKE ${} ESCAPE '!'", + exact + 1 + ) + }) + .collect::>() + .join(" OR "); + let tables = INVOCATION_TABLES + .iter() + .map(|table| { + format!( + "SELECT invocation_id, invocation_started_at FROM {} WHERE {predicates}", + table_spec(*table).name, + ) + }) + .collect::>() + .join(" UNION ALL "); + parameters.push(Parameter::Integer(i64::from(limit.min(1000)))); + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + &format!( + "SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at \ + FROM ({tables}) AS records GROUP BY invocation_id ORDER BY invocation_id LIMIT ${}", + parameters.len() + ), + parameters, + ) + .await?; + commit(transaction).await?; + Ok(rows) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/listen_changes.rs b/crates/sqlbuild-orchestration/src/store/main/listen_changes.rs new file mode 100644 index 0000000000..f4a2711a54 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/listen_changes.rs @@ -0,0 +1,15 @@ +//! Establish schema-isolated PostgreSQL LISTEN before subscribing clients or declaring recovery. + +use crate::store::errors::ControlError; + +pub(crate) async fn listen_changes( + pool: &sqlx_postgres::PgPool, +) -> Result { + let channel: String = + sqlx_core::query_scalar::query_scalar("SELECT 'sqb_changes_' || md5(current_schema())") + .fetch_one(pool) + .await?; + let mut listener = sqlx_postgres::PgListener::connect_with(pool).await?; + listener.listen(&channel).await?; + Ok(listener) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs new file mode 100644 index 0000000000..973d70e069 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -0,0 +1,50 @@ +//! Schema creation under the same lock used by all other control-plane writes. + +use uuid::Uuid; + +use crate::store::_helpers::migration::{SCHEMA, render, schema_digest, verify_schema}; +use crate::store::_helpers::recall_schema::recall_schema; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Create the schema once, or reject a schema this server did not create. +pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { + let mut transaction = begin(store, TransactionMode::Write).await?; + (transaction, _) = query(transaction, &render(store, + "CREATE TABLE IF NOT EXISTS sqb_schema_version (singleton BIGINT PRIMARY KEY CHECK (singleton = 1), \ + checksum TEXT NOT NULL, applied_at {time} NOT NULL)"), vec![]).await?; + let (mut transaction, rows) = query( + transaction, + "SELECT checksum FROM sqb_schema_version", + vec![], + ) + .await?; + let digest = schema_digest(store); + verify_schema(&rows, &digest)?; + if rows.is_empty() { + for statement in recall_schema(store) { + (transaction, _) = query(transaction, &statement, vec![]).await?; + } + for statement in SCHEMA + .split(';') + .map(str::trim) + .filter(|statement| !statement.is_empty()) + { + (transaction, _) = query(transaction, &render(store, statement), vec![]).await?; + } + let now = transaction.now; + (transaction, _) = query(transaction, + "INSERT INTO sqb_installation (singleton, installation_id, created_at) VALUES (1, $1, $2)", + vec![Parameter::Id(Uuid::now_v7()), Parameter::Time(now)]).await?; + (transaction, _) = query( + transaction, + "INSERT INTO sqb_schema_version (singleton, checksum, applied_at) VALUES (1, $1, $2)", + vec![Parameter::Text(digest), Parameter::Time(now)], + ) + .await?; + } + commit(transaction).await +} diff --git a/crates/sqlbuild-orchestration/src/store/main/mod.rs b/crates/sqlbuild-orchestration/src/store/main/mod.rs new file mode 100644 index 0000000000..9796d266c4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/mod.rs @@ -0,0 +1,16 @@ +pub(crate) mod append_control_event; +pub(crate) mod append_recall; +pub(crate) mod begin; +pub(crate) mod commit; +pub(crate) mod compress; +pub(crate) mod decode_row; +pub(crate) mod decompress; +pub mod list_recall_invocations; +pub(crate) mod listen_changes; +pub mod migrate_store; +pub mod open_store; +pub(crate) mod query; +pub mod read_recall_view; +pub(crate) mod refresh_clock; +pub(crate) mod store_source_text; +pub(crate) mod subscribe_changes; diff --git a/crates/sqlbuild-orchestration/src/store/main/open_store.rs b/crates/sqlbuild-orchestration/src/store/main/open_store.rs new file mode 100644 index 0000000000..37905528d2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/open_store.rs @@ -0,0 +1,22 @@ +//! Open a control database without Python or warehouse connections. + +use std::time::Duration; + +use crate::store::_helpers::open_sqlite::{open_sqlite, sqlite_path}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +/// Open PostgreSQL by URL, or SQLite by an explicit filesystem path. +pub async fn open_store(url: &str) -> Result { + if url.starts_with("postgres://") || url.starts_with("postgresql://") { + return Ok(ControlStore::Postgres( + sqlx_postgres::PgPoolOptions::new() + .max_connections(10) + .acquire_timeout(Duration::from_secs(30)) + .connect(url) + .await?, + )); + } + let path = sqlite_path(url)?; + tokio::task::spawn_blocking(move || open_sqlite(path)).await? +} diff --git a/crates/sqlbuild-orchestration/src/store/main/query.rs b/crates/sqlbuild-orchestration/src/store/main/query.rs new file mode 100644 index 0000000000..9e79cdc7a1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/query.rs @@ -0,0 +1,35 @@ +//! Execute inside an owned transaction, without blocking an async runtime on SQLite. + +use crate::store::_helpers::database::{postgres_query, sqlite_query}; +use crate::store::errors::ControlError; +use crate::store::models::{Connection, ControlTransaction, Parameter}; +use crate::store::types::Row; + +pub(crate) async fn query( + mut transaction: ControlTransaction, + sql: &str, + parameters: Vec, +) -> Result<(ControlTransaction, Vec), ControlError> { + let connection = transaction + .connection + .take() + .ok_or_else(|| ControlError::Invalid("transaction has already finished".into()))?; + let rows: Vec = match connection { + Connection::Sqlite(connection) => { + let sql = sql.to_owned(); + let (connection, result) = tokio::task::spawn_blocking(move || { + let result = sqlite_query(&connection, &sql, ¶meters); + (connection, result) + }) + .await?; + transaction.connection = Some(Connection::Sqlite(connection)); + result? + } + Connection::Postgres(mut connection) => { + let result = postgres_query(&mut connection, sql, parameters).await; + transaction.connection = Some(Connection::Postgres(connection)); + result? + } + }; + Ok((transaction, rows)) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs b/crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs new file mode 100644 index 0000000000..a07da01efb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs @@ -0,0 +1,44 @@ +//! Only the supported Recall views can be read; no caller-provided SQL is evaluated. + +use sqlbuild_recall::schema::constants::VIEW_NAMES; + +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::types::Row; + +/// Read authorized evidence, decoding only display text while preserving byte-exact captured content. +pub async fn read_recall_view( + store: &ControlStore, + (view, invocation_id): (&str, &str), + limit: u32, +) -> Result, ControlError> { + if !VIEW_NAMES.contains(&view) { + return Err(ControlError::Invalid("unknown Recall view".into())); + } + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, mut rows) = query( + transaction, + &format!("SELECT * FROM \"{view}\" WHERE invocation_id=$1 LIMIT $2"), + vec![ + Parameter::Text(invocation_id.into()), + Parameter::Integer(i64::from(limit.min(1000))), + ], + ) + .await?; + if matches!(view, "output_tail" | "artifacts") { + for row in &mut rows { + if let Some(text) = row.get_mut("text") + && text.is_array() + { + let bytes: Vec = serde_json::from_value(text.clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + *text = serde_json::Value::String(String::from_utf8_lossy(&bytes).into_owned()); + } + } + } + commit(transaction).await?; + Ok(rows) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs b/crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs new file mode 100644 index 0000000000..0453de1ae3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs @@ -0,0 +1,26 @@ +//! Refresh the authoritative database clock after taking transaction locks. + +use chrono::DateTime; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{Connection, ControlTransaction}; + +pub(crate) async fn refresh_clock( + transaction: ControlTransaction, +) -> Result { + let sql = match transaction.connection { + Some(Connection::Postgres(_)) => "SELECT clock_timestamp() AS now", + _ => "SELECT strftime('%Y-%m-%dT%H:%M:%f000Z','now') AS now", + }; + let (mut transaction, rows) = query(transaction, sql, vec![]).await?; + let now = rows + .first() + .and_then(|row| row.get("now")) + .and_then(|value| value.as_str()) + .ok_or_else(|| ControlError::Invalid("database clock did not return a timestamp".into()))?; + transaction.now = DateTime::parse_from_rfc3339(now) + .map_err(|error| ControlError::Invalid(error.to_string()))? + .to_utc(); + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/store_source_text.rs b/crates/sqlbuild-orchestration/src/store/main/store_source_text.rs new file mode 100644 index 0000000000..3ea443c604 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/store_source_text.rs @@ -0,0 +1,18 @@ +//! Snapshot sources and warehouse definitions share exact-content-addressed immutable bodies. + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn store_source_text( + transaction: ControlTransaction, + (hash, body): (String, Vec), +) -> Result { + let (transaction, _) = query( + transaction, + "INSERT INTO sqb_source_texts (text_hash, body) VALUES ($1, $2) ON CONFLICT DO NOTHING", + vec![Parameter::Text(hash), Parameter::Bytes(body)], + ) + .await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/subscribe_changes.rs b/crates/sqlbuild-orchestration/src/store/main/subscribe_changes.rs new file mode 100644 index 0000000000..1513d649ce --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/subscribe_changes.rs @@ -0,0 +1,33 @@ +//! SQLite shares an in-process channel; PostgreSQL listens across independent server replicas. + +use tokio::sync::broadcast; + +use crate::store::_helpers::changes::sqlite_changes; +use crate::store::errors::ControlError; +use crate::store::main::listen_changes::listen_changes; +use crate::store::models::{ChangeMessage, ControlStore}; + +pub(crate) async fn subscribe_changes( + store: &ControlStore, +) -> Result< + ( + broadcast::Sender, + Option, + ), + ControlError, +> { + match store { + ControlStore::Sqlite(path) => { + let path = path.clone(); + Ok(( + tokio::task::spawn_blocking(move || sqlite_changes(&path)).await??, + None, + )) + } + ControlStore::Postgres(pool) => { + let listener = listen_changes(pool).await?; + let (sender, _) = broadcast::channel(256); + Ok((sender, Some(listener))) + } + } +} diff --git a/crates/sqlbuild-orchestration/src/store/mod.rs b/crates/sqlbuild-orchestration/src/store/mod.rs new file mode 100644 index 0000000000..1228df7ae6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod _helpers; +pub mod errors; +pub mod main; +pub mod models; +pub(crate) mod types; + +#[cfg(test)] +#[path = "../../tests/store/test_contracts.rs"] +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/store/models.rs b/crates/sqlbuild-orchestration/src/store/models.rs new file mode 100644 index 0000000000..4f1ddeb631 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/models.rs @@ -0,0 +1,113 @@ +//! Native database handles and bound values shared by control-plane operations. + +use std::collections::BTreeSet; +use std::path::PathBuf; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sqlbuild_recall::schema::models::ColumnType; +use sqlbuild_recall::store::models::ConflictEvidence; +use tokio::sync::broadcast; +use uuid::Uuid; + +/// An installation's control store. Workers and locations never receive this handle. +#[derive(Clone, Debug)] +pub enum ControlStore { + Sqlite(PathBuf), + Postgres(sqlx_postgres::PgPool), +} + +/// The first copy wins; duplicate events retain their original storage cursor. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RecallAppendOutcome { + pub inserted: bool, + pub storage_id: Option, + pub conflict: Option, +} + +/// A control-plane fact shares Recall's durable sequencing and conflict handling. +#[derive(Debug)] +pub(crate) struct ControlEvent<'a> { + pub(crate) event_id: Uuid, + pub(crate) invocation_id: String, + pub(crate) event_type: &'a str, + pub(crate) producer: &'static str, + pub(crate) occurred_at: DateTime, + pub(crate) run_id: Option, + pub(crate) payload: serde_json::Value, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum TransactionMode { + Read, + Write, +} + +/// An owned transaction; abandoning it rolls back, including when an async task is cancelled. +#[derive(Debug)] +pub(crate) struct ControlTransaction { + pub(crate) connection: Option, + pub(crate) now: DateTime, + pub(crate) changes: BTreeSet, + pub(crate) notifications: Option>, +} + +#[derive(Debug)] +pub(crate) enum Connection { + Sqlite(rusqlite::Connection), + Postgres(sqlx_core::transaction::Transaction<'static, sqlx_postgres::Postgres>), +} + +/// Transaction-local invalidations, not another durable lifecycle or event log. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum Change { + Run(Uuid), + Trigger(Uuid), + Snapshot(Uuid, String), + NodeStatus(Uuid, String), + Credentials, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum ChangeTopic { + Runs, + Run(Uuid), + Ticks, + Snapshot, + NodeStatus, +} + +/// Fixed-size routing payloads fit PostgreSQL NOTIFY even with long target labels. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum ChangeMessage { + Changed { + location_id: Uuid, + target_hash: String, + topic: ChangeTopic, + }, + CredentialsChanged, + Resync, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct ChangedScope { + pub(crate) location_id: Uuid, + pub(crate) target: String, +} + +/// Parameters keep their PostgreSQL types; SQLite uses their corresponding storage classes. +#[derive(Clone, Debug)] +pub(crate) enum Parameter { + Null(ColumnType), + Text(String), + OptionalText(Option), + Bytes(Vec), + Integer(i64), + Real(f64), + Json(serde_json::Value), + Boolean(bool), + Id(Uuid), + OptionalId(Option), + Time(DateTime), + OptionalTime(Option>), +} diff --git a/crates/sqlbuild-orchestration/src/store/schema.sql b/crates/sqlbuild-orchestration/src/store/schema.sql new file mode 100644 index 0000000000..116abb7d2b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/schema.sql @@ -0,0 +1,541 @@ +CREATE TABLE sqb_installation ( + singleton BIGINT PRIMARY KEY CHECK (singleton = 1), + installation_id {uuid} NOT NULL, + created_at {time} NOT NULL +); +CREATE TABLE sqb_locations ( + location_id {uuid} PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + created_by TEXT NOT NULL, + created_at {time} NOT NULL, + archived_at {time} +); +CREATE TABLE sqb_source_texts ( + text_hash TEXT PRIMARY KEY, + body {binary} NOT NULL +); +CREATE TABLE sqb_node_definitions ( + node_hash TEXT PRIMARY KEY, + body {binary} NOT NULL +); +CREATE TABLE sqb_documents ( + document_hash TEXT PRIMARY KEY, + body {binary} NOT NULL +); +CREATE TABLE sqb_snapshots ( + snapshot_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + generation BIGINT NOT NULL CHECK (generation > 0), + git_revision TEXT, + git_dirty BOOLEAN NOT NULL, + git_author TEXT, + image_id TEXT, + environment_hash TEXT NOT NULL, + sqb_version TEXT NOT NULL, + snapshot_schema_version BIGINT NOT NULL, + manifest_digest TEXT NOT NULL, + published_at {time} NOT NULL, + UNIQUE (location_id, target, generation), + UNIQUE (location_id, target, snapshot_id) +); +CREATE INDEX sqb_snapshots_location_time ON sqb_snapshots(location_id, target, published_at); +CREATE TABLE sqb_snapshot_nodes ( + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + node_key TEXT NOT NULL, + node_kind TEXT NOT NULL, + node_hash TEXT NOT NULL REFERENCES sqb_node_definitions(node_hash), + PRIMARY KEY (snapshot_id, node_key) +); +CREATE TABLE sqb_snapshot_sources ( + snapshot_id {uuid} NOT NULL, + node_key TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('raw_sql', 'compiled_sql', 'python_source')), + text_hash TEXT NOT NULL REFERENCES sqb_source_texts(text_hash), + PRIMARY KEY (snapshot_id, node_key, kind), + FOREIGN KEY (snapshot_id, node_key) REFERENCES sqb_snapshot_nodes(snapshot_id, node_key) +); +CREATE TABLE sqb_snapshot_documents ( + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + kind TEXT NOT NULL CHECK (kind IN ('jobs', 'groups', 'triggers', 'roles', 'lineage', 'columns', 'providers')), + document_hash TEXT NOT NULL REFERENCES sqb_documents(document_hash), + PRIMARY KEY (snapshot_id, kind) +); +CREATE TABLE sqb_current_snapshots ( + location_id {uuid} NOT NULL, + target TEXT NOT NULL, + snapshot_id {uuid} NOT NULL, + since {time} NOT NULL, + PRIMARY KEY (location_id, target), + FOREIGN KEY (location_id, target, snapshot_id) + REFERENCES sqb_snapshots(location_id, target, snapshot_id) +); +CREATE TABLE sqb_snapshot_changes ( + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + change_seq BIGINT NOT NULL, + change_kind TEXT NOT NULL CHECK (change_kind IN ('added', 'changed', 'removed')), + subject_key TEXT NOT NULL, + before_hash TEXT, + after_hash TEXT, + PRIMARY KEY (snapshot_id, change_seq) +); +CREATE TABLE sqb_scheduler_lease ( + name TEXT PRIMARY KEY CHECK (name = 'scheduler'), + holder {uuid} NOT NULL, + epoch BIGINT NOT NULL CHECK (epoch > 0), + expires_at {time} NOT NULL +); +CREATE TABLE sqb_jobs ( + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + job_key TEXT NOT NULL, + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + kind TEXT NOT NULL CHECK (kind IN ('job', 'group')), + definition_json {json} NOT NULL, + PRIMARY KEY (location_id, target, job_key) +); +CREATE TABLE sqb_triggers ( + trigger_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('schedule', 'sensor')), + name TEXT NOT NULL, + job_key TEXT NOT NULL, + definition_json {json} NOT NULL, + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + removed_in_snapshot_id {uuid} REFERENCES sqb_snapshots(snapshot_id), + UNIQUE (location_id, target, kind, name) +); +CREATE TABLE sqb_trigger_state ( + trigger_id {uuid} PRIMARY KEY REFERENCES sqb_triggers(trigger_id), + status TEXT NOT NULL CHECK (status IN ('running', 'paused')), + status_override TEXT CHECK (status_override IN ('running', 'paused')), + started_at {time} NOT NULL, + cursor_json {json}, + last_tick_for {time}, + next_due_at {time}, + updated_at {time} NOT NULL +); +CREATE INDEX sqb_trigger_state_due ON sqb_trigger_state(next_due_at) WHERE status = 'running'; +CREATE TABLE sqb_ticks ( + tick_id {uuid} PRIMARY KEY, + trigger_id {uuid} NOT NULL REFERENCES sqb_triggers(trigger_id), + scheduled_for {time} NOT NULL, + evaluated_at {time} NOT NULL, + epoch BIGINT NOT NULL, + outcome TEXT NOT NULL CHECK (outcome IN ('requested', 'covered')), + run_id {uuid}, + covered_by_run_id {uuid}, + UNIQUE (trigger_id, scheduled_for) +); +CREATE INDEX sqb_ticks_trigger_time ON sqb_ticks(trigger_id, scheduled_for); +CREATE TABLE sqb_runs ( + run_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + image_id TEXT, + kind TEXT NOT NULL CHECK (kind IN ('job', 'group', 'adhoc')), + origin TEXT NOT NULL CHECK (origin IN ('manual', 'schedule', 'sensor', 'rerun', 'backfill')), + job_key TEXT, + parent_run_id {uuid} REFERENCES sqb_runs(run_id), + root_run_id {uuid} NOT NULL, + trigger_id {uuid} REFERENCES sqb_triggers(trigger_id), + tick_id {uuid} REFERENCES sqb_ticks(tick_id), + rerun_of_run_id {uuid} REFERENCES sqb_runs(run_id), + requested_by TEXT NOT NULL, + priority BIGINT NOT NULL DEFAULT 0, + selection_json {json} NOT NULL, + request_json {json} NOT NULL, + authored_json {json}, + write_set_digest TEXT NOT NULL, + rewrite_history BOOLEAN NOT NULL DEFAULT FALSE, + status TEXT NOT NULL CHECK (status IN ('queued', 'launched', 'running', 'unresponsive', 'success', 'warn', 'partial', 'failed', 'cancelled')), + status_reason TEXT, + queued_at {time} NOT NULL, + launched_at {time}, + started_at {time}, + ended_at {time}, + cancel_requested_at {time}, + node_counts_json {json} NOT NULL, + last_event_storage_id BIGINT NOT NULL, + worker_instance {uuid}, + container_id TEXT, + claimed_at {time}, + last_heartbeat_at {time} +); +CREATE INDEX sqb_runs_location_time ON sqb_runs(location_id, target, queued_at); +CREATE INDEX sqb_runs_queue ON sqb_runs(priority DESC, queued_at) WHERE status = 'queued'; +CREATE INDEX sqb_runs_active ON sqb_runs(location_id, target, job_key) WHERE status IN ('queued', 'launched', 'running', 'unresponsive'); +CREATE TABLE sqb_run_nodes ( + run_id {uuid} NOT NULL REFERENCES sqb_runs(run_id), + node_key TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending', 'running', 'success', 'warn', 'failed', 'skipped', 'cancelled')), + writes BOOLEAN NOT NULL, + started_at {time}, + ended_at {time}, + rows_affected BIGINT, + error_code TEXT, + skipped_because_of TEXT, + kind TEXT NOT NULL DEFAULT 'node' CHECK (kind IN ('node', 'check')), + finished_storage_id BIGINT, + PRIMARY KEY (run_id, node_key) +); +CREATE INDEX sqb_run_nodes_node ON sqb_run_nodes(node_key, run_id); +CREATE INDEX sqb_run_nodes_finished ON sqb_run_nodes(finished_storage_id) +WHERE finished_storage_id IS NOT NULL; +CREATE INDEX sqb_run_nodes_decisive ON sqb_run_nodes(node_key, finished_storage_id) +WHERE finished_storage_id IS NOT NULL AND status IN ('success', 'failed'); +CREATE TABLE sqb_write_claims ( + target TEXT NOT NULL, + node_key TEXT NOT NULL, + run_id {uuid} NOT NULL REFERENCES sqb_runs(run_id), + claimed_at {time} NOT NULL, + PRIMARY KEY (target, node_key) +); +CREATE INDEX sqb_write_claims_run ON sqb_write_claims(run_id); +CREATE TABLE sqb_concurrency_slots ( + group_key TEXT NOT NULL, + slot BIGINT NOT NULL CHECK (slot > 0), + run_id {uuid} NOT NULL REFERENCES sqb_runs(run_id), + claimed_at {time} NOT NULL, + PRIMARY KEY (group_key, slot), + UNIQUE (group_key, run_id) +); +CREATE INDEX sqb_concurrency_slots_run ON sqb_concurrency_slots(run_id); +CREATE TABLE sqb_principals ( + principal_id {uuid} PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + created_at {time} NOT NULL, + disabled_at {time} +); +CREATE TABLE sqb_access_tokens ( + token_id {uuid} PRIMARY KEY, + secret_hash TEXT NOT NULL, + location_id {uuid} REFERENCES sqb_locations(location_id), + run_id {uuid} REFERENCES sqb_runs(run_id), + claim_id {uuid}, + scope_json {json} NOT NULL, + created_at {time} NOT NULL, + expires_at {time} NOT NULL, + revoked_at {time}, + CHECK ( + (location_id IS NOT NULL AND run_id IS NULL AND claim_id IS NULL) + OR (location_id IS NULL AND run_id IS NOT NULL AND claim_id IS NOT NULL) + ) +); +CREATE INDEX sqb_access_tokens_location ON sqb_access_tokens(location_id); +CREATE INDEX sqb_access_tokens_run ON sqb_access_tokens(run_id); +CREATE TABLE sqb_security_audit ( + event_id {uuid} PRIMARY KEY, + occurred_at {time} NOT NULL, + actor TEXT NOT NULL, + action TEXT NOT NULL, + granting_role TEXT NOT NULL, + request_id {uuid} NOT NULL, + ip_address TEXT, + scope_json {json} NOT NULL, + before_json {json} NOT NULL, + after_json {json} NOT NULL, + storage_id BIGINT +); +CREATE INDEX sqb_security_audit_time ON sqb_security_audit(occurred_at, event_id); +CREATE INDEX sqb_security_audit_trigger ON sqb_security_audit((scope_json ->> 'trigger_id'), occurred_at, event_id) +WHERE action = 'trigger.changed'; +CREATE TABLE sqb_location_instances ( + instance_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + connected_at {time} NOT NULL, + last_seen_at {time} NOT NULL, + health_json {json} NOT NULL +); +CREATE INDEX sqb_location_instances_location_seen ON sqb_location_instances(location_id, last_seen_at); +CREATE TABLE sqb_location_work ( + request_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + generation BIGINT NOT NULL, + request_json {json} NOT NULL, + requested_by TEXT NOT NULL, + requested_at {time} NOT NULL, + expires_at {time} NOT NULL, + claim_id {uuid}, + session_id {uuid} REFERENCES sqb_location_instances(instance_id), + claimed_until {time}, + result_json {json}, + completed_at {time} +); +CREATE INDEX sqb_location_work_pending ON sqb_location_work(location_id, target, requested_at); +CREATE TABLE sqb_platform_sink_deliveries ( + sink_id TEXT NOT NULL, + storage_id BIGINT NOT NULL, + event_id TEXT NOT NULL, + delivered_at {time} NOT NULL, + PRIMARY KEY (sink_id, storage_id) +); +CREATE TABLE sqb_sensor_evaluation_events ( + event_id {uuid} PRIMARY KEY, + evaluation_id {uuid} NOT NULL, + trigger_id {uuid} NOT NULL REFERENCES sqb_triggers(trigger_id), + phase TEXT NOT NULL CHECK (phase IN ('requested', 'completed', 'failed', 'timed_out', 'abandoned')), + occurred_at {time} NOT NULL, + fact_json {json} NOT NULL, + UNIQUE (evaluation_id, phase) +); +CREATE INDEX sqb_sensor_evaluation_history ON sqb_sensor_evaluation_events(trigger_id, phase, occurred_at, evaluation_id); +CREATE INDEX sqb_sensor_evaluation_expiry ON sqb_sensor_evaluation_events(occurred_at); +CREATE TABLE sqb_run_requests ( + trigger_id {uuid} NOT NULL REFERENCES sqb_triggers(trigger_id), + run_key TEXT NOT NULL, + evaluation_id {uuid} NOT NULL, + run_id {uuid} NOT NULL REFERENCES sqb_runs(run_id), + PRIMARY KEY (trigger_id, run_key) +); +CREATE INDEX sqb_runs_terminal_cursor ON sqb_runs(location_id, target, job_key, last_event_storage_id) +WHERE status IN ('success', 'warn', 'partial', 'failed', 'cancelled'); +CREATE INDEX sqb_runs_parent ON sqb_runs(parent_run_id) WHERE parent_run_id IS NOT NULL; +CREATE TABLE sqb_warehouse_freshness ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at {time} NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL, + state_database TEXT, + state_schema TEXT, + scan_id {uuid} +); +CREATE INDEX sqb_warehouse_freshness_scope ON sqb_warehouse_freshness (location_id, target, observed_at, event_id); +CREATE TABLE sqb_warehouse_fingerprints ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at {time} NOT NULL, + definition_text_hash TEXT NOT NULL REFERENCES sqb_source_texts(text_hash), + record_json {json} NOT NULL, + received_at {time} NOT NULL, + state_database TEXT, + state_schema TEXT, + node_type TEXT, + node_name TEXT, + target_database TEXT, + target_schema TEXT, + target_name TEXT, + scan_id {uuid} +); +CREATE INDEX sqb_warehouse_fingerprints_scope ON sqb_warehouse_fingerprints (location_id, target, observed_at, event_id); +CREATE TABLE sqb_development_locations ( + project_path TEXT NOT NULL, + target TEXT NOT NULL, + location_id {uuid} NOT NULL UNIQUE REFERENCES sqb_locations(location_id), + PRIMARY KEY (project_path, target) +); +CREATE TABLE sqb_password_credentials ( + principal_id {uuid} PRIMARY KEY REFERENCES sqb_principals(principal_id), + credential_id {uuid} NOT NULL UNIQUE, + password_hash TEXT NOT NULL, + changed_at {time} NOT NULL +); +CREATE TABLE sqb_sessions ( + session_id {uuid} PRIMARY KEY, + principal_id {uuid} NOT NULL REFERENCES sqb_principals(principal_id), + credential_id {uuid}, + secret_hash TEXT NOT NULL, + csrf_hash TEXT NOT NULL, + created_at {time} NOT NULL, + expires_at {time} NOT NULL, + revoked_at {time} +); +CREATE INDEX sqb_sessions_person ON sqb_sessions(principal_id); +CREATE TABLE sqb_password_login_attempts ( + attempt_id {uuid} PRIMARY KEY, + identity_key TEXT NOT NULL, + peer_address TEXT NOT NULL, + attempted_at {time} NOT NULL +); +CREATE INDEX sqb_password_login_user_time ON sqb_password_login_attempts(identity_key, attempted_at); +CREATE INDEX sqb_password_login_peer_time ON sqb_password_login_attempts(peer_address, attempted_at); +CREATE INDEX sqb_password_login_time ON sqb_password_login_attempts(attempted_at); +CREATE TABLE sqb_oidc_login_flows ( + flow_id {uuid} PRIMARY KEY, + configuration_id TEXT NOT NULL, + state_hash TEXT NOT NULL UNIQUE, + browser_hash TEXT NOT NULL, + nonce TEXT NOT NULL, + pkce_verifier TEXT NOT NULL, + expires_at {time} NOT NULL +); +CREATE INDEX sqb_oidc_login_flows_expiry ON sqb_oidc_login_flows(expires_at); +CREATE TABLE sqb_role_assignments ( + assignment_id {uuid} PRIMARY KEY, + principal_id {uuid} NOT NULL REFERENCES sqb_principals(principal_id), + role_key TEXT NOT NULL, + location_id {uuid} REFERENCES sqb_locations(location_id), + target TEXT, + active_key TEXT NOT NULL UNIQUE +); +CREATE INDEX sqb_role_assignments_principal ON sqb_role_assignments(principal_id); +CREATE INDEX sqb_fingerprints_namespace ON sqb_warehouse_fingerprints (location_id,target,state_database,state_schema,observed_at,event_id); +CREATE INDEX sqb_freshness_namespace ON sqb_warehouse_freshness (location_id,target,state_database,state_schema,observed_at,event_id); +CREATE UNIQUE INDEX sqb_security_audit_storage_id ON sqb_security_audit(storage_id); +CREATE TABLE sqb_warehouse_audit_results ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at {time} NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + audit_name TEXT NOT NULL, + outcome TEXT NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL, + scan_id {uuid} +); +CREATE INDEX sqb_audit_results_namespace ON sqb_warehouse_audit_results (location_id,target,state_database,state_schema,observed_at,event_id); +CREATE TABLE sqb_warehouse_janitor_events ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at {time} NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + event_type TEXT NOT NULL, + relation_database TEXT, + relation_schema TEXT NOT NULL, + archive_name TEXT NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL, + scan_id {uuid} +); +CREATE INDEX sqb_janitor_events_namespace ON sqb_warehouse_janitor_events (location_id,target,state_database,state_schema,observed_at,event_id); +CREATE TABLE sqb_warehouse_migrations ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at {time} NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + target_name TEXT, + destination_model TEXT NOT NULL, + decision TEXT NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL, + scan_id {uuid} +); +CREATE INDEX sqb_migrations_namespace ON sqb_warehouse_migrations (location_id,target,state_database,state_schema,observed_at,event_id); +CREATE TABLE sqb_warehouse_old_name_views ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at {time} NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + target_name TEXT, + migration_event_id TEXT NOT NULL, + destination_model TEXT NOT NULL, + event_type TEXT NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL, + scan_id {uuid} +); +CREATE INDEX sqb_old_name_views_namespace ON sqb_warehouse_old_name_views (location_id,target,state_database,state_schema,observed_at,event_id); +CREATE TABLE sqb_warehouse_microbatches ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at {time} NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + model_name TEXT NOT NULL, + target_name TEXT NOT NULL, + scope_kind TEXT NOT NULL, + scope_key TEXT NOT NULL, + physical_generation_id TEXT NOT NULL, + record_type TEXT NOT NULL, + origin_run_id TEXT NOT NULL, + execution_run_id TEXT NOT NULL, + replay_requirement_id TEXT, + record_json {json} NOT NULL, + received_at {time} NOT NULL, + scan_id {uuid} +); +CREATE INDEX sqb_microbatches_namespace ON sqb_warehouse_microbatches (location_id,target,state_database,state_schema,observed_at,event_id); +CREATE INDEX sqb_freshness_source_latest ON sqb_warehouse_freshness ( + location_id, target, (record_json ->> 'source_name'), + (record_json ->> 'target_database'), (record_json ->> 'target_schema'), + (record_json ->> 'target_name'), observed_at DESC, event_id DESC +); +CREATE INDEX sqb_freshness_source_received ON sqb_warehouse_freshness ( + location_id, target, (record_json ->> 'source_name'), + (record_json ->> 'target_database'), (record_json ->> 'target_schema'), + (record_json ->> 'target_name'), received_at DESC, observed_at DESC, event_id DESC +); +CREATE TABLE sqb_security_audit_deliveries ( + sink_id TEXT NOT NULL, + storage_id BIGINT NOT NULL, + event_id TEXT NOT NULL, + delivered_at {time} NOT NULL, + PRIMARY KEY (sink_id, storage_id) +); +CREATE TABLE sqb_warehouse_catalog ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + observed_at {time} NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL, + scan_id {uuid} +); +CREATE INDEX sqb_warehouse_catalog_namespace ON sqb_warehouse_catalog + (location_id, target, state_schema, state_database, observed_at, event_id); +CREATE INDEX sqb_fingerprint_node_latest ON sqb_warehouse_fingerprints + (location_id,target,node_type,node_name,target_database,target_schema,target_name,observed_at DESC,event_id DESC); +CREATE TABLE sqb_warehouse_statistics ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + relation_name TEXT NOT NULL, + observed_at {time} NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL +); +CREATE INDEX sqb_statistics_relation_time ON sqb_warehouse_statistics + (location_id,target,state_schema,state_database,relation_name,observed_at,event_id); +CREATE INDEX sqb_statistics_expiration ON sqb_warehouse_statistics (observed_at,event_id); +CREATE INDEX sqb_catalog_statistics_scan ON sqb_warehouse_catalog + (location_id,target,state_schema,state_database,event_id); +CREATE TABLE sqb_cli_login_flows ( + flow_id {uuid} PRIMARY KEY, + client_hash TEXT NOT NULL, + browser_hash TEXT NOT NULL, + expires_at {time} NOT NULL, + principal_id {uuid}, + session_id {uuid} +); +CREATE INDEX sqb_cli_login_expiry ON sqb_cli_login_flows(expires_at); +CREATE INDEX IF NOT EXISTS sqb_runs_tick ON sqb_runs(tick_id); +CREATE INDEX IF NOT EXISTS sqb_ticks_expiry ON sqb_ticks(scheduled_for) WHERE outcome = 'covered'; +CREATE INDEX IF NOT EXISTS sqb_runs_snapshot ON sqb_runs(snapshot_id); +CREATE INDEX IF NOT EXISTS sqb_snapshot_nodes_hash ON sqb_snapshot_nodes(node_hash); +CREATE INDEX IF NOT EXISTS sqb_snapshot_sources_hash ON sqb_snapshot_sources(text_hash); +CREATE INDEX IF NOT EXISTS sqb_snapshot_documents_hash ON sqb_snapshot_documents(document_hash); +CREATE TABLE sqb_effective_roles ( + location_id {uuid} NOT NULL, + target TEXT NOT NULL, + role_key TEXT NOT NULL, + resolved_grants_json TEXT NOT NULL, + PRIMARY KEY (location_id, target, role_key) +); diff --git a/crates/sqlbuild-orchestration/src/store/types.rs b/crates/sqlbuild-orchestration/src/store/types.rs new file mode 100644 index 0000000000..c1df9d2d41 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/types.rs @@ -0,0 +1,3 @@ +//! Internal result rows use native JSON scalars, with UUIDs and timestamps as strings. + +pub(crate) type Row = std::collections::BTreeMap; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog.rs new file mode 100644 index 0000000000..90fa97bdfe --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog.rs @@ -0,0 +1,138 @@ +//! The current catalog projection and its paged namespace reads. + +use std::collections::BTreeSet; + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; +use crate::warehouse_metadata::_helpers::ingestion::{outside_scan, validate_fields}; +use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use crate::warehouse_metadata::models::{ + CatalogObservation, CatalogPage, CatalogQuery, MetadataBatch, WarehouseCatalogRecord, + WarehouseMirrorCursor, +}; + +pub(crate) fn validate_catalog( + batch: &MetadataBatch, + records: &[WarehouseCatalogRecord], +) -> Result<(), ControlError> { + let mut names: BTreeSet<&String> = BTreeSet::new(); + validate_fields( + batch, + records.iter().any(|record| { + record.name.trim().is_empty() + || record.relation_type.trim().is_empty() + || !names.insert(&record.name) + }), + batch.scan.is_none() + || outside_scan( + batch, + records + .iter() + .map(|record| (&record.state_database, &record.state_schema)), + ), + )?; + for record in records { + if record + .columns + .iter() + .any(|column| column.name.trim().is_empty() || column.data_type.trim().is_empty()) + { + return Err(ControlError::Invalid( + "catalog columns need nonempty names and types".into(), + )); + } + } + Ok(()) +} + +pub(crate) async fn project_catalog_record( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), + relation_id: Uuid, + record: &WarehouseCatalogRecord, +) -> Result { + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_warehouse_catalog (event_id,location_id,target,state_database,state_schema,observed_at,record_json,received_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$6) ON CONFLICT (event_id) DO UPDATE SET \ + observed_at=excluded.observed_at,record_json=excluded.record_json,received_at=excluded.received_at", + vec![Parameter::Id(relation_id), Parameter::Id(location_id), Parameter::Text(target.into()), + Parameter::OptionalText(record.state_database.clone()), Parameter::Text(record.state_schema.clone()), + Parameter::Time(now), Parameter::Json(json!(record))]).await?; + Ok(transaction) +} + +pub(crate) async fn read_catalog_page( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), + options: &CatalogQuery, +) -> Result<(ControlTransaction, CatalogPage), ControlError> { + if options.schema.trim().is_empty() + || options.after_observed_at.is_some() != options.after_event_id.is_some() + { + return Err(ControlError::Invalid( + "catalog reads require a namespace and both cursor fields or neither".into(), + )); + } + let limit = options + .limit + .unwrap_or(100) + .clamp(1, MAX_HISTORY_RECORDS as u32) as usize; + let mut parameters = vec![ + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::Text(options.schema.clone()), + ]; + let database = if let Some(database) = &options.database { + parameters.push(Parameter::Text(database.clone())); + "state_database=$4" + } else { + "state_database IS NULL" + }; + let mut after = String::new(); + if let (Some(time), Some(id)) = (options.after_observed_at, options.after_event_id) { + let offset = parameters.len(); + parameters.extend([Parameter::Time(time), Parameter::Id(id)]); + after = format!( + " AND (observed_at,event_id)>(${},${})", + offset + 1, + offset + 2 + ); + } + parameters.push(Parameter::Integer(limit as i64)); + let (transaction, rows) = query( + transaction, + &format!( + "SELECT event_id,observed_at,record_json AS record FROM sqb_warehouse_catalog \ + WHERE location_id=$1 AND target=$2 AND state_schema=$3 AND {database}{after} \ + ORDER BY observed_at,event_id LIMIT ${}", + parameters.len() + ), + parameters, + ) + .await?; + let more = rows.len() == limit; + let records: Vec = rows + .into_iter() + .map(|row| serde_json::from_value(json!(row))) + .collect::>() + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let next_after = more + .then(|| records.last()) + .flatten() + .map(|record| WarehouseMirrorCursor { + observed_at: record.observed_at, + event_id: record.event_id, + }); + Ok(( + transaction, + CatalogPage { + records, + next_after, + }, + )) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs new file mode 100644 index 0000000000..caed8c892d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/events.rs @@ -0,0 +1,96 @@ +//! One fenced, atomic persistence path for immutable warehouse events with typed filter columns. + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::ingestion::{ + begin_ingestion, commit_ingestion, outside_scan, scoped_event_id, validate_fields, +}; +use crate::warehouse_metadata::models::{MetadataBatch, StateEventImport, WarehouseSyncResponse}; + +pub(crate) async fn ingest_events( + store: &ControlStore, + (location_id, target): (Uuid, &str), + batch: &MetadataBatch, + (records, invalid_identity): (Vec, bool), +) -> Result { + validate_fields( + batch, + invalid_identity, + outside_scan( + batch, + records + .iter() + .map(|record| (&record.location.database, &record.location.schema)), + ), + )?; + let table = batch.kind.mirror_table(); + let source_table = format!("_sqlbuild_{}", batch.kind.as_str()); + let mut transaction = begin_ingestion( + store, + (location_id, target), + (batch.session_id, batch.generation), + ) + .await?; + let mut applied = false; + let mut keys: Vec = Vec::new(); + for record in records { + let event_id = scoped_event_id( + (location_id, target), + ( + record.location.database.as_deref(), + &record.location.schema, + &source_table, + ), + &record.source_id, + )?; + keys.push(event_id); + let mut columns = vec![ + "event_id", + "location_id", + "target", + "source_event_id", + "observed_at", + "state_database", + "state_schema", + ]; + let mut parameters = vec![ + Parameter::Id(event_id), + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::Text(record.source_id), + Parameter::Time(record.observed_at), + Parameter::OptionalText(record.location.database), + Parameter::Text(record.location.schema), + ]; + for (column, value) in record.columns { + columns.push(column); + parameters.push(value); + } + columns.extend(["record_json", "received_at"]); + parameters.extend([ + Parameter::Json(record.record_json), + Parameter::Time(transaction.now), + ]); + let placeholders = (1..=parameters.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + let sql = format!( + "INSERT INTO {table} ({}) VALUES ({placeholders}) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", + columns.join(",") + ); + let inserted; + (transaction, inserted) = query(transaction, &sql, parameters).await?; + applied |= !inserted.is_empty(); + } + commit_ingestion( + transaction, + (table, location_id, target), + (batch.scan.as_ref(), &keys), + applied, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprint_reads.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprint_reads.rs new file mode 100644 index 0000000000..dd3bd789e8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprint_reads.rs @@ -0,0 +1,56 @@ +//! One indexed exact-node/destination read, shared by evidence and definition comparison. + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; +use crate::warehouse_metadata::models::{FingerprintObservation, FingerprintQuery}; + +pub(crate) async fn latest_fingerprint( + transaction: ControlTransaction, + (location_id, target, options): (Uuid, &str, &FingerprintQuery), +) -> Result<(ControlTransaction, Option<(FingerprintObservation, String)>), ControlError> { + if options.node_type.trim().is_empty() || options.node_name.trim().is_empty() { + return Err(ControlError::Invalid( + "fingerprint reads require an exact node".into(), + )); + } + let mut parameters = vec![ + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::Text(options.node_type.clone()), + Parameter::Text(options.node_name.clone()), + ]; + let mut predicates: Vec = Vec::new(); + for (column, value) in [ + ("target_database", &options.database), + ("target_schema", &options.schema), + ("target_name", &options.name), + ] { + predicates.push(if let Some(value) = value { + parameters.push(Parameter::Text(value.clone())); + format!("{column}=${}", parameters.len()) + } else { + format!("{column} IS NULL") + }); + } + let (transaction, rows) = query(transaction, + &format!("SELECT event_id,observed_at,source_event_id,record_json ->> 'run_id' AS run_id, \ + record_json ->> 'definition_hash' AS definition_hash,record_json ->> 'version_hash' AS version_hash, \ + record_json ->> 'schema_fingerprint' AS schema_fingerprint,record_json ->> 'metadata_json' AS metadata_json FROM sqb_warehouse_fingerprints \ + WHERE location_id=$1 AND target=$2 AND node_type=$3 AND node_name=$4 AND {} \ + ORDER BY target_database,target_schema,target_name,observed_at DESC,event_id DESC LIMIT 1", predicates.join(" AND ")), parameters).await?; + let evidence = rows + .into_iter() + .next() + .map(|mut row| { + let metadata = row + .remove("metadata_json") + .and_then(|value| value.as_str().map(str::to_owned)) + .unwrap_or_default(); + Ok::<_, ControlError>((decode_row(row)?, metadata)) + }) + .transpose()?; + Ok((transaction, evidence)) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs new file mode 100644 index 0000000000..887cd6fa21 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs @@ -0,0 +1,65 @@ +//! Fixed legacy fingerprint keys and validation of bounded source history pages. + +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::store::errors::ControlError; +use crate::warehouse_metadata::_helpers::ingestion::{outside_scan, validate_fields}; +use crate::warehouse_metadata::models::{MetadataBatch, WarehouseFingerprintRecord}; + +pub(crate) fn validate_batch( + batch: &MetadataBatch, + records: &[WarehouseFingerprintRecord], +) -> Result<(), ControlError> { + let invalid = records + .iter() + .flat_map(|record| { + [ + &record.state_schema, + &record.node_type, + &record.node_name, + &record.run_id, + ] + }) + .any(|value| value.trim().is_empty()); + let invalid_id = records + .iter() + .filter_map(|record| record.event_id.as_deref()) + .any(|value| value.trim().is_empty()); + validate_fields( + batch, + invalid || invalid_id, + outside_scan( + batch, + records + .iter() + .map(|record| (&record.state_database, &record.state_schema)), + ), + ) +} + +/// Hash fixed predecessor fields only when the warehouse has no persisted identity. +pub(crate) fn source_event_id(record: &WarehouseFingerprintRecord) -> String { + if let Some(event_id) = &record.event_id { + return event_id.clone(); + } + sha256_hex( + json!([ + "sqlbuild.fingerprint.legacy.v1", + record.node_type, + record.node_name, + record.target_database, + record.target_schema, + record.target_name, + record.run_id, + record.definition_hash, + record.version_hash, + record.schema_fingerprint, + record.definition, + record.metadata_json, + record.ts.format("%Y-%m-%dT%H:%M:%S%.6f+00:00").to_string(), + ]) + .to_string() + .as_bytes(), + ) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/freshness.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/freshness.rs new file mode 100644 index 0000000000..8a8551b84d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/freshness.rs @@ -0,0 +1,38 @@ +//! Validate source observations before they are stored. + +use crate::store::errors::ControlError; +use crate::warehouse_metadata::_helpers::ingestion::{outside_scan, validate_fields}; +use crate::warehouse_metadata::models::{MetadataBatch, WarehouseFreshnessRecord}; + +pub(crate) fn validate_batch( + batch: &MetadataBatch, + records: &[WarehouseFreshnessRecord], +) -> Result<(), ControlError> { + let invalid_identity = records + .iter() + .flat_map(|record| { + [ + &record.state_schema, + &record.source_name, + &record.run_id, + &record.strategy, + &record.value_kind, + &record.data_version_hash, + ] + }) + .any(|value| value.trim().is_empty()); + let invalid_source_id = records + .iter() + .filter_map(|record| record.event_id.as_deref()) + .any(|value| value.trim().is_empty()); + validate_fields( + batch, + invalid_identity || invalid_source_id, + outside_scan( + batch, + records + .iter() + .map(|record| (&record.state_database, &record.state_schema)), + ), + ) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs new file mode 100644 index 0000000000..b4e55f9337 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/ingestion.rs @@ -0,0 +1,150 @@ +//! A metadata page shares one registered session, snapshot fence and physical identity scope. + +use serde::de::DeserializeOwned; +use uuid::Uuid; + +use crate::platform::main::require_instance::require_instance; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{Change, ControlStore, ControlTransaction, Parameter, TransactionMode}; +use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use crate::warehouse_metadata::models::{MetadataBatch, MetadataScan, WarehouseSyncResponse}; + +pub(crate) async fn begin_ingestion( + store: &ControlStore, + (location_id, target): (Uuid, &str), + (session_id, generation): (Uuid, i64), +) -> Result { + let transaction = require_instance( + begin(store, TransactionMode::Write).await?, + (location_id, session_id, target), + ) + .await?; + let (transaction, rows) = query( + transaction, + "SELECT s.snapshot_id FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id \ + WHERE c.location_id=$1 AND c.target=$2 AND s.generation=$3", + vec![Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Integer(generation)], + ).await?; + if rows.is_empty() { + return Err(ControlError::Fenced); + } + Ok(transaction) +} + +pub(crate) fn batch_records( + batch: &MetadataBatch, +) -> Result, ControlError> { + batch + .records + .iter() + .map(|record| serde_json::from_value(record.clone())) + .collect::>() + .map_err(|error| { + ControlError::Invalid(format!("invalid {} record: {error}", batch.kind.as_str())) + }) +} + +pub(crate) fn scoped_event_id( + (location_id, target): (Uuid, &str), + (database, schema, table): (Option<&str>, &str, &str), + source_event_id: &str, +) -> Result { + let identity = serde_json::to_vec(&(target, database, schema, table, source_event_id)) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + Ok(Uuid::new_v5(&location_id, &identity)) +} + +/// Delivered rows take the scan's id; a complete scan deletes every row it did not deliver. +pub(crate) async fn commit_ingestion( + mut transaction: ControlTransaction, + (table, location_id, target): (&str, Uuid, &str), + (scan, keys): (Option<&MetadataScan>, &[Uuid]), + mut applied: bool, +) -> Result { + if let Some(scan) = scan { + if !keys.is_empty() { + let mut parameters = vec![Parameter::Id(scan.scan_id)]; + parameters.extend(keys.iter().copied().map(Parameter::Id)); + let placeholders = (2..=parameters.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + (transaction, _) = query( + transaction, + &format!("UPDATE {table} SET scan_id=$1 WHERE event_id IN ({placeholders})"), + parameters, + ) + .await?; + } + if scan.complete { + let mut parameters = vec![ + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::Text(scan.location.schema.clone()), + Parameter::Id(scan.scan_id), + ]; + let database = match &scan.location.database { + Some(database) => { + parameters.push(Parameter::Text(database.clone())); + "state_database=$5" + } + None => "state_database IS NULL", + }; + let removed; + (transaction, removed) = query( + transaction, + &format!( + "DELETE FROM {table} WHERE location_id=$1 AND target=$2 AND state_schema=$3 \ + AND {database} AND (scan_id IS NULL OR scan_id<>$4) RETURNING event_id" + ), + parameters, + ) + .await?; + applied |= !removed.is_empty(); + } + } + if applied { + transaction + .changes + .insert(Change::NodeStatus(location_id, target.into())); + } + commit(transaction).await?; + Ok(WarehouseSyncResponse { applied }) +} + +/// Every record of a scan page must come from the scanned physical namespace. +pub(crate) fn outside_scan<'a>( + batch: &MetadataBatch, + mut locations: impl Iterator, &'a String)>, +) -> bool { + batch.scan.as_ref().is_some_and(|scan| { + locations.any(|(database, schema)| { + *database != scan.location.database || *schema != scan.location.schema + }) + }) +} + +pub(crate) fn validate_fields( + batch: &MetadataBatch, + invalid_identity: bool, + invalid_scope: bool, +) -> Result<(), ControlError> { + let kind = batch.kind.as_str().replace('_', " "); + if batch.generation < 1 || batch.records.len() > MAX_HISTORY_RECORDS || invalid_identity { + return Err(ControlError::Invalid(format!( + "{kind} sync needs a positive generation, at most {MAX_HISTORY_RECORDS} records and nonempty state identities" + ))); + } + if invalid_scope + || batch + .scan + .as_ref() + .is_some_and(|scan| scan.location.schema.trim().is_empty()) + { + return Err(ControlError::Invalid(format!( + "{kind} scan records must belong to its physical namespace" + ))); + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs new file mode 100644 index 0000000000..4726f72d56 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod catalog; +pub(crate) mod events; +pub(crate) mod fingerprint_reads; +pub(crate) mod fingerprints; +pub(crate) mod freshness; +pub(crate) mod ingestion; +pub(crate) mod node_status; +pub(crate) mod person_reads; +pub(crate) mod statistics; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/node_status.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/node_status.rs new file mode 100644 index 0000000000..57a1ede606 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/node_status.rs @@ -0,0 +1,75 @@ +//! Physical observations and planner-local definitions remain independent evidence dimensions. + +use sqlbuild_core::fingerprint_identity::main::recorded_model_local_hash::recorded_model_local_hash; +use uuid::Uuid; + +use crate::snapshots::models::SnapshotNode; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; +use crate::warehouse_metadata::_helpers::ingestion::scoped_event_id; +use crate::warehouse_metadata::models::{ + DefinitionEvidence, FingerprintObservation, FingerprintQuery, PhysicalEvidence, +}; + +pub(crate) fn destination(node: &SnapshotNode) -> Option { + let destination = node.definition.get("destination")?; + Some(FingerprintQuery { + node_type: node.kind.clone(), + node_name: node.definition["name"].as_str()?.into(), + database: destination["database"].as_str().map(str::to_owned), + schema: Some(destination["schema"].as_str()?.into()), + name: Some(destination["name"].as_str()?.into()), + }) +} + +pub(crate) fn definition_evidence( + node: &SnapshotNode, + build: Option<&(FingerprintObservation, String)>, +) -> DefinitionEvidence { + let current = node.definition["planner_identity"]["local_hash"].as_str(); + let recorded = build.and_then(|(record, metadata)| match node.kind.as_str() { + "model" => recorded_model_local_hash(&record.definition_hash, metadata), + _ => None, + }); + match (current, recorded) { + (Some(current), Some(recorded)) if current == recorded => DefinitionEvidence::Matched { + local_hash: recorded, + }, + (Some(current), Some(recorded)) => DefinitionEvidence::Changed { + current_local_hash: current.into(), + recorded_local_hash: recorded, + }, + _ => DefinitionEvidence::Unavailable { + reason: "comparable planner-local model identities are unavailable".into(), + current_local_hash: current.map(str::to_owned), + }, + } +} + +pub(crate) async fn physical_evidence( + transaction: ControlTransaction, + (location_id, target, destination): (Uuid, &str, &FingerprintQuery), +) -> Result<(ControlTransaction, PhysicalEvidence), ControlError> { + let relation_id = scoped_event_id( + (location_id, target), + ( + destination.database.as_deref(), + destination.schema.as_deref().unwrap_or_default(), + "catalog", + ), + destination.name.as_deref().unwrap_or_default(), + )?; + let (transaction, rows) = query(transaction, + "SELECT event_id,observed_at,record_json AS record FROM sqb_warehouse_catalog WHERE event_id=$1 AND location_id=$2 AND target=$3 LIMIT 1", + vec![Parameter::Id(relation_id), Parameter::Id(location_id), Parameter::Text(target.into())]).await?; + let evidence = match rows.into_iter().next() { + Some(row) => PhysicalEvidence::Observed { + observation: decode_row(row)?, + }, + None => PhysicalEvidence::Unavailable { + reason: "no retained physical catalog observation for the exact destination".into(), + }, + }; + Ok((transaction, evidence)) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/person_reads.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/person_reads.rs new file mode 100644 index 0000000000..6e798a1c2d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/person_reads.rs @@ -0,0 +1,25 @@ +//! Person metadata reads share a live permission decision and the operation's writer fence. + +use uuid::Uuid; + +use crate::authorization::main::authorize_person::authorize_person; +use crate::credentials::models::{AuditContext, DefaultRole, PermissionScope}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::models::{ControlStore, ControlTransaction, TransactionMode}; + +pub(crate) async fn begin_view( + store: &ControlStore, + credentials: (&AuditContext, Uuid), + scope: &PermissionScope, + default_role: DefaultRole, +) -> Result { + let (transaction, _) = authorize_person( + begin(store, TransactionMode::Write).await?, + credentials, + scope, + default_role, + ) + .await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/statistics.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/statistics.rs new file mode 100644 index 0000000000..80562dda84 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/statistics.rs @@ -0,0 +1,154 @@ +//! Exact relation/time selection never joins definitions or infers data currency. + +use std::collections::BTreeSet; + +use chrono::{DateTime, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; +use crate::store::types::Row; +use crate::warehouse_metadata::_helpers::ingestion::scoped_event_id; +use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use crate::warehouse_metadata::models::{ + StatisticsObservation, StatisticsPage, StatisticsQuery, StatisticsRetention, + WarehouseCatalogRecord, WarehouseMirrorCursor, +}; + +pub(crate) fn statistics_event_id( + scope: (Uuid, &str), + (database, schema, name): (Option<&str>, &str, &str), + observed_at: DateTime, +) -> Result { + scoped_event_id( + scope, + (database, schema, "statistics"), + &json!((name, observed_at.timestamp().div_euclid(3600))).to_string(), + ) +} + +/// A partial namespace page never suppresses another relation's missing current-hour sample. +pub(crate) async fn eligible_names( + mut transaction: ControlTransaction, + scope: (Uuid, &str), + rows: Vec, +) -> Result<(ControlTransaction, Vec), ControlError> { + let mut candidates: Vec<(Uuid, String)> = Vec::new(); + for row in rows { + let record: WarehouseCatalogRecord = serde_json::from_value(row["record_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let id = statistics_event_id( + scope, + ( + record.state_database.as_deref(), + &record.state_schema, + &record.name, + ), + transaction.now, + )?; + candidates.push((id, record.name)); + } + let mut committed: BTreeSet = BTreeSet::new(); + if !candidates.is_empty() { + let placeholders = (1..=candidates.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + let found; + (transaction, found) = query( + transaction, + &format!( + "SELECT event_id FROM sqb_warehouse_statistics WHERE event_id IN ({placeholders})" + ), + candidates + .iter() + .map(|(id, _)| Parameter::Id(*id)) + .collect(), + ) + .await?; + for row in found { + committed.insert( + serde_json::from_value::(row["event_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + } + } + Ok(( + transaction, + candidates + .into_iter() + .filter(|(id, _)| !committed.contains(id)) + .map(|(_, name)| name) + .collect(), + )) +} + +pub(crate) async fn read_statistics_page( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), + (options, retention): (&StatisticsQuery, StatisticsRetention), +) -> Result<(ControlTransaction, StatisticsPage), ControlError> { + if options.schema.trim().is_empty() + || options.name.trim().is_empty() + || options.after_observed_at.is_some() != options.after_event_id.is_some() + { + return Err(ControlError::Invalid( + "statistics reads need an exact relation and both cursor fields or neither".into(), + )); + } + let limit = options + .limit + .unwrap_or(MAX_HISTORY_RECORDS as u32) + .clamp(1, MAX_HISTORY_RECORDS as u32) as usize; + let cutoff = retention.cutoff(transaction.now); + let mut parameters = vec![ + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::Text(options.schema.clone()), + Parameter::Text(options.name.clone()), + Parameter::Time(cutoff), + ]; + let database = if let Some(database) = &options.database { + parameters.push(Parameter::Text(database.clone())); + "state_database=$6" + } else { + "state_database IS NULL" + }; + let mut after = String::new(); + if let (Some(time), Some(id)) = (options.after_observed_at, options.after_event_id) { + let offset = parameters.len(); + parameters.extend([Parameter::Time(time), Parameter::Id(id)]); + after = format!( + " AND (observed_at,event_id)>(${},${})", + offset + 1, + offset + 2 + ); + } + parameters.push(Parameter::Integer(limit as i64)); + let (transaction, rows) = query(transaction, + &format!("SELECT event_id,record_json AS record FROM sqb_warehouse_statistics \ + WHERE location_id=$1 AND target=$2 AND state_schema=$3 AND relation_name=$4 AND observed_at>=$5 \ + AND {database}{after} ORDER BY observed_at,event_id LIMIT ${}", parameters.len()), parameters).await?; + let more = rows.len() == limit; + let records: Vec = rows + .into_iter() + .map(|row| serde_json::from_value(json!(row))) + .collect::>() + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let next_after = + more.then(|| records.last()) + .flatten() + .map(|observation| WarehouseMirrorCursor { + observed_at: observation.record.observed_at, + event_id: observation.event_id, + }); + Ok(( + transaction, + StatisticsPage { + records, + next_after, + }, + )) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs new file mode 100644 index 0000000000..c52cb938ed --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -0,0 +1,5 @@ +//! Limits bound each authenticated warehouse observation transaction. + +pub const MAX_HISTORY_RECORDS: usize = 100; +pub(crate) const CATALOG_MIRROR: &str = "sqb_warehouse_catalog"; +pub(crate) const DEFAULT_STATISTICS_RETENTION_DAYS: u16 = 90; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs new file mode 100644 index 0000000000..c02579a321 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs @@ -0,0 +1,49 @@ +//! Import warehouse-owned check results without deriving scheduler execution or recovery. + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::events::ingest_events; +use crate::warehouse_metadata::_helpers::ingestion::batch_records; +use crate::warehouse_metadata::models::{ + MetadataBatch, StateEventImport, WarehouseAuditRecord, WarehouseStateLocation, + WarehouseSyncResponse, +}; + +pub async fn ingest_audit_results( + store: &ControlStore, + scope: (Uuid, &str), + batch: &MetadataBatch, +) -> Result { + let records: Vec = batch_records(batch)?; + ingest_events( + store, + scope, + batch, + ( + records + .iter() + .map(|record| StateEventImport { + source_id: record.result_id.clone(), + location: WarehouseStateLocation { + database: record.state_database.clone(), + schema: record.state_schema.clone(), + }, + observed_at: record.occurred_at, + columns: vec![ + ("audit_name", Parameter::Text(record.audit_name.clone())), + ("outcome", Parameter::Text(record.outcome.clone())), + ], + record_json: json!(record), + }) + .collect(), + records + .iter() + .flat_map(|record| [&record.result_id, &record.state_schema, &record.audit_name]) + .any(|value| value.trim().is_empty()), + ), + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_catalog.rs new file mode 100644 index 0000000000..998ff9dc9a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_catalog.rs @@ -0,0 +1,63 @@ +//! A full catalog scan replaces its namespace, never inferring relation existence otherwise. + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::catalog::{project_catalog_record, validate_catalog}; +use crate::warehouse_metadata::_helpers::ingestion::{ + batch_records, begin_ingestion, commit_ingestion, scoped_event_id, +}; +use crate::warehouse_metadata::constants::CATALOG_MIRROR; +use crate::warehouse_metadata::models::{ + MetadataBatch, WarehouseCatalogRecord, WarehouseSyncResponse, +}; + +pub async fn ingest_catalog( + store: &ControlStore, + scope: (Uuid, &str), + batch: &MetadataBatch, +) -> Result { + let records: Vec = batch_records(batch)?; + validate_catalog(batch, &records)?; + let mut transaction = + begin_ingestion(store, scope, (batch.session_id, batch.generation)).await?; + let mut keys: Vec = Vec::new(); + let mut applied = false; + for record in &records { + let relation_id = scoped_event_id( + scope, + ( + record.state_database.as_deref(), + &record.state_schema, + "catalog", + ), + &record.name, + )?; + keys.push(relation_id); + let existing; + (transaction, existing) = query( + transaction, + "SELECT record_json FROM sqb_warehouse_catalog WHERE event_id=$1", + vec![Parameter::Id(relation_id)], + ) + .await?; + if existing + .first() + .is_some_and(|row| row["record_json"] == json!(record)) + { + continue; + } + transaction = project_catalog_record(transaction, scope, relation_id, record).await?; + applied = true; + } + commit_ingestion( + transaction, + (CATALOG_MIRROR, scope.0, scope.1), + (batch.scan.as_ref(), &keys), + applied, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs new file mode 100644 index 0000000000..9857bbc6ce --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_fingerprints.rs @@ -0,0 +1,87 @@ +//! Import bounded fingerprint history without conflating semantic and exact-content hashes. + +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::{compress::compress, query::query, store_source_text::store_source_text}; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::fingerprints::{source_event_id, validate_batch}; +use crate::warehouse_metadata::_helpers::ingestion::{ + batch_records, begin_ingestion, commit_ingestion, scoped_event_id, +}; +use crate::warehouse_metadata::models::{ + MetadataBatch, WarehouseFingerprintRecord, WarehouseSyncResponse, +}; + +pub async fn ingest_fingerprints( + store: &ControlStore, + (location_id, target): (Uuid, &str), + batch: &MetadataBatch, +) -> Result { + let records: Vec = batch_records(batch)?; + validate_batch(batch, &records)?; + let mut transaction = begin_ingestion( + store, + (location_id, target), + (batch.session_id, batch.generation), + ) + .await?; + let mut applied = false; + let mut keys: Vec = Vec::new(); + for record in &records { + let source_event_id = source_event_id(record); + let event_id = scoped_event_id( + (location_id, target), + ( + record.state_database.as_deref(), + &record.state_schema, + "_sqlbuild_fingerprints", + ), + &source_event_id, + )?; + keys.push(event_id); + let existing; + (transaction, existing) = query( + transaction, + "SELECT event_id FROM sqb_warehouse_fingerprints WHERE event_id=$1", + vec![Parameter::Id(event_id)], + ) + .await?; + if !existing.is_empty() { + continue; + } + let definition_text_hash = sha256_hex(record.definition.as_bytes()); + let mut record_json = json!(record); + if let serde_json::Value::Object(object) = &mut record_json { + object.remove("definition"); + } + let received_at = transaction.now; + transaction = store_source_text( + transaction, + ( + definition_text_hash.clone(), + compress(record.definition.as_bytes())?, + ), + ) + .await?; + let inserted; + (transaction, inserted) = query(transaction, + "INSERT INTO sqb_warehouse_fingerprints (event_id,location_id,target,source_event_id,observed_at,definition_text_hash,record_json,received_at,state_database,state_schema,node_type,node_name,target_database,target_schema,target_name) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", + vec![Parameter::Id(event_id), Parameter::Id(location_id), Parameter::Text(target.into()), + Parameter::Text(source_event_id), Parameter::Time(record.ts), Parameter::Text(definition_text_hash), + Parameter::Json(record_json), Parameter::Time(received_at), Parameter::OptionalText(record.state_database.clone()), Parameter::Text(record.state_schema.clone()), + Parameter::Text(record.node_type.clone()), Parameter::Text(record.node_name.clone()), Parameter::OptionalText(record.target_database.clone()), + Parameter::OptionalText(record.target_schema.clone()), Parameter::OptionalText(record.target_name.clone())]).await?; + applied |= !inserted.is_empty(); + } + commit_ingestion( + transaction, + ("sqb_warehouse_fingerprints", location_id, target), + (batch.scan.as_ref(), &keys), + applied, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_janitor_events.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_janitor_events.rs new file mode 100644 index 0000000000..e5893281e0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_janitor_events.rs @@ -0,0 +1,66 @@ +//! Mirror archive/delete history; the warehouse remains the owner of physical cleanup. + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::events::ingest_events; +use crate::warehouse_metadata::_helpers::ingestion::batch_records; +use crate::warehouse_metadata::models::{ + MetadataBatch, StateEventImport, WarehouseJanitorRecord, WarehouseStateLocation, + WarehouseSyncResponse, +}; + +pub async fn ingest_janitor_events( + store: &ControlStore, + scope: (Uuid, &str), + batch: &MetadataBatch, +) -> Result { + let records: Vec = batch_records(batch)?; + ingest_events( + store, + scope, + batch, + ( + records + .iter() + .map(|record| StateEventImport { + source_id: record.event_id.clone(), + location: WarehouseStateLocation { + database: record.state_database.clone(), + schema: record.state_schema.clone(), + }, + observed_at: record.occurred_at, + columns: vec![ + ("event_type", Parameter::Text(record.event_type.clone())), + ( + "relation_database", + Parameter::OptionalText(record.relation_database.clone()), + ), + ( + "relation_schema", + Parameter::Text(record.relation_schema.clone()), + ), + ("archive_name", Parameter::Text(record.archive_name.clone())), + ], + record_json: json!(record), + }) + .collect(), + records + .iter() + .flat_map(|record| { + [ + &record.event_id, + &record.state_schema, + &record.relation_schema, + &record.event_type, + &record.archive_name, + &record.archive_qualified_name, + ] + }) + .any(|value| value.trim().is_empty()), + ), + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_microbatches.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_microbatches.rs new file mode 100644 index 0000000000..123631e532 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_microbatches.rs @@ -0,0 +1,79 @@ +//! Mirror append-only microbatch facts without reading or changing execution coordination. + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::events::ingest_events; +use crate::warehouse_metadata::_helpers::ingestion::batch_records; +use crate::warehouse_metadata::models::{ + MetadataBatch, StateEventImport, WarehouseMicrobatchRecord, WarehouseStateLocation, + WarehouseSyncResponse, +}; + +pub async fn ingest_microbatches( + store: &ControlStore, + scope: (Uuid, &str), + batch: &MetadataBatch, +) -> Result { + let records: Vec = batch_records(batch)?; + ingest_events( + store, + scope, + batch, + ( + records + .iter() + .map(|record| StateEventImport { + source_id: record.event_id.clone(), + location: WarehouseStateLocation { + database: record.state_database.clone(), + schema: record.state_schema.clone(), + }, + observed_at: record.created_at, + columns: vec![ + ("model_name", Parameter::Text(record.model_name.clone())), + ("target_name", Parameter::Text(record.target_name.clone())), + ("scope_kind", Parameter::Text(record.scope_kind.clone())), + ("scope_key", Parameter::Text(record.scope_key.clone())), + ( + "physical_generation_id", + Parameter::Text(record.physical_generation_id.clone()), + ), + ("record_type", Parameter::Text(record.record_type.clone())), + ( + "origin_run_id", + Parameter::Text(record.origin_run_id.clone()), + ), + ( + "execution_run_id", + Parameter::Text(record.execution_run_id.clone()), + ), + ( + "replay_requirement_id", + Parameter::OptionalText(record.replay_requirement_id.clone()), + ), + ], + record_json: json!(record), + }) + .collect(), + records + .iter() + .flat_map(|record| { + [ + &record.event_id, + &record.state_schema, + &record.scope_kind, + &record.scope_key, + &record.model_name, + &record.target_name, + &record.physical_generation_id, + &record.record_type, + ] + }) + .any(|value| value.trim().is_empty()), + ), + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_migrations.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_migrations.rs new file mode 100644 index 0000000000..f37354b5b7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_migrations.rs @@ -0,0 +1,64 @@ +//! Mirror model-move facts without changing warehouse migration ownership. + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::events::ingest_events; +use crate::warehouse_metadata::_helpers::ingestion::batch_records; +use crate::warehouse_metadata::models::{ + MetadataBatch, StateEventImport, WarehouseMigrationRecord, WarehouseStateLocation, + WarehouseSyncResponse, +}; + +pub async fn ingest_migrations( + store: &ControlStore, + scope: (Uuid, &str), + batch: &MetadataBatch, +) -> Result { + let records: Vec = batch_records(batch)?; + ingest_events( + store, + scope, + batch, + ( + records + .iter() + .map(|record| StateEventImport { + source_id: record.event_id.clone(), + location: WarehouseStateLocation { + database: record.state_database.clone(), + schema: record.state_schema.clone(), + }, + observed_at: record.created_at, + columns: vec![ + ( + "target_name", + Parameter::OptionalText(record.target_name.clone()), + ), + ( + "destination_model", + Parameter::Text(record.destination_model.clone()), + ), + ("decision", Parameter::Text(record.decision.clone())), + ], + record_json: json!(record), + }) + .collect(), + records + .iter() + .flat_map(|record| { + [ + &record.event_id, + &record.state_schema, + &record.destination_model, + &record.destination_schema, + &record.destination_name, + ] + }) + .any(|value| value.trim().is_empty()), + ), + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_old_name_views.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_old_name_views.rs new file mode 100644 index 0000000000..d0e3ce7d7d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_old_name_views.rs @@ -0,0 +1,69 @@ +//! Mirror compatibility-view history, never an assertion about current relation existence. + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::events::ingest_events; +use crate::warehouse_metadata::_helpers::ingestion::batch_records; +use crate::warehouse_metadata::models::{ + MetadataBatch, StateEventImport, WarehouseOldNameRecord, WarehouseStateLocation, + WarehouseSyncResponse, +}; + +pub async fn ingest_old_name_views( + store: &ControlStore, + scope: (Uuid, &str), + batch: &MetadataBatch, +) -> Result { + let records: Vec = batch_records(batch)?; + ingest_events( + store, + scope, + batch, + ( + records + .iter() + .map(|record| StateEventImport { + source_id: record.event_id.clone(), + location: WarehouseStateLocation { + database: record.state_database.clone(), + schema: record.state_schema.clone(), + }, + observed_at: record.created_at, + columns: vec![ + ( + "target_name", + Parameter::OptionalText(record.target_name.clone()), + ), + ( + "migration_event_id", + Parameter::Text(record.migration_event_id.clone()), + ), + ( + "destination_model", + Parameter::Text(record.destination_model.clone()), + ), + ("event_type", Parameter::Text(record.event_type.clone())), + ], + record_json: json!(record), + }) + .collect(), + records + .iter() + .flat_map(|record| { + [ + &record.event_id, + &record.state_schema, + &record.migration_event_id, + &record.destination_model, + &record.new_schema, + &record.new_name, + ] + }) + .any(|value| value.trim().is_empty()), + ), + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs new file mode 100644 index 0000000000..85bf4d37a2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs @@ -0,0 +1,78 @@ +//! Persist source identities; only legacy rows use a fixed observation-key fallback. + +use serde_json::json; +use sqlbuild_state::freshness::main::event_id::event_id as legacy_event_id; +use sqlbuild_state::freshness::models::FreshnessEventIdentity; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::freshness::validate_batch; +use crate::warehouse_metadata::_helpers::ingestion::{ + batch_records, begin_ingestion, commit_ingestion, scoped_event_id, +}; +use crate::warehouse_metadata::models::{ + MetadataBatch, WarehouseFreshnessRecord, WarehouseSyncResponse, +}; + +pub async fn ingest_source_freshness( + store: &ControlStore, + (location_id, target): (Uuid, &str), + batch: &MetadataBatch, +) -> Result { + let records: Vec = batch_records(batch)?; + validate_batch(batch, &records)?; + let mut transaction = begin_ingestion( + store, + (location_id, target), + (batch.session_id, batch.generation), + ) + .await?; + let mut applied = false; + let mut keys: Vec = Vec::new(); + for record in &records { + let source_event_id = record.event_id.clone().unwrap_or_else(|| { + legacy_event_id(&FreshnessEventIdentity { + source_name: &record.source_name, + target_database: record.target_database.as_deref(), + target_schema: record.target_schema.as_deref(), + target_name: record.target_name.as_deref(), + run_id: &record.run_id, + strategy: &record.strategy, + value_kind: &record.value_kind, + data_version_hash: &record.data_version_hash, + observed_at: &record + .observed_at + .format("%Y-%m-%dT%H:%M:%S%.6f+00:00") + .to_string(), + }) + }); + let event_id = scoped_event_id( + (location_id, target), + ( + record.state_database.as_deref(), + &record.state_schema, + "_sqlbuild_source_freshness", + ), + &source_event_id, + )?; + keys.push(event_id); + let received_at = transaction.now; + let inserted; + (transaction, inserted) = query(transaction, + "INSERT INTO sqb_warehouse_freshness (event_id,location_id,target,source_event_id,observed_at,record_json,received_at,state_database,state_schema) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", + vec![Parameter::Id(event_id), Parameter::Id(location_id), Parameter::Text(target.into()), + Parameter::Text(source_event_id), Parameter::Time(record.observed_at), Parameter::Json(json!(record)), Parameter::Time(received_at), + Parameter::OptionalText(record.state_database.clone()), Parameter::Text(record.state_schema.clone())]).await?; + applied |= !inserted.is_empty(); + } + commit_ingestion( + transaction, + ("sqb_warehouse_freshness", location_id, target), + (batch.scan.as_ref(), &keys), + applied, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_statistics.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_statistics.rs new file mode 100644 index 0000000000..bf0046c67e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_statistics.rs @@ -0,0 +1,77 @@ +//! First committed metadata sample per relation and UTC hour wins; samples are not change facts. + +use std::collections::BTreeSet; + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::ingestion::{ + batch_records, begin_ingestion, commit_ingestion, validate_fields, +}; +use crate::warehouse_metadata::_helpers::statistics::statistics_event_id; +use crate::warehouse_metadata::models::{ + MetadataBatch, WarehouseStatisticsRecord, WarehouseSyncResponse, +}; + +pub async fn ingest_statistics( + store: &ControlStore, + scope: (Uuid, &str), + batch: &MetadataBatch, +) -> Result { + let records: Vec = batch_records(batch)?; + let mut names: BTreeSet<_> = BTreeSet::new(); + validate_fields( + batch, + records.iter().any(|record| { + record.state_schema.trim().is_empty() + || record.name.trim().is_empty() + || record.byte_count.is_some() != record.byte_count_basis.is_some() + || record + .byte_count_basis + .as_ref() + .is_some_and(|basis| basis.trim().is_empty()) + || !names.insert(( + ( + record.state_database.as_deref(), + &record.state_schema, + &record.name, + ), + record.observed_at.timestamp().div_euclid(3600), + )) + }), + batch.scan.is_some(), + )?; + let mut transaction = + begin_ingestion(store, scope, (batch.session_id, batch.generation)).await?; + let mut applied = false; + for record in &records { + let event_id = statistics_event_id( + scope, + ( + record.state_database.as_deref(), + &record.state_schema, + &record.name, + ), + record.observed_at, + )?; + let now = transaction.now; + let rows; + (transaction, rows) = query(transaction, + "INSERT INTO sqb_warehouse_statistics (event_id,location_id,target,state_database,state_schema,relation_name,observed_at,record_json,received_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", + vec![Parameter::Id(event_id),Parameter::Id(scope.0),Parameter::Text(scope.1.into()), + Parameter::OptionalText(record.state_database.clone()),Parameter::Text(record.state_schema.clone()), + Parameter::Text(record.name.clone()),Parameter::Time(record.observed_at),Parameter::Json(json!(record)),Parameter::Time(now)]).await?; + applied |= !rows.is_empty(); + } + commit_ingestion( + transaction, + ("sqb_warehouse_statistics", scope.0, scope.1), + (None, &[]), + applied, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs new file mode 100644 index 0000000000..4ebb52f84f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -0,0 +1,16 @@ +pub mod ingest_audit_results; +pub mod ingest_catalog; +pub mod ingest_fingerprints; +pub mod ingest_janitor_events; +pub mod ingest_microbatches; +pub mod ingest_migrations; +pub mod ingest_old_name_views; +pub mod ingest_source_freshness; +pub mod ingest_statistics; +pub(crate) mod prune_statistics; +pub mod read_person_catalog; +pub mod read_person_fingerprint; +pub mod read_person_node_status; +pub mod read_person_statistics; +pub mod read_statistics_due; +pub mod read_watermark; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/prune_statistics.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/prune_statistics.rs new file mode 100644 index 0000000000..de4629f1dc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/prune_statistics.rs @@ -0,0 +1,22 @@ +//! Expire statistics samples past their retention horizon in bounded pages. + +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use crate::warehouse_metadata::models::StatisticsRetention; + +/// Expiration is bounded global cleanup, never a lifecycle status or warehouse write. +pub(crate) async fn prune_statistics( + store: &ControlStore, + retention: StatisticsRetention, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let cutoff = retention.cutoff(transaction.now); + let (transaction, rows) = query(transaction, + "DELETE FROM sqb_warehouse_statistics WHERE event_id IN (SELECT event_id FROM sqb_warehouse_statistics \ + WHERE observed_at<$1 ORDER BY observed_at,event_id LIMIT $2) RETURNING event_id", + vec![Parameter::Time(cutoff), Parameter::Integer(MAX_HISTORY_RECORDS as i64)]).await?; + commit(transaction).await?; + Ok(rows.len()) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_catalog.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_catalog.rs new file mode 100644 index 0000000000..416d5002ab --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_catalog.rs @@ -0,0 +1,35 @@ +//! Live view permission and a bounded namespace page share the same transaction. + +use uuid::Uuid; + +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::store::errors::ControlError; +use crate::store::main::commit::commit; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::_helpers::catalog::read_catalog_page; +use crate::warehouse_metadata::_helpers::person_reads::begin_view; +use crate::warehouse_metadata::models::{CatalogPage, CatalogQuery}; + +pub async fn read_person_catalog( + store: &ControlStore, + (context, credentials): (&AuditContext, Uuid), + (location_id, target, options): (Uuid, &str, &CatalogQuery), + default_role: DefaultRole, +) -> Result { + let transaction = begin_view( + store, + (context, credentials), + &PermissionScope { + action: Action::View, + location_id: Some(location_id), + target: Some(target.into()), + nodes: Default::default(), + }, + default_role, + ) + .await?; + let (transaction, page) = + read_catalog_page(transaction, (location_id, target), options).await?; + commit(transaction).await?; + Ok(page) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_fingerprint.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_fingerprint.rs new file mode 100644 index 0000000000..8b156a3408 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_fingerprint.rs @@ -0,0 +1,35 @@ +//! Read one exact latest build through the node/destination index, never infer live state. + +use uuid::Uuid; + +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::store::errors::ControlError; +use crate::store::main::commit::commit; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::_helpers::fingerprint_reads::latest_fingerprint; +use crate::warehouse_metadata::_helpers::person_reads::begin_view; +use crate::warehouse_metadata::models::{FingerprintObservation, FingerprintQuery}; + +pub async fn read_person_fingerprint( + store: &ControlStore, + credentials: (&AuditContext, Uuid), + (location_id, target, options): (Uuid, &str, &FingerprintQuery), + default_role: DefaultRole, +) -> Result, ControlError> { + let transaction = begin_view( + store, + credentials, + &PermissionScope { + action: Action::View, + location_id: Some(location_id), + target: Some(target.into()), + nodes: [format!("{}:{}", options.node_type, options.node_name)].into(), + }, + default_role, + ) + .await?; + let (transaction, evidence) = + latest_fingerprint(transaction, (location_id, target, options)).await?; + commit(transaction).await?; + Ok(evidence.map(|(record, _)| record)) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_node_status.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_node_status.rs new file mode 100644 index 0000000000..814119940a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_node_status.rs @@ -0,0 +1,90 @@ +//! One authorized current node, one physical observation and one latest build; never warehouse polling. + +use uuid::Uuid; + +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::snapshots::main::read_current_record::read_current_record; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::store::errors::ControlError; +use crate::store::main::commit::commit; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::_helpers::fingerprint_reads::latest_fingerprint; +use crate::warehouse_metadata::_helpers::node_status::{ + definition_evidence, destination, physical_evidence, +}; +use crate::warehouse_metadata::_helpers::person_reads::begin_view; +use crate::warehouse_metadata::models::{ + NodeStatus, NodeStatusQuery, PhysicalEvidence, SourceDataStatus, +}; + +pub async fn read_person_node_status( + store: &ControlStore, + credentials: (&AuditContext, Uuid), + (location_id, target, options): (Uuid, &str, &NodeStatusQuery), + default_role: DefaultRole, +) -> Result, ControlError> { + let transaction = begin_view( + store, + credentials, + &PermissionScope { + action: Action::View, + location_id: Some(location_id), + target: Some(target.into()), + nodes: [options.node_key.clone()].into(), + }, + default_role, + ) + .await?; + if options.node_key.trim().is_empty() { + return Err(ControlError::Invalid( + "node status requires an exact node key".into(), + )); + } + let (transaction, snapshot) = read_current_record(transaction, (location_id, target)).await?; + let Some(snapshot) = snapshot else { + commit(transaction).await?; + return Ok(None); + }; + let (mut transaction, mut nodes) = read_snapshot_nodes( + transaction, + snapshot.snapshot_id, + Some(&[options.node_key.clone()].into()), + ) + .await?; + let Some(node) = nodes.remove(&options.node_key) else { + commit(transaction).await?; + return Ok(None); + }; + let mut build = None; + let physical = if let Some(destination) = destination(&node) { + let physical = if matches!(node.kind.as_str(), "model" | "seed") { + let physical; + (transaction, physical) = + physical_evidence(transaction, (location_id, target, &destination)).await?; + physical + } else { + PhysicalEvidence::Unavailable { + reason: "node is not a physical relation".into(), + } + }; + (transaction, build) = + latest_fingerprint(transaction, (location_id, target, &destination)).await?; + physical + } else { + PhysicalEvidence::Unavailable { + reason: "node has no physical relation destination".into(), + } + }; + let status = NodeStatus { + snapshot_id: snapshot.snapshot_id, + node_key: options.node_key.clone(), + physical, + definition: definition_evidence(&node, build.as_ref()), + build: build.map(|(record, _)| record), + source_data: SourceDataStatus::Unknown { + reason: "consumed-input provenance is not recorded".into(), + }, + }; + commit(transaction).await?; + Ok(Some(status)) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_statistics.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_statistics.rs new file mode 100644 index 0000000000..07f5ccd2ba --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_statistics.rs @@ -0,0 +1,35 @@ +//! Live View authority and bounded historical estimates share one writer transaction. + +use uuid::Uuid; + +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::store::errors::ControlError; +use crate::store::main::commit::commit; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::_helpers::person_reads::begin_view; +use crate::warehouse_metadata::_helpers::statistics::read_statistics_page; +use crate::warehouse_metadata::models::{StatisticsPage, StatisticsQuery, StatisticsRetention}; + +pub async fn read_person_statistics( + store: &ControlStore, + (context, credentials): (&AuditContext, Uuid), + (location_id, target, options): (Uuid, &str, &StatisticsQuery), + (default_role, retention): (DefaultRole, StatisticsRetention), +) -> Result { + let transaction = begin_view( + store, + (context, credentials), + &PermissionScope { + action: Action::View, + location_id: Some(location_id), + target: Some(target.into()), + nodes: Default::default(), + }, + default_role, + ) + .await?; + let (transaction, page) = + read_statistics_page(transaction, (location_id, target), (options, retention)).await?; + commit(transaction).await?; + Ok(page) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_statistics_due.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_statistics_due.rs new file mode 100644 index 0000000000..b8feaeb60d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_statistics_due.rs @@ -0,0 +1,58 @@ +//! Bounded catalog traversal excludes only relations with a committed sample this UTC hour. + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::{commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::ingestion::begin_ingestion; +use crate::warehouse_metadata::_helpers::statistics::eligible_names; +use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use crate::warehouse_metadata::models::{StatisticsDueRead, StatisticsRead}; + +pub async fn read_statistics_due( + store: &ControlStore, + scope: (Uuid, &str), + request: &StatisticsDueRead, +) -> Result { + if request.location.schema.trim().is_empty() + || !(1..=MAX_HISTORY_RECORDS).contains(&request.limit) + { + return Err(ControlError::Invalid( + "statistics eligibility needs a namespace and a bounded page".into(), + )); + } + let transaction = + begin_ingestion(store, scope, (request.session_id, request.generation)).await?; + let mut parameters = vec![ + Parameter::Id(scope.0), + Parameter::Text(scope.1.into()), + Parameter::Text(request.location.schema.clone()), + Parameter::Id(request.after.unwrap_or(Uuid::nil())), + ]; + let database = if let Some(database) = &request.location.database { + parameters.push(Parameter::Text(database.clone())); + "state_database=$5" + } else { + "state_database IS NULL" + }; + parameters.push(Parameter::Integer(request.limit as i64)); + let (transaction, rows) = query(transaction, + &format!("SELECT event_id,record_json FROM sqb_warehouse_catalog WHERE location_id=$1 AND target=$2 \ + AND state_schema=$3 AND event_id>$4 AND {database} ORDER BY event_id LIMIT ${}", parameters.len()), parameters).await?; + let next_after = if rows.len() == request.limit { + rows.last() + .map(|row| serde_json::from_value(row["event_id"].clone())) + .transpose() + .map_err(|error| ControlError::Invalid(error.to_string()))? + } else { + None + }; + let (transaction, names) = eligible_names(transaction, scope, rows).await?; + commit(transaction).await?; + Ok(StatisticsRead { + location: request.location.clone(), + names, + next_after, + }) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs new file mode 100644 index 0000000000..e6841fc058 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_watermark.rs @@ -0,0 +1,47 @@ +//! The durable mirror, not a separately persisted cursor, owns incremental progress. + +use chrono::{DateTime, Utc}; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::{commit::commit, query::query}; +use crate::store::models::Parameter; +use crate::warehouse_metadata::_helpers::ingestion::begin_ingestion; +use crate::warehouse_metadata::models::{WarehouseHistoryKind, WarehouseWatermarkRead}; + +pub async fn read_watermark( + store: &crate::store::models::ControlStore, + scope: (Uuid, &str), + request: &WarehouseWatermarkRead, +) -> Result>, ControlError> { + let transaction = + begin_ingestion(store, scope, (request.session_id, request.generation)).await?; + if matches!( + request.kind, + WarehouseHistoryKind::Catalog | WarehouseHistoryKind::Statistics + ) { + return Err(ControlError::Invalid( + "catalog and statistics observations do not support timestamp watermarks".into(), + )); + } + let table = request.kind.mirror_table(); + let mut parameters = vec![ + Parameter::Id(scope.0), + Parameter::Text(scope.1.into()), + Parameter::Text(request.location.schema.clone()), + ]; + let database = match &request.location.database { + Some(database) => { + parameters.push(Parameter::Text(database.clone())); + "state_database=$4" + } + None => "state_database IS NULL", + }; + let (transaction, rows) = query(transaction, + &format!("SELECT MAX(observed_at) AS watermark FROM {table} WHERE location_id=$1 AND target=$2 AND state_schema=$3 \ + AND {database}"), parameters).await?; + let watermark = serde_json::from_value(rows[0]["watermark"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + commit(transaction).await?; + Ok(watermark) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs new file mode 100644 index 0000000000..de8b740018 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs @@ -0,0 +1,6 @@ +//! Warehouse-owned observations mirrored by authenticated code locations, never execution state. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs new file mode 100644 index 0000000000..61e87d6a6c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -0,0 +1,611 @@ +//! Existing warehouse state rows, with their physical state-table provenance. + +use chrono::{DateTime, Duration, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::warehouse_metadata::constants::DEFAULT_STATISTICS_RETENTION_DAYS; + +/// Preserve observations, not an inferred latest version, age status or run outcome. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseFreshnessRecord { + pub event_id: Option, + pub state_database: Option, + pub state_schema: String, + pub source_name: String, + pub target_database: Option, + pub target_schema: Option, + pub target_name: Option, + pub run_id: String, + pub strategy: String, + pub value_kind: String, + pub data_version: Option, + pub data_version_hash: String, + pub observed_at: DateTime, +} + +/// Preserve the warehouse representation, including text-encoded measurements and booleans. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseAuditRecord { + pub state_database: Option, + pub state_schema: String, + pub result_id: String, + pub schema_version: i64, + pub occurred_at: DateTime, + pub invocation_id: String, + pub run_id: String, + pub audit_name: String, + pub audit_definition_name: String, + pub audit_description: Option, + pub binding_key: String, + pub definition_fingerprint: String, + pub execution_fingerprint: String, + pub evaluation_mode: String, + pub run_scope_phase: String, + pub attachment_kind: String, + pub attached_target_kind: Option, + pub attached_target_name: Option, + pub attached_column_name: Option, + pub target_database: Option, + pub target_schema: Option, + pub target_name: Option, + pub severity: String, + pub outcome: String, + pub execution_error: Option, + pub violation_count: Option, + pub measured_value: Option, + pub sample_count: Option, + pub sample_unit: Option, + pub minimum_samples: Option, + pub thresholds_json: Option, + pub evidence_json: Option, + pub evidence_count: Option, + pub evidence_truncated: Option, + pub evidence_error: Option, + pub measurement_sql: Option, + pub evidence_sql: Option, + pub executed_sql: Option, + pub sql_digest: Option, + pub metadata_json: Option, + pub reused: String, + pub audit_meta_json: Option, +} + +/// Warehouse cleanup facts do not authorize control-plane deletion or worker recovery. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseJanitorRecord { + pub state_database: Option, + pub state_schema: String, + pub event_id: String, + pub schema_version: i64, + pub event_type: String, + pub occurred_at: DateTime, + pub run_id: String, + pub relation_database: Option, + pub relation_schema: String, + pub relation_type: Option, + pub original_name: Option, + pub original_qualified_name: Option, + pub archive_name: String, + pub archive_qualified_name: String, + pub archived_at: DateTime, +} + +/// Model-move history is evidence, not authority to migrate a warehouse relation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseMigrationRecord { + pub state_database: Option, + pub state_schema: String, + pub event_id: String, + pub target_name: Option, + pub origin_model: Option, + pub origin_database: Option, + pub origin_schema: Option, + pub origin_name: String, + pub destination_model: String, + pub destination_database: Option, + pub destination_schema: String, + pub destination_name: String, + pub origin_version_hash: String, + pub discovery: String, + pub decision: String, + pub run_id: String, + pub created_at: DateTime, +} + +/// Preserve nullable compatibility-view facts without projecting live warehouse existence. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseOldNameRecord { + pub state_database: Option, + pub state_schema: String, + pub event_id: String, + pub target_name: Option, + pub event_type: String, + pub migration_event_id: String, + pub destination_model: String, + pub old_database: Option, + pub old_schema: Option, + pub old_name: String, + pub new_database: Option, + pub new_schema: String, + pub new_name: String, + pub view_retention: Option, + pub archive_name: Option, + pub column_aliases: Option, + pub grants_copied: Option, + pub view_sql: Option, + pub expires_at: Option>, + pub drop_reason: Option, + pub run_id: String, + pub created_at: DateTime, +} + +/// Immutable coordination evidence, never authority to execute or mark a batch complete. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseMicrobatchRecord { + pub state_database: Option, + pub state_schema: String, + pub event_id: String, + pub record_type: String, + pub scope_kind: String, + pub scope_key: String, + pub model_name: String, + pub target_database: Option, + pub target_schema: Option, + pub target_name: String, + pub physical_generation_id: String, + pub origin_run_id: String, + pub origin_run_started_at: Option>, + pub execution_run_id: String, + pub execution_run_started_at: Option>, + pub run_type: String, + pub completion_type: Option, + pub run_start: String, + pub run_end: String, + pub partition_start: Option, + pub partition_end: Option, + pub batch_size: String, + pub cursor_column: String, + pub cursor_type: String, + pub cursor_grain: Option, + pub model_version_hash: Option, + pub definition_hash: Option, + pub fingerprint_status: String, + pub replay_requirement_id: Option, + pub required_model_version_hash: Option, + pub previous_model_version_hash: Option, + pub replay_policy: Option, + pub rows_affected: Option, + pub completed_at: Option>, + pub coverage_source: Option, + pub observed_row_count: Option, + pub observed_at: Option>, + pub synthetic_reason: Option, + pub unaccounted_policy: Option, + pub created_at: DateTime, +} + +/// Physical namespace of SQLBuild's source-freshness state table, never a source query. +#[derive( + Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, utoipa::ToSchema, +)] +#[serde(deny_unknown_fields)] +pub struct WarehouseStateLocation { + pub database: Option, + pub schema: String, +} + +/// Catalog observations require full scans; only immutable history supports a watermark. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum WarehouseHistoryKind { + Fingerprints, + SourceFreshness, + AuditResults, + JanitorEvents, + Migrations, + OldNameViews, + Microbatches, + Catalog, + Statistics, +} + +impl WarehouseHistoryKind { + pub const ALL: [Self; 9] = [ + Self::Fingerprints, + Self::SourceFreshness, + Self::AuditResults, + Self::JanitorEvents, + Self::Migrations, + Self::OldNameViews, + Self::Microbatches, + Self::Catalog, + Self::Statistics, + ]; + + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Fingerprints => "fingerprints", + Self::SourceFreshness => "source_freshness", + Self::AuditResults => "audit_results", + Self::JanitorEvents => "janitor_events", + Self::Migrations => "migrations", + Self::OldNameViews => "old_name_views", + Self::Microbatches => "microbatches", + Self::Catalog => "catalog", + Self::Statistics => "statistics", + } + } + + #[must_use] + pub const fn mirror_table(self) -> &'static str { + match self { + Self::Fingerprints => "sqb_warehouse_fingerprints", + Self::SourceFreshness => "sqb_warehouse_freshness", + Self::AuditResults => "sqb_warehouse_audit_results", + Self::JanitorEvents => "sqb_warehouse_janitor_events", + Self::Migrations => "sqb_warehouse_migrations", + Self::OldNameViews => "sqb_warehouse_old_name_views", + Self::Microbatches => "sqb_warehouse_microbatches", + Self::Catalog => "sqb_warehouse_catalog", + Self::Statistics => "sqb_warehouse_statistics", + } + } +} + +/// Derive progress from committed mirror rows, scoped to the active physical reader. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseWatermarkRead { + pub session_id: Uuid, + pub generation: i64, + pub location: WarehouseStateLocation, + pub kind: WarehouseHistoryKind, +} + +/// Private interpreter input; cursor values are data and receive adapter literal rendering. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceFreshnessRead { + pub location: WarehouseStateLocation, + pub after: Option<[Option; 11]>, + /// Inclusive time bound, held fixed across pages; independent of the row-key cursor. + pub observed_since: Option>, + pub limit: Option, +} + +/// Fingerprints remain history here; latest-row selection belongs to the existing planner reader. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseFingerprintRecord { + pub event_id: Option, + pub state_database: Option, + pub state_schema: String, + pub node_type: String, + pub node_name: String, + pub target_database: Option, + pub target_schema: Option, + pub target_name: Option, + pub run_id: String, + pub definition_hash: String, + pub version_hash: String, + pub schema_fingerprint: String, + pub definition: String, + pub metadata_json: String, + pub ts: DateTime, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseMirrorCursor { + pub observed_at: DateTime, + pub event_id: Uuid, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct WarehouseSyncResponse { + pub applied: bool, +} + +/// One bounded page of one state table from the registered reader of a published generation. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct MetadataBatch { + pub kind: WarehouseHistoryKind, + pub session_id: Uuid, + pub generation: i64, + #[schema(value_type = Vec)] + pub records: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scan: Option, +} + +/// A full scan's complete page removes the namespace's rows that the scan did not deliver. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct MetadataScan { + pub scan_id: Uuid, + pub location: WarehouseStateLocation, + pub complete: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FingerprintRead { + pub location: WarehouseStateLocation, + pub after: Option<[Option; 13]>, + pub observed_since: Option>, + pub limit: Option, +} + +/// Stable event identities and timestamps order observations without comparing their payloads. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StateEventsRead { + pub location: WarehouseStateLocation, + pub after: Option<[Option; 3]>, + pub observed_since: Option>, + pub limit: Option, +} + +/// The private interpreter accepts only known state tables, never arbitrary SQL. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WarehouseHistoryRead { + SourceFreshness(SourceFreshnessRead), + Fingerprints(FingerprintRead), + AuditResults(StateEventsRead), + JanitorEvents(StateEventsRead), + Migrations(StateEventsRead), + OldNameViews(StateEventsRead), + Microbatches(StateEventsRead), + Catalog(CatalogRead), + Statistics(StatisticsRead), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(untagged)] +pub enum WarehouseHistoryRecord { + SourceFreshness(WarehouseFreshnessRecord), + Fingerprint(WarehouseFingerprintRecord), + AuditResult(Box), + JanitorEvent(WarehouseJanitorRecord), + Migration(Box), + OldNameView(Box), + Microbatch(Box), + Catalog(WarehouseCatalogRecord), + Statistics(WarehouseStatisticsRecord), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WarehouseHistoryPage { + pub records: Vec, + pub next_after: Option>>, +} + +/// Typed import columns are selected by code, never from location-supplied field names. +#[derive(Debug)] +pub(crate) struct StateEventImport { + pub source_id: String, + pub location: WarehouseStateLocation, + pub observed_at: DateTime, + pub columns: Vec<(&'static str, crate::store::models::Parameter)>, + pub record_json: serde_json::Value, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseCatalogColumn { + pub name: String, + pub data_type: String, +} + +/// Physical identity and ordered columns come from the adapter catalog, not fingerprints. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseCatalogRecord { + pub state_database: Option, + pub state_schema: String, + pub name: String, + pub relation_type: String, + pub columns: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CatalogRead { + pub location: WarehouseStateLocation, + pub after: Option<[Option; 1]>, + /// Catalog reads reject watermarks; the shared reader envelope supplies null. + pub observed_since: Option>, + pub limit: Option, +} + +/// Read one exact physical namespace; an omitted database means SQL NULL, never a wildcard. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CatalogQuery { + pub database: Option, + pub schema: String, + pub after_observed_at: Option>, + pub after_event_id: Option, + pub limit: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct CatalogObservation { + pub event_id: Uuid, + pub observed_at: DateTime, + pub record: WarehouseCatalogRecord, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct CatalogPage { + pub records: Vec, + pub next_after: Option, +} + +/// Exact node and physical destination, not a selector; omitted namespace values match SQL NULL. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FingerprintQuery { + pub node_type: String, + pub node_name: String, + pub database: Option, + pub schema: Option, + pub name: Option, +} + +/// Build evidence only: no SQL, arbitrary metadata, or inference about live relation existence. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct FingerprintObservation { + pub event_id: Uuid, + pub observed_at: DateTime, + pub source_event_id: String, + pub run_id: String, + pub definition_hash: String, + pub version_hash: String, + pub schema_fingerprint: String, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct NodeStatusQuery { + pub node_key: String, +} + +/// Observed physical evidence is not a live warehouse-existence guarantee. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "status", rename_all = "snake_case")] +pub enum PhysicalEvidence { + Observed { observation: CatalogObservation }, + Unavailable { reason: String }, +} + +/// Local definition identity only, not upstream data or graph currency. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "status", rename_all = "snake_case")] +pub enum DefinitionEvidence { + Matched { + local_hash: String, + }, + Changed { + current_local_hash: String, + recorded_local_hash: String, + }, + Unavailable { + reason: String, + current_local_hash: Option, + }, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "status", rename_all = "snake_case")] +pub enum SourceDataStatus { + Unknown { reason: String }, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct NodeStatus { + pub snapshot_id: Uuid, + pub node_key: String, + pub physical: PhysicalEvidence, + pub definition: DefinitionEvidence, + pub build: Option, + pub source_data: SourceDataStatus, +} + +/// A metadata estimate at collection time, never evidence of source-data currency. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseStatisticsRecord { + pub state_database: Option, + pub state_schema: String, + pub name: String, + pub observed_at: DateTime, + pub row_count: Option, + pub detail: Option, + pub byte_count: Option, + pub byte_count_basis: Option, +} + +/// Eligibility comes from each relation's committed hour, never a namespace watermark. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct StatisticsDueRead { + pub session_id: Uuid, + pub generation: i64, + pub location: WarehouseStateLocation, + pub after: Option, + pub limit: usize, +} + +/// One bounded catalog page; sampled and unsampled relations both advance its cursor. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct StatisticsRead { + pub location: WarehouseStateLocation, + pub names: Vec, + pub next_after: Option, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StatisticsQuery { + pub database: Option, + pub schema: String, + pub name: String, + pub after_observed_at: Option>, + pub after_event_id: Option, + pub limit: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct StatisticsObservation { + pub event_id: Uuid, + pub record: WarehouseStatisticsRecord, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct StatisticsPage { + pub records: Vec, + pub next_after: Option, +} + +/// Installation configuration: expired samples are hidden on read before bounded pruning removes them. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StatisticsRetention { + days: u16, +} + +impl StatisticsRetention { + pub fn days(days: u16) -> Result { + if days == 0 { + return Err(ControlError::Invalid( + "statistics retention must keep at least one day".into(), + )); + } + Ok(Self { days }) + } + + pub(crate) fn cutoff(self, now: DateTime) -> DateTime { + now - Duration::days(i64::from(self.days)) + } +} + +impl Default for StatisticsRetention { + fn default() -> Self { + Self { + days: DEFAULT_STATISTICS_RETENTION_DAYS, + } + } +} diff --git a/crates/sqlbuild-orchestration/src/webhooks/_helpers/delivery.rs b/crates/sqlbuild-orchestration/src/webhooks/_helpers/delivery.rs new file mode 100644 index 0000000000..1e39cc0409 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/webhooks/_helpers/delivery.rs @@ -0,0 +1,64 @@ +//! Ordered, at-least-once delivery; a response is acknowledged only after HTTP success. + +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::webhooks::_helpers::events::read_events; +use crate::webhooks::models::Webhook; + +/// A lost acknowledgement may resend the same event_id; receivers deduplicate by that ID. +pub(crate) async fn deliver_webhook( + store: &ControlStore, + sink: &Webhook, +) -> Result { + let transaction = begin(store, TransactionMode::Read).await?; + let table = sink.source.deliveries(); + let (transaction, rows) = query( + transaction, + &format!("SELECT COALESCE(MAX(storage_id), 0) AS cursor FROM {table} WHERE sink_id=$1"), + vec![Parameter::Text(sink.sink_id.clone())], + ) + .await?; + let cursor = serde_json::from_value(rows[0]["cursor"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let (transaction, events) = read_events(transaction, sink.source, cursor).await?; + commit(transaction).await?; + let count = events.len(); + for event in events { + let (storage_id, event_id) = event.identity(); + let response = sink + .client + .post(sink.url.clone()) + .header("Idempotency-Key", event_id.to_string()) + .json(&event) + .send() + .await + .map_err(|_| ControlError::Invalid("webhook delivery failed; event retained".into()))?; + if !response.status().is_success() { + return Err(ControlError::Invalid(format!( + "webhook returned HTTP {}; event retained", + response.status().as_u16() + ))); + } + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, _) = query( + transaction, + &format!( + "INSERT INTO {table} (sink_id, storage_id, event_id, delivered_at) \ + VALUES ($1,$2,$3,$4) ON CONFLICT (sink_id, storage_id) DO NOTHING" + ), + vec![ + Parameter::Text(sink.sink_id.clone()), + Parameter::Integer(storage_id), + Parameter::Text(event_id.to_string()), + Parameter::Time(now), + ], + ) + .await?; + commit(transaction).await?; + } + Ok(count) +} diff --git a/crates/sqlbuild-orchestration/src/webhooks/_helpers/events.rs b/crates/sqlbuild-orchestration/src/webhooks/_helpers/events.rs new file mode 100644 index 0000000000..50c830a7a8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/webhooks/_helpers/events.rs @@ -0,0 +1,34 @@ +//! Source-specific readers retain authority; delivery owns only acknowledgements. + +use crate::authorization::main::read_audit_events::read_audit_events; +use crate::platform::main::read_platform_events::read_platform_events; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; +use crate::webhooks::models::{WebhookEvent, WebhookSource}; + +pub(crate) async fn read_events( + transaction: ControlTransaction, + source: WebhookSource, + after: i64, +) -> Result<(ControlTransaction, Vec), ControlError> { + match source { + WebhookSource::Platform => { + let (transaction, events) = read_platform_events(transaction, after).await?; + Ok(( + transaction, + events.into_iter().map(WebhookEvent::Platform).collect(), + )) + } + WebhookSource::SecurityAudit => { + let (transaction, events) = read_audit_events(transaction, (after, 100)).await?; + Ok(( + transaction, + events + .into_iter() + .map(Box::new) + .map(WebhookEvent::SecurityAudit) + .collect(), + )) + } + } +} diff --git a/crates/sqlbuild-orchestration/src/webhooks/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/webhooks/_helpers/mod.rs new file mode 100644 index 0000000000..3fe549c1b3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/webhooks/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod delivery; +pub(crate) mod events; diff --git a/crates/sqlbuild-orchestration/src/webhooks/constants.rs b/crates/sqlbuild-orchestration/src/webhooks/constants.rs new file mode 100644 index 0000000000..e0742aaf9f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/webhooks/constants.rs @@ -0,0 +1,7 @@ +//! Delivery bounds shared by independent native event consumers. + +/// Only server operators configure this destination; never inherit it in a location or worker. +pub const SECURITY_AUDIT_WEBHOOK_VARIABLE: &str = "SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL"; + +pub(crate) const REQUEST_TIMEOUT_SECONDS: u64 = 10; +pub(crate) const POLL_SECONDS: u64 = 5; diff --git a/crates/sqlbuild-orchestration/src/webhooks/main/connect_webhook.rs b/crates/sqlbuild-orchestration/src/webhooks/main/connect_webhook.rs new file mode 100644 index 0000000000..e724005a6c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/webhooks/main/connect_webhook.rs @@ -0,0 +1,30 @@ +//! Validate operator-owned delivery configuration without exposing credentials in errors. + +use std::time::Duration; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::store::errors::ControlError; +use crate::webhooks::constants::REQUEST_TIMEOUT_SECONDS; +use crate::webhooks::models::{Webhook, WebhookSource}; + +/// Changing the endpoint creates a new consumer, starting from the retained event history. +pub fn connect_webhook(endpoint: &str, source: WebhookSource) -> Result { + let url = reqwest::Url::parse(endpoint) + .map_err(|_| ControlError::Invalid("webhook needs an HTTP(S) URL".into()))?; + if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() { + return Err(ControlError::Invalid("webhook needs an HTTP(S) URL".into())); + } + let sink_id = sha256_hex(url.as_str().as_bytes()); + let client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(REQUEST_TIMEOUT_SECONDS)) + .build() + .map_err(|_| ControlError::Invalid("webhook client could not start".into()))?; + Ok(Webhook { + client, + url, + sink_id, + source, + }) +} diff --git a/crates/sqlbuild-orchestration/src/webhooks/main/mod.rs b/crates/sqlbuild-orchestration/src/webhooks/main/mod.rs new file mode 100644 index 0000000000..76523ab16d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/webhooks/main/mod.rs @@ -0,0 +1,2 @@ +pub mod connect_webhook; +pub mod run_webhook; diff --git a/crates/sqlbuild-orchestration/src/webhooks/main/run_webhook.rs b/crates/sqlbuild-orchestration/src/webhooks/main/run_webhook.rs new file mode 100644 index 0000000000..2eeda4a457 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/webhooks/main/run_webhook.rs @@ -0,0 +1,48 @@ +//! Deliver durable events independently of user interpreters and scheduler leadership. + +use std::time::Duration; + +use tokio::sync::watch; + +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; +use crate::webhooks::_helpers::delivery::deliver_webhook; +use crate::webhooks::constants::POLL_SECONDS; +use crate::webhooks::models::Webhook; + +pub async fn run_webhook( + store: &ControlStore, + sink: Webhook, + mut shutdown: watch::Receiver, +) -> Result<(), ControlError> { + let mut failed = false; + while !*shutdown.borrow() { + tokio::select! { + result = deliver_webhook(store, &sink) => { + match result { + Ok(count) => { + if failed { + eprintln!("{} webhook delivery recovered", sink.source.label()); + } + failed = false; + if count > 0 { continue; } + }, + Err(_) => { + if !failed { + eprintln!("{} webhook delivery failed; retaining events and retrying", sink.source.label()); + } + failed = true; + }, + } + }, + result = shutdown.changed() => { + if result.is_err() || *shutdown.borrow() { break; } + }, + } + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(POLL_SECONDS)) => {}, + result = shutdown.changed() => { if result.is_err() { break; } }, + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/webhooks/mod.rs b/crates/sqlbuild-orchestration/src/webhooks/mod.rs new file mode 100644 index 0000000000..eea9ff20d1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/webhooks/mod.rs @@ -0,0 +1,6 @@ +//! Operator-configured, durable outbound delivery without user-code or scheduler ownership. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/webhooks/models.rs b/crates/sqlbuild-orchestration/src/webhooks/models.rs new file mode 100644 index 0000000000..d71271a5c3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/webhooks/models.rs @@ -0,0 +1,61 @@ +//! A closed event source determines its independent durable acknowledgement namespace. + +use crate::credentials::models::SecurityAuditEvent; +use crate::platform::models::PlatformEventEnvelope; + +#[derive(Clone, Copy, Debug)] +pub enum WebhookSource { + Platform, + SecurityAudit, +} + +impl WebhookSource { + pub(crate) const fn deliveries(self) -> &'static str { + match self { + Self::Platform => "sqb_platform_sink_deliveries", + Self::SecurityAudit => "sqb_security_audit_deliveries", + } + } + + pub(crate) const fn label(self) -> &'static str { + match self { + Self::Platform => "Platform", + Self::SecurityAudit => "Security audit", + } + } +} + +/// Neither errors nor Debug may expose an operator endpoint containing credentials. +#[derive(Clone)] +pub struct Webhook { + pub(crate) client: reqwest::Client, + pub(crate) url: reqwest::Url, + pub(crate) sink_id: String, + pub(crate) source: WebhookSource, +} + +impl std::fmt::Debug for Webhook { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("Webhook") + .field("source", &self.source) + .finish_non_exhaustive() + } +} + +/// Preserve the existing source envelope, not a delivery-specific copy of its fields. +#[derive(Debug, serde::Serialize)] +#[serde(untagged)] +pub(crate) enum WebhookEvent { + Platform(PlatformEventEnvelope), + SecurityAudit(Box), +} + +impl WebhookEvent { + pub(crate) fn identity(&self) -> (i64, uuid::Uuid) { + match self { + Self::Platform(event) => (event.storage_id, event.event_id), + Self::SecurityAudit(event) => (event.storage_id, event.event_id), + } + } +} diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs new file mode 100644 index 0000000000..813c99c3ac --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs @@ -0,0 +1,73 @@ +//! A worker's authenticated HTTP requests carry only its own run and claim. + +use sqlbuild_recall::records::models::OutputChunk; +use uuid::Uuid; + +use crate::api::main::client_response::client_response; +use crate::api::models::{HeartbeatResponse, MutationResponse}; +use crate::runs::models::{OutputPublication, RunChange, RunChangeRequest, RunExecution}; +use crate::worker::errors::WorkerError; +use crate::worker::models::WorkerClient; + +impl WorkerClient { + /// Fetch the launcher's immutable request, not the location's latest deployment. + pub async fn execution(&self) -> Result { + let execution: RunExecution = client_response( + self.connection + .client + .get(format!("{}execution", self.run_url)), + ) + .await?; + if execution.claim.request.run_id != self.run_id + || execution.claim.claim_id != self.claim_id + { + return Err(WorkerError::Configuration( + "execution response does not match the worker's run and claim", + )); + } + Ok(execution) + } + + /// Call with the same event ID and payload when retrying an uncertain response. + pub async fn report(&self, event_id: Uuid, change: RunChange) -> Result { + let request = RunChangeRequest { + event_id, + run_id: self.run_id, + claim_id: Some(self.claim_id), + change, + }; + let result: MutationResponse = client_response( + self.connection + .client + .post(format!("{}events", self.run_url)) + .json(&request), + ) + .await?; + Ok(result.applied) + } + + /// Retry the same chunks after an uncertain response; differing retries never overwrite bytes. + pub async fn publish_output( + &self, + chunks: &[OutputChunk], + ) -> Result { + Ok(client_response( + self.connection + .client + .post(format!("{}output", self.run_url)) + .json(chunks), + ) + .await?) + } + + /// Run independently from warehouse execution; cancellation is a cooperative request. + pub async fn heartbeat(&self) -> Result { + let result: HeartbeatResponse = client_response( + self.connection + .client + .post(format!("{}heartbeat", self.run_url)), + ) + .await?; + Ok(result.cancel_requested) + } +} diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs new file mode 100644 index 0000000000..25beaf886c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs @@ -0,0 +1,89 @@ +//! A dedicated native runtime keeps heartbeat IO independent of warehouse work and Python's GIL. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use crate::runs::models::RunExecution; +use crate::worker::errors::WorkerError; +use crate::worker::models::{WorkerClient, WorkerSession}; + +async fn heartbeat_loop( + client: WorkerClient, + cancelled: Arc, + mut stop: tokio::sync::watch::Receiver, +) { + let mut interval = tokio::time::interval(Duration::from_secs(10)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + biased; + _ = stop.changed() => return, + _ = interval.tick() => {} + } + let result = tokio::select! { + biased; + _ = stop.changed() => return, + result = client.heartbeat() => result, + }; + match result { + Ok(true) => cancelled.store(true, Ordering::Release), + Err(WorkerError::Rejected(status)) + if matches!(status.as_u16(), 401 | 403 | 404 | 410) => + { + cancelled.store(true, Ordering::Release); + return; + } + Ok(false) | Err(_) => {} + } + } +} + +pub(crate) async fn start_session( + client: WorkerClient, + execution: RunExecution, +) -> Result { + let cancelled = Arc::new(AtomicBool::new(false)); + let signal = Arc::clone(&cancelled); + let (stop, stopping) = tokio::sync::watch::channel(false); + let (ready, started) = tokio::sync::oneshot::channel(); + let thread = std::thread::Builder::new() + .name("sqb-worker-heartbeat".into()) + .spawn(move || { + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + let _ = ready.send(Err(WorkerError::from(error))); + return; + } + }; + runtime.block_on(async move { + let heartbeat = match client.connection.independent() { + Ok(connection) => WorkerClient { + connection, + ..client + }, + Err(error) => { + let _ = ready.send(Err(WorkerError::from(error))); + return; + } + }; + let _ = ready.send(Ok(())); + heartbeat_loop(heartbeat, signal, stopping).await; + }); + runtime.shutdown_timeout(Duration::from_secs(1)); + })?; + let session = WorkerSession { + execution, + cancel_requested: cancelled, + stop, + thread: Some(thread), + }; + started + .await + .map_err(|_| WorkerError::Configuration("heartbeat thread stopped during startup"))??; + Ok(session) +} diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs new file mode 100644 index 0000000000..e3fde3c92a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod client; +pub(crate) mod heartbeat; +pub(crate) mod recall; +pub(crate) mod session; diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/recall.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/recall.rs new file mode 100644 index 0000000000..b259195962 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/recall.rs @@ -0,0 +1,215 @@ +//! Reuse the native delivery thread and accounting over claim-scoped HTTP, without Python. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use sqlbuild_recall::records::models::{RecallRow, RowEnvelope}; +use sqlbuild_recall::schema::main::table_spec::table_spec; +use sqlbuild_recall::schema::models::Table; +use sqlbuild_recall::store::errors::{StoreError, StoreErrorKind}; +use sqlbuild_recall::store::models::InsertOutcome; +use sqlbuild_recall::store::types::{RecallStore, RecallStoreOpener}; +use sqlbuild_recall::writer::constants::{BATCH_ROWS, BATCH_WINDOW_MS}; +use sqlbuild_recall::writer::models::SessionStore; + +use crate::api::constants::BODY_BYTES; +use crate::api::main::client_response::client_response; +use crate::runs::constants::RECALL_BATCH_ROWS; +use crate::runs::models::{RecallBatch, RecallRecord}; +use crate::worker::models::WorkerClient; + +#[derive(Debug)] +struct WorkerRecallOpener(WorkerClient); + +struct WorkerRecallStore { + runtime: tokio::runtime::Runtime, + client: WorkerClient, + invocation_id: String, +} + +fn failure(message: impl Into, retryable: bool) -> StoreError { + StoreError { + kind: if retryable { + StoreErrorKind::Busy + } else { + StoreErrorKind::Other + }, + message: message.into(), + } +} + +fn transport_record(row: &RecallRow) -> RecallRecord { + RecallRecord { + table: row.table, + values: table_spec(row.table) + .columns + .iter() + .filter_map(|column| { + let value = row.value(column.name)?; + Some((column.name.to_owned(), value.clone())) + }) + .collect(), + } +} + +impl WorkerClient { + /// Pass this to the native Recall session; all delivery finishes before terminal reporting. + #[must_use] + pub fn recall_store(&self) -> SessionStore { + SessionStore::Hosted(Arc::new(WorkerRecallOpener(self.clone()))) + } + + /// The server owns common envelope columns and recomputes each row's content hash. + pub async fn publish_recall( + &self, + batch: &RecallBatch, + ) -> Result { + Ok(client_response( + self.connection + .client + .post(format!("{}recall", self.run_url)) + .json(batch), + ) + .await?) + } +} + +impl RecallStoreOpener for WorkerRecallOpener { + fn open( + &self, + envelope: &RowEnvelope, + give_up: Instant, + ) -> Result, StoreError> { + if Instant::now() >= give_up { + return Err(failure("Recall delivery deadline expired", true)); + } + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|_| failure("Recall delivery runtime could not start", false))?; + let client = { + let _entered = runtime.enter(); + WorkerClient { + connection: self + .0 + .connection + .independent() + .map_err(|_| failure("Recall HTTP client could not start", false))?, + ..self.0.clone() + } + }; + Ok(Box::new(WorkerRecallStore { + runtime, + client, + invocation_id: envelope.invocation_id.clone(), + })) + } + + fn batch_rows(&self) -> usize { + BATCH_ROWS + } + + fn batch_window(&self) -> Duration { + Duration::from_millis(BATCH_WINDOW_MS) + } +} + +/// Native materialization emits content/reference pairs, whose proof stays in one HTTP request. +impl RecallStore for WorkerRecallStore { + fn insert(&self, rows: &[RecallRow], give_up: Instant) -> Result { + if Instant::now() >= give_up { + return Err(failure("Recall delivery deadline expired", true)); + } + let mut batch = RecallBatch { + invocation_id: self.invocation_id.clone(), + records: Vec::new(), + }; + let base_bytes = serde_json::to_vec(&batch) + .map_err(|_| failure("Recall batch could not be encoded", false))? + .len(); + self.runtime.block_on(async { + let mut result = InsertOutcome::default(); + let mut bytes = base_bytes; + let mut offset = 0; + while offset < rows.len() { + if Instant::now() >= give_up { + return Err(failure("Recall delivery deadline expired", true)); + } + let paired = rows[offset].table == Table::Artifacts + && rows.get(offset + 1).is_some_and(|next| { + next.table == Table::ArtifactRefs + && next.value("digest") == rows[offset].value("digest") + }); + let end = offset + if paired { 2 } else { 1 }; + let records = rows[offset..end] + .iter() + .map(transport_record) + .collect::>(); + let added = serde_json::to_vec(&records) + .map_err(|_| failure("Recall records could not be encoded", false))? + .len() + - 2; + if base_bytes + added > BODY_BYTES { + return Err(failure("Recall record exceeds the HTTP body limit", false)); + } + if !batch.records.is_empty() + && (batch.records.len() + records.len() > RECALL_BATCH_ROWS + || bytes + added + 1 > BODY_BYTES) + { + (batch, result) = self.flush(batch, result, give_up).await?; + bytes = base_bytes; + } + bytes += added + usize::from(!batch.records.is_empty()); + batch.records.extend(records); + offset = end; + } + if !batch.records.is_empty() || rows.is_empty() { + (_, result) = self.flush(batch, result, give_up).await?; + } + result.delivered.sort_by_key(|(table, _)| *table); + Ok(result) + }) + } +} + +impl WorkerRecallStore { + async fn flush( + &self, + mut batch: RecallBatch, + mut result: InsertOutcome, + give_up: Instant, + ) -> Result<(RecallBatch, InsertOutcome), StoreError> { + let outcome = tokio::time::timeout_at( + tokio::time::Instant::from_std(give_up), + self.client.publish_recall(&batch), + ) + .await + .map_err(|_| failure("Recall delivery deadline expired", true))? + .map_err(|error| { + let retryable = match &error { + crate::worker::errors::WorkerError::Transport(_) => true, + crate::worker::errors::WorkerError::Rejected(status) => { + status.is_server_error() || *status == reqwest::StatusCode::TOO_MANY_REQUESTS + } + _ => false, + }; + failure(error.to_string(), retryable) + })?; + for (table, count) in outcome.delivered { + if let Some((_, previous)) = result + .delivered + .iter_mut() + .find(|(found, _)| *found == table) + { + *previous += count; + } else { + result.delivered.push((table, count)); + } + } + result.conflicts.extend(outcome.conflicts); + result.failed.extend(outcome.failed); + result.failure = result.failure.take().or(outcome.failure); + batch.records.clear(); + Ok((batch, result)) + } +} diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/session.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/session.rs new file mode 100644 index 0000000000..e7bdd3ad08 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/session.rs @@ -0,0 +1,22 @@ +//! The execution owner drains its heartbeat before worker runtime teardown. + +use std::sync::atomic::Ordering; + +use crate::worker::models::WorkerSession; + +impl WorkerSession { + /// Cancellation and credential revocation are monotonic for one pinned worker invocation. + #[must_use] + pub fn cancel_requested(&self) -> bool { + self.cancel_requested.load(Ordering::Acquire) + } +} + +impl Drop for WorkerSession { + fn drop(&mut self) { + let _ = self.stop.send(true); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} diff --git a/crates/sqlbuild-orchestration/src/worker/errors.rs b/crates/sqlbuild-orchestration/src/worker/errors.rs new file mode 100644 index 0000000000..49240df632 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/errors.rs @@ -0,0 +1,27 @@ +//! Transport failures are distinct from rejected pinned execution inputs. + +#[derive(Debug, thiserror::Error)] +pub enum WorkerError { + #[error("invalid worker connection: {0}")] + Configuration(&'static str), + #[error("worker API request failed: {0}")] + Transport(#[from] reqwest::Error), + #[error("worker API rejected the request with HTTP {0}")] + Rejected(reqwest::StatusCode), + #[error("worker heartbeat could not start: {0}")] + Heartbeat(#[from] std::io::Error), + #[error(transparent)] + Verification(#[from] crate::store::errors::ControlError), + #[error("run cancellation was requested before execution")] + Cancelled, +} + +impl From for WorkerError { + fn from(error: crate::api::errors::ClientError) -> Self { + match error { + crate::api::errors::ClientError::Configuration(message) => Self::Configuration(message), + crate::api::errors::ClientError::Transport(error) => Self::Transport(error), + crate::api::errors::ClientError::Rejected(status) => Self::Rejected(status), + } + } +} diff --git a/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs b/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs new file mode 100644 index 0000000000..15a7a4d081 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs @@ -0,0 +1,23 @@ +//! Bound worker requests have deadlines and must never follow credential-bearing redirects. + +use uuid::Uuid; + +use crate::api::main::connect_client::connect_client; +use crate::worker::errors::WorkerError; +use crate::worker::models::WorkerClient; + +/// Connect to an API origin or path prefix using the launcher's run-scoped bearer. +pub fn connect_worker( + server_url: &str, + bearer: &str, + (run_id, claim_id): (Uuid, Uuid), +) -> Result { + let connection = connect_client(server_url, bearer)?; + let run_url = connection.endpoint(&["api", "runs", &run_id.to_string(), ""])?; + Ok(WorkerClient { + connection, + run_url, + run_id, + claim_id, + }) +} diff --git a/crates/sqlbuild-orchestration/src/worker/main/mod.rs b/crates/sqlbuild-orchestration/src/worker/main/mod.rs new file mode 100644 index 0000000000..05f5586133 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/main/mod.rs @@ -0,0 +1,2 @@ +pub mod connect_worker; +pub mod start_worker; diff --git a/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs b/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs new file mode 100644 index 0000000000..72569448be --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs @@ -0,0 +1,26 @@ +//! Verify the baseline publication before any run overrides or warehouse mutation. + +use uuid::Uuid; + +use crate::runs::models::RunChange; +use crate::snapshots::main::verify_execution::verify_execution; +use crate::snapshots::models::SnapshotInput; +use crate::worker::_helpers::heartbeat::start_session; +use crate::worker::errors::WorkerError; +use crate::worker::models::{WorkerClient, WorkerSession}; + +/// Verify the baseline compile before planning, cloning, applying run settings or executing. +pub async fn start_worker( + client: &WorkerClient, + baseline: &SnapshotInput, +) -> Result { + let expected = client.execution().await?; + verify_execution(baseline, &expected)?; + if client.heartbeat().await? { + return Err(WorkerError::Cancelled); + } + let event_id = Uuid::new_v5(&expected.claim.claim_id, b"worker-started"); + let session = start_session(client.clone(), expected).await?; + client.report(event_id, RunChange::Started).await?; + Ok(session) +} diff --git a/crates/sqlbuild-orchestration/src/worker/mod.rs b/crates/sqlbuild-orchestration/src/worker/mod.rs new file mode 100644 index 0000000000..59f273cbec --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod _helpers; +pub mod errors; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/worker/models.rs b/crates/sqlbuild-orchestration/src/worker/models.rs new file mode 100644 index 0000000000..81799ca75f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/models.rs @@ -0,0 +1,26 @@ +//! A worker owns only an API credential, never a control-store connection. + +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +use uuid::Uuid; + +use crate::api::models::ControlClient; +use crate::runs::models::RunExecution; + +#[derive(Clone, Debug)] +pub struct WorkerClient { + pub(crate) connection: ControlClient, + pub(crate) run_url: reqwest::Url, + pub(crate) run_id: Uuid, + pub(crate) claim_id: Uuid, +} + +/// Keep this owner alive through execution and terminal reporting, then stop and join its heartbeat. +#[derive(Debug)] +pub struct WorkerSession { + pub execution: RunExecution, + pub(crate) cancel_requested: Arc, + pub(crate) stop: tokio::sync::watch::Sender, + pub(crate) thread: Option>, +} diff --git a/crates/sqlbuild-orchestration/tests/api/changes/helpers.rs b/crates/sqlbuild-orchestration/tests/api/changes/helpers.rs new file mode 100644 index 0000000000..9a7ff3e5d9 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/changes/helpers.rs @@ -0,0 +1,172 @@ +//! Real HTTP event parsing and commit-time fault injection shared by SSE contracts. + +use std::collections::BTreeSet; +use std::time::Duration; + +use uuid::Uuid; + +use crate::api::tests::test_types::{ResetProxy, SseConnection, TestApi}; +use crate::store::models::ControlStore; + +pub(crate) async fn connect( + client: &reqwest::Client, + api: &TestApi, + (location_id, target, bearer): (Uuid, &str, &str), +) -> SseConnection { + let response = client + .get(format!( + "{}/api/locations/{location_id}/targets/{target}/changes", + api.url + )) + .bearer_auth(bearer) + .send() + .await + .expect("open scoped SSE connection"); + assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!(response.headers()["content-type"], "text/event-stream"); + assert_eq!(response.headers()["cache-control"], "no-store"); + let mut stream = SseConnection { + response, + pending: Vec::new(), + }; + assert_eq!( + next_topic(&mut stream).await.as_deref(), + Some("all"), + "a reconnect must refetch REST state" + ); + stream +} + +pub(crate) async fn next_topic(stream: &mut SseConnection) -> Option { + tokio::time::timeout(Duration::from_secs(3), async { + for _ in 0..4096 { + for _ in 0..4096 { + let Some(end) = stream.pending.windows(2).position(|bytes| bytes == b"\n\n") else { + break; + }; + let bytes: Vec<_> = stream.pending.drain(..end + 2).collect(); + let frame = std::str::from_utf8(&bytes).expect("UTF-8 event frame"); + let Some(data) = frame.lines().find_map(|line| line.strip_prefix("data: ")) else { + continue; + }; + assert!(frame.lines().any(|line| line == "event: changed")); + let body: serde_json::Value = + serde_json::from_str(data).expect("typed change notice"); + return Some(body["topic"].as_str().expect("topic").to_owned()); + } + let chunk = stream + .response + .chunk() + .await + .expect("SSE body remains readable")?; + stream.pending.extend_from_slice(&chunk); + } + panic!("SSE stream exhausted bounded frame reads") + }) + .await + .expect("bounded SSE delivery or shutdown") +} + +pub(crate) async fn expect_topics(stream: &mut SseConnection, expected: &[String]) { + let mut actual = BTreeSet::new(); + for _ in expected { + assert!( + actual.insert(next_topic(stream).await.expect("committed change")), + "transaction-local topics deduplicate" + ); + } + assert_eq!(actual, expected.iter().cloned().collect()); +} + +pub(crate) async fn reject_snapshot_commit(store: &ControlStore) { + execute_fault_sql(store, ( + "CREATE TABLE sse_commit_parent (id INTEGER PRIMARY KEY); \ + CREATE TABLE sse_commit_guard (id INTEGER REFERENCES sse_commit_parent(id) DEFERRABLE INITIALLY DEFERRED); \ + CREATE TRIGGER sse_reject_snapshot AFTER INSERT ON sqb_snapshots BEGIN INSERT INTO sse_commit_guard VALUES (1); END;", + "CREATE TABLE sse_commit_parent (id INTEGER PRIMARY KEY); \ + CREATE TABLE sse_commit_guard (id INTEGER REFERENCES sse_commit_parent(id) DEFERRABLE INITIALLY DEFERRED); \ + CREATE FUNCTION sse_reject_snapshot() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN INSERT INTO sse_commit_guard VALUES (1); RETURN NEW; END $$; \ + CREATE TRIGGER sse_reject_snapshot AFTER INSERT ON sqb_snapshots FOR EACH ROW EXECUTE FUNCTION sse_reject_snapshot();" + )).await; +} + +pub(crate) async fn restore_snapshot_commits(store: &ControlStore) { + execute_fault_sql(store, ( + "DROP TRIGGER sse_reject_snapshot; DROP TABLE sse_commit_guard; DROP TABLE sse_commit_parent;", + "DROP TRIGGER sse_reject_snapshot ON sqb_snapshots; DROP FUNCTION sse_reject_snapshot(); DROP TABLE sse_commit_guard; DROP TABLE sse_commit_parent;" + )).await; +} + +async fn execute_fault_sql(store: &ControlStore, (sqlite, postgres): (&str, &str)) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + rusqlite::Connection::open(path) + .expect("fault connection") + .execute_batch(sqlite) + .expect("SQLite commit fault setup"); + return; + }; + sqlx_core::raw_sql::raw_sql(postgres) + .execute(pool) + .await + .expect("PostgreSQL commit fault setup"); +} + +/// TCP RST, unlike a PostgreSQL termination message, exercises SQLx's unhandled reset error. +pub(crate) async fn reset_proxy(store: &ControlStore) -> ResetProxy { + let ControlStore::Postgres(pool) = store else { + panic!("PostgreSQL fixture") + }; + let options = pool.connect_options().as_ref().clone(); + let upstream = (options.get_host().to_owned(), options.get_port()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("proxy socket"); + let port = listener.local_addr().expect("proxy address").port(); + let (reset, resets) = tokio::sync::watch::channel(0_u64); + let (shutdown, mut stopped) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + let mut connections = tokio::task::JoinSet::new(); + for _ in 0..10_000 { + tokio::select! { + accepted = listener.accept() => { + let (mut downstream, _) = accepted.expect("accept database connection"); + let upstream = upstream.clone(); + let mut reset = resets.clone(); + reset.borrow_and_update(); + connections.spawn(async move { + let mut upstream = tokio::net::TcpStream::connect(upstream).await.expect("connect real PostgreSQL"); + downstream.set_zero_linger().expect("reset rather than EOF on drop"); + tokio::select! { + _ = tokio::io::copy_bidirectional(&mut downstream, &mut upstream) => {}, + _ = reset.changed() => {}, + } + }); + }, + Some(result) = connections.join_next(), if !connections.is_empty() => { result.expect("proxy connection task"); }, + _ = &mut stopped => return, + } + } + panic!("database proxy exhausted bounded connection events"); + }); + let store = ControlStore::Postgres( + sqlx_postgres::PgPoolOptions::new() + .max_connections(10) + .connect_with( + options + .host("127.0.0.1") + .port(port) + .ssl_mode(sqlx_postgres::PgSslMode::Disable), + ) + .await + .expect("connect through reset proxy"), + ); + ResetProxy { + store, + reset, + shutdown, + task, + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/fingerprint_reads/helpers.rs b/crates/sqlbuild-orchestration/tests/api/fingerprint_reads/helpers.rs new file mode 100644 index 0000000000..3018d28bb1 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/fingerprint_reads/helpers.rs @@ -0,0 +1,161 @@ +//! Actual ingestion and additive upgrade retain bounded, exact-destination build evidence. + +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::models::{LocationHealth, LocationHealthReport}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::main::{ + ingest_fingerprints::ingest_fingerprints, read_person_fingerprint::read_person_fingerprint, +}; +use crate::warehouse_metadata::models::{FingerprintQuery, MetadataBatch}; + +pub(crate) async fn fingerprint_reads( + store: &ControlStore, + credentials: (&AuditContext, Uuid), + (location, other): (Uuid, Uuid), +) -> Uuid { + let session_id = Uuid::now_v7(); + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: "development".into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("reader session"); + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/fingerprint_lookup.json"); + let data: Value = serde_json::from_str(&std::fs::read_to_string(path).expect("fixture")) + .expect("fingerprint inputs"); + let latest = data["record"].clone(); + let mut older = latest.clone(); + older["event_id"] = json!("orders-build-older"); + older["ts"] = data["older_ts"].clone(); + older["version_hash"] = json!("orders-version-older"); + let mut records = vec![latest.clone(), older]; + for (key, value) in [ + ("target_database", json!("orders_data")), + ("target_schema", json!("landing")), + ("target_name", json!("Orders")), + ("node_type", json!("source")), + ("node_name", json!("customers")), + ] { + let mut record = latest.clone(); + record["event_id"] = json!(format!("orders-distractor-{key}")); + record["ts"] = data["distractor_ts"].clone(); + record[key] = value; + records.push(record); + } + let batch: MetadataBatch = serde_json::from_value(json!({"kind":"fingerprints", + "session_id":session_id,"generation":1,"records":records, + })) + .expect("batch"); + assert!( + ingest_fingerprints(store, (location, "development"), &batch) + .await + .expect("ingest") + .applied + ); + let mut options = FingerprintQuery { + node_type: "model".into(), + node_name: "orders".into(), + database: None, + schema: Some("main".into()), + name: Some("orders".into()), + }; + let first = read_person_fingerprint( + store, + credentials, + (location, "development", &options), + DefaultRole::None, + ) + .await + .expect("latest build") + .expect("evidence"); + let serialized = json!(&first); + assert_eq!( + serialized, + json!({ + "event_id":first.event_id,"observed_at":latest["ts"],"source_event_id":latest["event_id"], + "run_id":latest["run_id"],"definition_hash":latest["definition_hash"], + "version_hash":latest["version_hash"],"schema_fingerprint":latest["schema_fingerprint"], + }) + ); + for scope in [(other, "development"), (location, "preview")] { + assert!(matches!( + read_person_fingerprint( + store, + credentials, + (scope.0, scope.1, &options), + DefaultRole::None + ) + .await, + Err(ControlError::Denied { .. }) + )); + assert!( + read_person_fingerprint( + store, + credentials, + (scope.0, scope.1, &options), + DefaultRole::Viewer, + ) + .await + .expect("authorized empty scope") + .is_none() + ); + } + options.schema = None; + assert!( + read_person_fingerprint( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await + .expect("NULL schema is exact") + .is_none() + ); + options.schema = Some("main".into()); + options.name = None; + assert!( + read_person_fingerprint( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await + .expect("NULL relation is exact") + .is_none() + ); + options.name = Some("orders".into()); + assert!( + !ingest_fingerprints(store, (location, "development"), &batch) + .await + .expect("replay") + .applied + ); + assert_eq!( + json!( + read_person_fingerprint( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await + .expect("indexed fingerprint") + ), + serialized + ); + session_id +} diff --git a/crates/sqlbuild-orchestration/tests/api/helpers.rs b/crates/sqlbuild-orchestration/tests/api/helpers.rs new file mode 100644 index 0000000000..23df808669 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/helpers.rs @@ -0,0 +1,82 @@ +use std::time::Duration; + +use crate::api::main::serve_api::serve_api; +use crate::api::models::ApiAuthentication; +use crate::api::tests::test_types::TestApi; +use crate::store::models::ControlStore; + +pub(crate) async fn start_api(store: &ControlStore) -> TestApi { + start_api_with_authentication(store, ApiAuthentication::default()).await +} + +pub(crate) async fn start_api_with_authentication( + store: &ControlStore, + authentication: ApiAuthentication, +) -> TestApi { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("isolated API socket"); + let url = format!("http://{}", listener.local_addr().expect("API address")); + let (shutdown, signal) = tokio::sync::oneshot::channel(); + let store = store.clone(); + let task = tokio::spawn(serve_api( + store, + listener, + (authentication, Vec::new(), Default::default()), + (|| {}, async { + let _ = signal.await; + }), + )); + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(1)) + .build() + .expect("HTTP client"); + tokio::time::timeout(Duration::from_secs(5), async { + for _ in 0..250 { + assert!(!task.is_finished(), "API failed during startup"); + tokio::time::sleep(Duration::from_millis(20)).await; + let Ok(response) = client + .get(format!("{url}/api/openapi.json")) + .send() + .await + .and_then(reqwest::Response::error_for_status) + else { + continue; + }; + drop(response); + return; + } + panic!("API exhausted readiness attempts"); + }) + .await + .expect("API becomes ready"); + TestApi { + url, + shutdown, + task, + } +} + +pub(crate) async fn stop_api(api: TestApi) { + api.shutdown.send(()).expect("request API shutdown"); + tokio::time::timeout(Duration::from_secs(2), api.task) + .await + .expect("prompt API shutdown") + .expect("join API server") + .expect("API server stopped cleanly"); +} + +pub(crate) async fn read_json( + client: &reqwest::Client, + (url, bearer): (&str, &str), +) -> serde_json::Value { + let response = client + .get(url) + .bearer_auth(bearer) + .send() + .await + .expect("HTTP read"); + assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!(response.headers()["cache-control"], "no-store"); + response.json().await.expect("typed JSON response") +} diff --git a/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs b/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs new file mode 100644 index 0000000000..9c08fe2846 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs @@ -0,0 +1,130 @@ +//! Sensitive SQL reads disclose only the current exact node after durable audit publication. + +use reqwest::{Client, Method, StatusCode}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::tests::person_sessions_helpers::request; +use crate::api::tests::person_sessions_test_types::BrowserSession; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::SnapshotInput; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::logical_state::logical_state; + +pub(crate) async fn node_sql_boundaries( + store: &ControlStore, + (client, session): (&Client, &BrowserSession), + (location, snapshot_id): (Uuid, Uuid), + (input, data): (&SnapshotInput, &Value), +) { + let path = format!("/api/control/locations/{location}/targets/development/nodes/orders/sql"); + for other in [ + path.replace("development", "preview"), + path.replace(&location.to_string(), &Uuid::now_v7().to_string()), + ] { + assert_eq!( + request(client, session, (Method::GET, &other), (&Value::Null, "")) + .await + .status(), + StatusCode::FORBIDDEN + ); + } + for (key, expected) in [ + ( + "udf:orders", + json!({"snapshot_id":snapshot_id,"node_key":"udf:orders","raw_sql":null,"compiled_sql":data["function_sql"]}), + ), + ("missing-orders", Value::Null), + ("orders+", Value::Null), + ] { + let response = request( + client, + session, + ( + Method::GET, + &path.replace("nodes/orders/", &format!("nodes/{key}/")), + ), + (&Value::Null, ""), + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.json::().await.expect("exact node"), + expected + ); + } + let mut before = logical_state(store).await; + let read = request(client, session, (Method::GET, &path), (&Value::Null, "")).await; + assert_eq!(read.status(), StatusCode::OK); + assert_eq!( + read.json::().await.expect("current SQL"), + json!({"snapshot_id":snapshot_id,"node_key":"orders","raw_sql":data["raw_sql"],"compiled_sql":data["compiled_sql"]}) + ); + let mut after = logical_state(store).await; + assert_ne!( + before.remove("sqb_security_audit"), + after.remove("sqb_security_audit") + ); + assert_eq!( + before, after, + "SQL inspection changes only the security audit" + ); + let transaction = begin(store, TransactionMode::Read) + .await + .expect("audit cut"); + let (transaction, audit) = query(transaction, "SELECT actor,action,granting_role,scope_json,before_json,after_json FROM sqb_security_audit WHERE action='sql.read' ORDER BY storage_id DESC LIMIT 1", vec![]).await.expect("audit"); + commit(transaction).await.expect("close audit cut"); + assert_eq!(audit[0]["granting_role"], "developer"); + assert_eq!(audit[0]["scope_json"]["action"], "view.sql"); + assert_eq!(audit[0]["scope_json"]["nodes"], json!(["orders"])); + assert_eq!( + audit[0]["after_json"], + json!({"snapshot_id":snapshot_id,"found":true}) + ); + assert!( + !json!(audit) + .to_string() + .contains(data["raw_sql"].as_str().expect("raw SQL")) + ); + let fault = TransactionFault { + description: "SQL read requires durable audit", + table: "sqb_security_audit", + operation: "INSERT", + condition: "WHEN NEW.action='sql.read'", + sqlite_body: "SELECT RAISE(ABORT, 'synthetic SQL audit failure')", + postgres_body: "RAISE EXCEPTION 'synthetic SQL audit failure'", + }; + install_fault(store, &fault).await; + let before = logical_state(store).await; + let response = request(client, session, (Method::GET, &path), (&Value::Null, "")).await; + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!( + response.json::().await.expect("sanitized refusal"), + json!({"error":"control-store operation failed"}) + ); + assert_eq!( + logical_state(store).await, + before, + "audit failure discloses no SQL and changes no rows" + ); + remove_fault(store, &fault).await; + let mut updated = input.clone(); + updated.generation += 1; + updated + .nodes + .get_mut("orders") + .expect("orders") + .compiled_sql = Some(data["updated_sql"].as_str().expect("updated SQL").into()); + let current = publish_snapshot(store, &updated) + .await + .expect("new publication"); + let response = request(client, session, (Method::GET, &path), (&Value::Null, "")).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.json::().await.expect("latest SQL"), + json!({"snapshot_id":current.snapshot_id,"node_key":"orders","raw_sql":data["raw_sql"],"compiled_sql":data["updated_sql"]}) + ); +} diff --git a/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs b/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs new file mode 100644 index 0000000000..0c8b545af4 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs @@ -0,0 +1,227 @@ +//! Real stored evidence never substitutes observation clocks for consumed inputs. + +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::SnapshotNode; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::main::{ + ingest_catalog::ingest_catalog, ingest_fingerprints::ingest_fingerprints, + ingest_source_freshness::ingest_source_freshness, + read_person_node_status::read_person_node_status, +}; +use crate::warehouse_metadata::models::{MetadataBatch, NodeStatusQuery}; + +pub(crate) async fn node_status_reads( + store: &ControlStore, + credentials: (&AuditContext, Uuid), + (location, session_id): (Uuid, Uuid), +) { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration"); + let data: Value = serde_json::from_str( + &std::fs::read_to_string(root.join("node_status.json")).expect("fixture"), + ) + .expect("status inputs"); + let mut input = snapshot(location); + input.generation = 2; + input.nodes.insert("model:orders".into(), SnapshotNode { + kind: "model".into(), + definition: json!({"name":"orders","destination":{"database":null,"schema":"main","name":"orders"}, + "planner_identity":{"local_hash":data["model_local_hash"]},"deps":["source:raw_orders"]}), + raw_sql: None, compiled_sql: None, python_source: None, + }); + input.nodes.insert( + "udf:orders".into(), + SnapshotNode { + kind: "udf".into(), + definition: data["same_named_function"].clone(), + raw_sql: None, + compiled_sql: Some("23".into()), + python_source: None, + }, + ); + publish_snapshot(store, &input) + .await + .expect("identity snapshot"); + let options = NodeStatusQuery { + node_key: "model:orders".into(), + }; + let initial = json!( + read_person_node_status( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await + .expect("legacy build") + ); + assert_eq!(initial["definition"]["status"], "unavailable"); + assert_eq!(initial["physical"]["status"], "unavailable"); + let fixture: Value = serde_json::from_str( + &std::fs::read_to_string(root.join("fingerprint_lookup.json")).expect("build fixture"), + ) + .expect("build inputs"); + let mut record = fixture["record"].clone(); + let mut metadata = data["model_metadata"].clone(); + metadata["audit_gate"] = json!({"run_id":"audit-run","outcome":"passed"}); + metadata["migration_fingerprint"] = json!("migration-v1"); + metadata["declared_columns_hash"] = json!("columns-v1"); + record["metadata_json"] = json!(metadata.to_string()); + record["event_id"] = json!("orders-model-identity"); + record["ts"] = json!("2026-01-05T10:00:00Z"); + let batch: MetadataBatch = serde_json::from_value( + json!({"kind":"fingerprints","session_id":session_id,"generation":2,"records":[record]}), + ) + .expect("model build"); + ingest_fingerprints(store, (location, "development"), &batch) + .await + .expect("build ingestion"); + let before = json!( + read_person_node_status( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await + .expect("matched definition without catalog") + ); + assert_eq!( + before["definition"], + json!({"status":"matched","local_hash":data["model_local_hash"]}) + ); + assert_eq!(before["physical"]["status"], "unavailable"); + let catalog: MetadataBatch = serde_json::from_value(json!({"kind":"catalog","session_id":session_id,"generation":2,"records":[data["physical"]], + "scan":{"scan_id":Uuid::now_v7(),"location":{"database":null,"schema":"main"},"complete":false}})).expect("catalog page"); + ingest_catalog(store, (location, "development"), &catalog) + .await + .expect("physical evidence"); + let freshness: MetadataBatch = serde_json::from_value( + json!({"kind":"source_freshness","session_id":session_id,"generation":2,"records":[data["freshness"]]}), + ) + .expect("newer source observation"); + assert!( + serde_json::from_value::>( + freshness.records[0]["observed_at"].clone() + ) + .expect("observed_at") + > serde_json::from_value::>( + batch.records[0]["ts"].clone() + ) + .expect("ts") + ); + ingest_source_freshness(store, (location, "development"), &freshness) + .await + .expect("source observation"); + let after = json!( + read_person_node_status( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await + .expect("independent evidence") + ); + assert_eq!(after["physical"]["status"], "observed"); + assert_eq!(after["physical"]["observation"]["record"], data["physical"]); + assert_eq!(after["definition"], before["definition"]); + assert_eq!(after["source_data"], before["source_data"]); + assert_eq!( + after["source_data"], + json!({"status":"unknown","reason":"consumed-input provenance is not recorded"}) + ); + let mut function_batch = batch.clone(); + function_batch.records[0]["node_type"] = json!("udf"); + function_batch.records[0]["event_id"] = json!("orders-function-build"); + function_batch.records[0]["run_id"] = json!("orders-function-run"); + ingest_fingerprints(store, (location, "development"), &function_batch) + .await + .expect("same-named function build"); + let function_status = json!( + read_person_node_status( + store, + credentials, + ( + location, + "development", + &NodeStatusQuery { + node_key: "udf:orders".into() + } + ), + DefaultRole::None + ) + .await + .expect("function namespace") + ); + assert_eq!(function_status["physical"]["status"], "unavailable"); + assert_eq!(function_status["definition"]["status"], "unavailable"); + assert_eq!(function_status["build"]["run_id"], "orders-function-run"); + assert_eq!(function_status["source_data"], after["source_data"]); + let mut changed = batch.clone(); + metadata["config"]["lookback"] = json!("7d"); + changed.records[0]["metadata_json"] = json!(metadata.to_string()); + changed.records[0]["event_id"] = json!("orders-config-changed"); + changed.records[0]["ts"] = json!( + serde_json::from_value::>(changed.records[0]["ts"].clone()) + .expect("ts") + + chrono::Duration::days(1) + ); + ingest_fingerprints(store, (location, "development"), &changed) + .await + .expect("config-only build"); + let changed_status = json!( + read_person_node_status( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await + .expect("config comparison") + ); + assert_eq!(changed_status["definition"]["status"], "changed"); + assert_eq!( + changed_status["definition"]["current_local_hash"], + data["model_local_hash"] + ); + assert_ne!( + changed_status["definition"]["recorded_local_hash"], + data["model_local_hash"] + ); + assert_eq!(changed_status["source_data"], after["source_data"]); + assert!(!changed_status.to_string().contains("metadata_json")); + assert!(matches!( + read_person_node_status( + store, + credentials, + (location, "preview", &options), + DefaultRole::None + ) + .await, + Err(ControlError::Denied { .. }) + )); + assert!( + read_person_node_status( + store, + credentials, + ( + location, + "development", + &NodeStatusQuery { + node_key: "model:unlisted".into() + } + ), + DefaultRole::None + ) + .await + .expect("unknown node") + .is_none() + ); +} diff --git a/crates/sqlbuild-orchestration/tests/api/oidc/helpers.rs b/crates/sqlbuild-orchestration/tests/api/oidc/helpers.rs new file mode 100644 index 0000000000..4543161662 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/oidc/helpers.rs @@ -0,0 +1,259 @@ +//! Real TLS discovery, JWKS, authorization and code exchange for native HTTP acceptance. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use axum::extract::{Query, State}; +use axum::response::Redirect; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use chrono::{Duration, Utc}; +use openidconnect::core::{ + CoreEdDsaPrivateSigningKey, CoreIdToken, CoreIdTokenClaims, CoreJwsSigningAlgorithm, +}; +use openidconnect::{ + AccessToken, Audience, EmptyAdditionalClaims, IssuerUrl, JsonWebKeyId, Nonce, + PkceCodeChallenge, PkceCodeVerifier, PrivateSigningKey, StandardClaims, SubjectIdentifier, +}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::tests::oidc_test_types::{BrowserFlow, ProviderFault, ProviderState, TestProvider}; + +pub(crate) async fn start_provider() -> TestProvider { + let directory = tempfile::tempdir().expect("disposable provider certificates"); + let certificate = rcgen::generate_simple_self_signed(vec!["localhost".into()]) + .expect("fresh TLS key and certificate"); + let pem = certificate.cert.pem(); + let authority = directory.path().join("orders-ca.pem"); + std::fs::write(&authority, &pem).expect("fixture authority"); + let tls = axum_server::tls_rustls::RustlsConfig::from_pem( + pem.as_bytes().to_vec(), + certificate.signing_key.serialize_pem().into_bytes(), + ) + .await + .expect("TLS configuration"); + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("provider socket"); + listener + .set_nonblocking(true) + .expect("asynchronous TLS listener"); + let port = listener.local_addr().expect("provider address").port(); + let key = |id: &str| { + let key = rcgen::KeyPair::generate_for(&rcgen::PKCS_ED25519).expect("fresh signing key"); + Arc::new( + CoreEdDsaPrivateSigningKey::from_ed25519_pem( + &key.serialize_pem(), + Some(JsonWebKeyId::new(id.into())), + ) + .expect("OIDC signing key"), + ) + }; + let state = ProviderState { + issuer: format!("https://localhost:{port}"), + keys: Arc::new(BTreeMap::from([ + ("orders".into(), key("orders")), + ("customers".into(), key("customers")), + ])), + published_key: Arc::new(Mutex::new("orders".into())), + codes: Arc::new(Mutex::new(BTreeMap::new())), + fault: Arc::new(Mutex::new(ProviderFault::default())), + subject: Arc::new(Mutex::new("orders-person".into())), + }; + let router = Router::new() + .route("/.well-known/openid-configuration", get(discovery)) + .route("/jwks", get(jwks)) + .route("/authorize", get(authorize)) + .route("/token", post(token)) + .with_state(state.clone()); + let handle = axum_server::Handle::new(); + let server = axum_server::from_tcp_rustls(listener, tls) + .expect("TLS listener") + .handle(handle.clone()); + let task = tokio::spawn(server.serve(router.into_make_service())); + let client = reqwest::Client::builder() + .add_root_certificate(reqwest::Certificate::from_pem(pem.as_bytes()).expect("root")) + .redirect(reqwest::redirect::Policy::none()) + .timeout(std::time::Duration::from_secs(5)) + .build() + .expect("verified TLS client"); + TestProvider { + state, + authority, + client, + handle, + task, + directory, + } +} + +pub(crate) async fn stop_provider(provider: TestProvider) { + provider + .handle + .graceful_shutdown(Some(std::time::Duration::from_secs(1))); + provider + .task + .await + .expect("join provider") + .expect("provider shutdown"); + provider + .directory + .close() + .expect("remove disposable provider keys"); +} + +async fn discovery(State(state): State) -> Json { + Json(json!({ + "issuer": state.issuer, + "authorization_endpoint": format!("{}/authorize", state.issuer), + "token_endpoint": format!("{}/token", state.issuer), + "jwks_uri": format!("{}/jwks", state.issuer), + "response_types_supported": ["code"], "subject_types_supported": ["public"], + "id_token_signing_alg_values_supported": ["EdDSA"], + "code_challenge_methods_supported": ["S256"] + })) +} + +async fn jwks(State(state): State) -> Json { + let key = state.published_key.lock().expect("published key"); + Json(json!({"keys": [state.keys[key.as_str()].as_verification_key()]})) +} + +async fn authorize( + State(state): State, + Query(request): Query>, +) -> Redirect { + assert_eq!(request["client_id"], "orders-control"); + assert_eq!(request["response_type"], "code"); + assert_eq!(request["code_challenge_method"], "S256"); + assert!( + request["scope"] + .split_whitespace() + .any(|scope| scope == "openid") + ); + let code = Uuid::now_v7().to_string(); + let mut callback = reqwest::Url::parse(&request["redirect_uri"]).expect("callback URI"); + callback + .query_pairs_mut() + .append_pair("code", &code) + .append_pair("state", &request["state"]); + state.codes.lock().expect("code lock").insert(code, request); + Redirect::to(callback.as_str()) +} + +async fn token(State(state): State, body: String) -> Json { + let form = reqwest::Url::parse(&format!("https://provider.example.com/?{body}")) + .expect("token form") + .query_pairs() + .into_owned() + .collect::>(); + assert_eq!(form["grant_type"], "authorization_code"); + let request = state + .codes + .lock() + .expect("code lock") + .remove(&form["code"]) + .expect("one-use authorization code"); + assert_eq!(form["redirect_uri"], request["redirect_uri"]); + let challenge = PkceCodeChallenge::from_code_verifier_sha256(&PkceCodeVerifier::new( + form["code_verifier"].clone(), + )); + assert_eq!(challenge.as_str(), request["code_challenge"]); + let fault = state.fault.lock().expect("fault lock").clone(); + let claims = CoreIdTokenClaims::new( + IssuerUrl::new(state.issuer.clone()).expect("issuer"), + vec![Audience::new("orders-control".into())], + Utc::now() + Duration::seconds(300), + Utc::now(), + StandardClaims::new(SubjectIdentifier::new( + state.subject.lock().expect("subject").clone(), + )), + EmptyAdditionalClaims {}, + ) + .set_nonce(Some(Nonce::new(request["nonce"].clone()))); + let mut claims = serde_json::to_value(claims).expect("claims JSON"); + claims + .as_object_mut() + .expect("claims object") + .extend(fault.claims); + let claims = serde_json::from_value::(claims).expect("fixture claims"); + let signing = &state.keys[fault.signing_key.as_deref().unwrap_or("orders")]; + let id_token = CoreIdToken::new( + claims, + signing.as_ref(), + CoreJwsSigningAlgorithm::EdDsa, + Some(&AccessToken::new("orders-access".into())), + None, + ) + .expect("signed token"); + let mut response = json!({"token_type":"Bearer", "expires_in":300, + "access_token": "orders-access", + "id_token": id_token.to_string()}); + response + .as_object_mut() + .expect("response") + .extend(fault.response); + for field in fault.remove_fields { + response.as_object_mut().expect("response").remove(&field); + } + Json(response) +} + +pub(crate) async fn login(provider: &TestProvider, url: &str) -> BrowserFlow { + let response = provider + .client + .get(format!("{url}/api/auth/oidc/login")) + .header(reqwest::header::HOST, "control.example.com") + .send() + .await + .expect("login"); + assert_eq!(response.status(), reqwest::StatusCode::SEE_OTHER); + let cookie = response.headers()[reqwest::header::SET_COOKIE] + .to_str() + .expect("cookie"); + for attribute in [ + "Secure", + "HttpOnly", + "SameSite=Lax", + "Path=/", + "Max-Age=300", + ] { + assert!(cookie.contains(attribute), "{attribute}"); + } + assert!(!cookie.contains("Domain=")); + let cookie = cookie.split(';').next().expect("browser binding").into(); + let authorize = response.headers()[reqwest::header::LOCATION] + .to_str() + .expect("authorization URL"); + let response = provider + .client + .get(authorize) + .send() + .await + .expect("provider authorize"); + assert_eq!(response.status(), reqwest::StatusCode::SEE_OTHER); + let callback = reqwest::Url::parse( + response.headers()[reqwest::header::LOCATION] + .to_str() + .expect("callback"), + ) + .expect("callback URL"); + BrowserFlow { cookie, callback } +} + +pub(crate) async fn callback( + provider: &TestProvider, + url: &str, + flow: &BrowserFlow, +) -> reqwest::Response { + let mut endpoint = reqwest::Url::parse(url).expect("API URL"); + endpoint.set_path(flow.callback.path()); + endpoint.set_query(flow.callback.query()); + provider + .client + .get(endpoint) + .header(reqwest::header::HOST, "control.example.com") + .header(reqwest::header::COOKIE, &flow.cookie) + .send() + .await + .expect("callback response") +} diff --git a/crates/sqlbuild-orchestration/tests/api/oidc/test_types.rs b/crates/sqlbuild-orchestration/tests/api/oidc/test_types.rs new file mode 100644 index 0000000000..38a686f66d --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/oidc/test_types.rs @@ -0,0 +1,39 @@ +//! Disposable HTTPS identity-provider state; signing keys are generated for each fixture. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use openidconnect::core::CoreEdDsaPrivateSigningKey; + +#[derive(Clone)] +pub(crate) struct ProviderState { + pub(crate) issuer: String, + pub(crate) keys: Arc>>, + pub(crate) published_key: Arc>, + pub(crate) codes: Arc>>>, + pub(crate) fault: Arc>, + pub(crate) subject: Arc>, +} + +pub(crate) struct TestProvider { + pub(crate) state: ProviderState, + pub(crate) authority: std::path::PathBuf, + pub(crate) client: reqwest::Client, + pub(crate) handle: axum_server::Handle, + pub(crate) task: tokio::task::JoinHandle>, + pub(crate) directory: tempfile::TempDir, +} + +pub(crate) struct BrowserFlow { + pub(crate) cookie: String, + pub(crate) callback: reqwest::Url, +} + +#[derive(Clone, Default, serde::Deserialize)] +#[serde(default)] +pub(crate) struct ProviderFault { + pub(crate) claims: serde_json::Map, + pub(crate) response: serde_json::Map, + pub(crate) remove_fields: Vec, + pub(crate) signing_key: Option, +} diff --git a/crates/sqlbuild-orchestration/tests/api/output/helpers.rs b/crates/sqlbuild-orchestration/tests/api/output/helpers.rs new file mode 100644 index 0000000000..6d9f455567 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/output/helpers.rs @@ -0,0 +1,178 @@ +//! Worker output contract helpers inspect durable bytes on both databases. + +use crate::runs::models::OutputPublication; +use crate::store::main::read_recall_view::read_recall_view; +use crate::store::models::ControlStore; +use crate::worker::errors::WorkerError; +use crate::worker::models::WorkerClient; +use serde_json::json; +use sqlbuild_recall::records::models::{OmittedOutput, OutputChunk, OutputStream}; +use uuid::Uuid; + +pub(crate) fn chunk(sequence: u64, stream: OutputStream, data: &[u8]) -> OutputChunk { + OutputChunk { + sequence, + stream, + occurred_at: "2026-01-02T03:04:05Z".into(), + data: data.to_vec(), + byte_offset: 0, + dropped_before: 0, + omitted: None, + } +} + +pub(crate) async fn stored_bytes(store: &ControlStore, invocation: &str) -> Vec<(i64, Vec)> { + let sql = "SELECT sequence, data FROM recall_output WHERE invocation_id=$1 ORDER BY sequence"; + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!("SQLite store") + }; + let connection = rusqlite::Connection::open(path).expect("inspect SQLite bytes"); + let mut statement = connection.prepare(sql).expect("output query"); + return statement + .query_map([invocation], |row| Ok((row.get(0)?, row.get(1)?))) + .expect("stored rows") + .collect::>() + .expect("decode bytes"); + }; + sqlx_core::query_as::query_as(sql) + .bind(invocation) + .fetch_all(pool) + .await + .expect("inspect PostgreSQL bytes") +} + +pub(crate) async fn output_contract(store: &ControlStore, worker: &WorkerClient, claim_id: Uuid) { + let invocation = format!("worker/{claim_id}"); + let first = chunk(0, OutputStream::Stdout, b"orders\0\xff\n"); + let warning = chunk(1, OutputStream::Stderr, b"warning\xc3\x28"); + let omission = OutputChunk { + omitted: Some(OmittedOutput { + bytes: 43, + sequences: 2, + }), + byte_offset: first.data.len() as u64, + dropped_before: 1, + ..chunk(3, OutputStream::Stdout, b"") + }; + let chunks = [first.clone(), warning.clone(), omission]; + assert_eq!( + worker + .publish_output(&chunks) + .await + .expect("publish exact bytes"), + OutputPublication { + inserted: 3, + duplicates: 0, + conflicts: 0 + } + ); + assert_eq!( + worker + .publish_output(&chunks) + .await + .expect("retry accepted output"), + OutputPublication { + inserted: 0, + duplicates: 3, + conflicts: 0 + } + ); + let mut equivalent = first.clone(); + equivalent.occurred_at = "2026-01-02T04:04:05+01:00".into(); + assert_eq!( + worker + .publish_output(&[equivalent]) + .await + .expect("equivalent timestamp retry") + .duplicates, + 1 + ); + let mut conflict = first.clone(); + conflict.data = b"different".to_vec(); + assert_eq!( + worker + .publish_output(&[conflict]) + .await + .expect("record retry conflict") + .conflicts, + 1 + ); + assert_eq!( + stored_bytes(store, &invocation).await, + vec![ + (0, first.data.clone()), + (1, warning.data.clone()), + (3, vec![]) + ] + ); + let conflicts = read_recall_view(store, ("conflicts", &invocation), 10) + .await + .expect("durable conflicts"); + assert_eq!(conflicts.len(), 1); + let counts = read_recall_view(store, ("output_invocations", &invocation), 10) + .await + .expect("output accounting"); + assert_eq!(counts[0]["stdout_bytes"], json!(first.data.len())); + assert_eq!(counts[0]["stderr_bytes"], json!(warning.data.len())); + assert_eq!(counts[0]["truncated_bytes"], json!(43)); + + let valid = chunk(4, OutputStream::Stdout, b"must roll back"); + let mut invalid = chunk(5, OutputStream::Stderr, b"invalid timestamp"); + invalid.occurred_at = "not-a-time".into(); + for rejected in [ + vec![valid.clone(), invalid], + vec![OutputChunk { + sequence: u64::MAX, + ..valid.clone() + }], + vec![OutputChunk { + sequence: i64::MAX as u64, + ..valid.clone() + }], + vec![OutputChunk { + byte_offset: u64::MAX, + ..valid.clone() + }], + vec![OutputChunk { + dropped_before: u64::MAX, + ..valid.clone() + }], + vec![OutputChunk { + omitted: Some(OmittedOutput { + bytes: 1, + sequences: 0, + }), + ..valid.clone() + }], + vec![valid.clone(); 257], + vec![chunk(4, OutputStream::Stdout, &vec![0; 1_048_577])], + ] { + assert!(matches!( + worker.publish_output(&rejected).await, + Err(WorkerError::Rejected(reqwest::StatusCode::BAD_REQUEST)) + )); + } + assert_eq!( + stored_bytes(store, &invocation).await.len(), + 3, + "invalid batches are atomic" + ); + assert_eq!( + worker + .publish_output(&vec![first; 256]) + .await + .expect("chunk count boundary") + .duplicates, + 256 + ); + let boundary = chunk(4, OutputStream::Stdout, &vec![0; 1_048_576]); + assert_eq!( + worker + .publish_output(&[boundary]) + .await + .expect("byte count boundary") + .inserted, + 1 + ); +} diff --git a/crates/sqlbuild-orchestration/tests/api/person_sessions/helpers.rs b/crates/sqlbuild-orchestration/tests/api/person_sessions/helpers.rs new file mode 100644 index 0000000000..4d07050835 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/person_sessions/helpers.rs @@ -0,0 +1,160 @@ +//! Authenticate real browser transports and observe their durable queue effects. + +use std::collections::{BTreeMap, BTreeSet}; + +use reqwest::{Client, Method, Response, StatusCode, header}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::models::{ + ApiAuthentication, HumanLogin, OidcAuthentication, TrustedProxyAuthentication, +}; +use crate::api::tests::helpers::start_api_with_authentication; +use crate::api::tests::oidc_helpers::{callback, login}; +use crate::api::tests::oidc_test_types::TestProvider; +use crate::api::tests::person_sessions_test_types::BrowserSession; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub(crate) async fn browser_sessions( + store: &ControlStore, + provider: &TestProvider, + person: Uuid, +) -> Vec { + let origin = "https://control.example.com"; + let password = start_api_with_authentication( + store, + ApiAuthentication { + login: HumanLogin::Password(origin.into()), + ..Default::default() + }, + ) + .await; + let proxy = start_api_with_authentication( + store, + ApiAuthentication { + login: HumanLogin::TrustedProxy(TrustedProxyAuthentication { + origin: origin.into(), + trusted_peers: BTreeSet::from(["127.0.0.1".parse().expect("socket peer")]), + identities: BTreeMap::from([("orders-person".into(), person)]), + }), + ..Default::default() + }, + ) + .await; + let oidc = start_api_with_authentication( + store, + ApiAuthentication { + login: HumanLogin::Oidc(OidcAuthentication { + origin: origin.into(), + issuer: provider.state.issuer.clone(), + client_id: "orders-control".into(), + client_secret_env: None, + certificate_authority: Some(provider.authority.clone()), + identities: BTreeMap::from([("orders-person".into(), person)]), + }), + ..Default::default() + }, + ) + .await; + let password_response = provider + .client + .post(format!("{}/api/auth/password/login", password.url)) + .header(header::HOST, "control.example.com") + .header(header::ORIGIN, origin) + .json(&json!({"username":"orders-person", "password":"orders-password-2026"})) + .send() + .await + .expect("password login"); + let flow = login(provider, &oidc.url).await; + let oidc_response = callback(provider, &oidc.url, &flow).await; + let mut sessions = Vec::new(); + for (api, response, identity_path) in [ + (password, password_response, "/identity/principal_id"), + (oidc, oidc_response, "/principal_id"), + ] { + assert_eq!(response.status(), StatusCode::OK); + let cookie = response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .map(|value| value.to_str().expect("cookie")) + .find(|value| value.starts_with("__Host-sqb-session=")) + .expect("browser session") + .split(';') + .next() + .expect("cookie pair") + .to_owned(); + let body: Value = response.json().await.expect("session response"); + assert_eq!(body.pointer(identity_path), Some(&json!(person))); + assert!(body.get("bearer").is_none()); + sessions.push(BrowserSession { + api, + credential: ("cookie", cookie), + csrf: body["csrf_token"].as_str().expect("CSRF").into(), + forgery: ("x-sqb-csrf", "wrong-csrf"), + }); + } + sessions.push(BrowserSession { + api: proxy, + credential: ("x-sqb-user", "orders-person".into()), + csrf: String::new(), + forgery: ("origin", "https://other.example.com"), + }); + sessions +} + +pub(crate) async fn request( + client: &Client, + session: &BrowserSession, + (method, path): (Method, &str), + (body, csrf): (&Value, &str), +) -> Response { + client + .request(method, format!("{}{path}", session.api.url)) + .header(header::HOST, "control.example.com") + .header(header::ORIGIN, "https://control.example.com") + .header(session.credential.0, &session.credential.1) + .header("x-sqb-csrf", csrf) + .json(body) + .send() + .await + .expect("browser operation") +} + +/// The request a cross-site page could forge: its CSRF proof or origin is wrong. +pub(crate) async fn forged_request( + client: &Client, + session: &BrowserSession, + (method, path): (Method, &str), + body: &Value, +) -> Response { + client + .request(method, format!("{}{path}", session.api.url)) + .header(header::HOST, "control.example.com") + .header(header::ORIGIN, "https://control.example.com") + .header(session.credential.0, &session.credential.1) + .header("x-sqb-csrf", &session.csrf) + .header(session.forgery.0, session.forgery.1) + .json(body) + .send() + .await + .expect("forged browser operation") +} + +pub(crate) async fn queue_counts(store: &ControlStore) -> Value { + let transaction = begin(store, TransactionMode::Read) + .await + .expect("inspect queues"); + let (transaction, rows) = query( + transaction, + "SELECT (SELECT count(*) FROM sqb_location_work) AS work, \ + (SELECT count(*) FROM sqb_runs) AS runs, \ + (SELECT count(*) FROM sqb_write_claims) AS claims", + vec![], + ) + .await + .expect("queue counts"); + commit(transaction).await.expect("close inspection"); + json!(rows[0]) +} diff --git a/crates/sqlbuild-orchestration/tests/api/person_sessions/test_types.rs b/crates/sqlbuild-orchestration/tests/api/person_sessions/test_types.rs new file mode 100644 index 0000000000..552889f4b8 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/person_sessions/test_types.rs @@ -0,0 +1,20 @@ +//! Browser session fixtures and independent role-matrix expectations. + +use crate::api::tests::test_types::TestApi; +use crate::credentials::models::BuiltinRole; + +pub(crate) struct BrowserSession { + pub(crate) api: TestApi, + pub(crate) credential: (&'static str, String), + pub(crate) csrf: String, + pub(crate) forgery: (&'static str, &'static str), +} + +#[derive(serde::Deserialize)] +pub(crate) struct SessionRoleCase { + pub(crate) role: Option, + pub(crate) expected_metadata: u16, + pub(crate) expected_connection: u16, + pub(crate) expected_job: u16, + pub(crate) expected_sql: u16, +} diff --git a/crates/sqlbuild-orchestration/tests/api/recall/helpers.rs b/crates/sqlbuild-orchestration/tests/api/recall/helpers.rs new file mode 100644 index 0000000000..d6bf078a67 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/recall/helpers.rs @@ -0,0 +1,411 @@ +//! Exercise native writer delivery and authenticated retries against both physical stores. + +use std::time::Duration; + +use crate::runs::models::{RecallBatch, RecallRecord}; +use crate::store::main::read_recall_view::read_recall_view; +use crate::store::models::ControlStore; +use crate::worker::errors::WorkerError; +use crate::worker::models::WorkerClient; +use serde_json::json; +use sqlbuild_recall::records::main::artifact_rows::artifact_rows; +use sqlbuild_recall::records::main::event_object_row::event_object_row; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use sqlbuild_recall::records::models::{ + ArtifactInput, InvocationContext, OutputStream, RecallRow, RecallValue, RowEnvelope, +}; +use sqlbuild_recall::schema::main::table_spec::table_spec; +use sqlbuild_recall::schema::models::Table; +use sqlbuild_recall::writer::main::start_recall_session::start_recall_session; +use sqlbuild_recall::writer::models::SessionConfig; +use uuid::Uuid; + +use crate::api::tests::output_helpers::stored_bytes; + +fn record(row: RecallRow) -> RecallRecord { + RecallRecord { + table: row.table, + values: table_spec(row.table) + .columns + .iter() + .filter_map(|column| { + row.value(column.name) + .map(|value| (column.name.to_owned(), value.clone())) + }) + .collect(), + } +} + +pub(crate) async fn recall_contract( + store: &ControlStore, + worker: &WorkerClient, + claim_id: Uuid, +) -> RecallBatch { + let envelope = RowEnvelope { + invocation_id: "execution".into(), + invocation_started_at: "2026-01-02T03:04:05Z".into(), + producer_version: "untrusted-version".into(), + }; + let invocation = format!("worker/{claim_id}/execution"); + let event = json!({ + "event_id": "same-event-id", "occurred_at": "2026-01-02T03:04:05Z", + "event_type": "statement_completed", "schema_version": 1, "producer": "executor", + "statement_id": "shared-producer-id", + "payload": {"duration_ms": 12.75, "row_count": 9, "outcome": "success"} + }); + let mut batch = RecallBatch { + invocation_id: envelope.invocation_id.clone(), + records: vec![record( + event_object_row(&envelope, event.as_object().expect("event object")) + .expect("encode event"), + )], + }; + for _ in 0..2 { + let outcome = worker + .publish_recall(&batch) + .await + .expect("retry event batch"); + assert_eq!(outcome.delivered, [(Table::Events, 1)]); + assert!(outcome.conflicts.is_empty()); + } + let events = read_recall_view(store, ("events", &invocation), 10) + .await + .expect("read stored events"); + assert_eq!(events.len(), 1); + assert_eq!(events[0]["duration_ms"], json!(12.75)); + assert_eq!(events[0]["row_count"], json!(9)); + assert_ne!(events[0]["event_id"], json!("same-event-id")); + assert_ne!(events[0]["producer_version"], json!("untrusted-version")); + batch.records[0] + .values + .insert("duration_ms".into(), RecallValue::Real(13.25)); + assert_eq!( + worker + .publish_recall(&batch) + .await + .expect("publish conflicting event") + .conflicts + .len(), + 1 + ); + assert_eq!( + read_recall_view(store, ("events", &invocation), 10) + .await + .expect("read original event")[0]["duration_ms"], + json!(12.75) + ); + assert_eq!( + read_recall_view(store, ("conflicts", &invocation), 10) + .await + .expect("read conflict evidence") + .len(), + 1 + ); + + let mut separate = batch.clone(); + separate.invocation_id = "another-invocation".into(); + assert_eq!( + worker + .publish_recall(&separate) + .await + .expect("publish another invocation") + .delivered, + [(Table::Events, 1)] + ); + assert_eq!( + read_recall_view(store, ("statements", &invocation), 10) + .await + .expect("read first invocation statement")[0]["duration_ms"], + json!(12.75) + ); + assert_eq!( + read_recall_view( + store, + ( + "statements", + &format!("worker/{claim_id}/another-invocation") + ), + 10 + ) + .await + .expect("read second invocation statement")[0]["duration_ms"], + json!(13.25) + ); + let artifact = ArtifactInput { + digest: format!("sha256:{}", sha256_hex(b"orders\0\xff")), + kind: "evidence".into(), + media_type: "application/octet-stream".into(), + content: b"orders\0\xff".to_vec(), + seen_at: "2026-01-02T03:04:05Z".into(), + ref_id: "first".into(), + role: "evidence".into(), + statement_id: None, + resource_id: None, + }; + let [content, reference] = artifact_rows(&envelope, &artifact).map(record); + let artifacts = RecallBatch { + invocation_id: "execution".into(), + records: vec![content.clone(), reference.clone()], + }; + assert_eq!( + worker + .publish_recall(&artifacts) + .await + .expect("publish artifact with proof") + .delivered + .len(), + 2 + ); + let mut later_ref = reference.clone(); + later_ref + .values + .insert("ref_id".into(), RecallValue::Text("second".into())); + assert_eq!( + worker + .publish_recall(&RecallBatch { + invocation_id: "execution".into(), + records: vec![later_ref] + }) + .await + .expect("publish later artifact reference") + .delivered, + [(Table::ArtifactRefs, 1)] + ); + assert_eq!( + read_recall_view(store, ("artifacts", &invocation), 10) + .await + .expect("read both artifact references") + .len(), + 2 + ); + + let mut forged = content; + forged + .values + .insert("content".into(), RecallValue::Blob(b"different".to_vec())); + let mut event_next = batch.records[0].clone(); + event_next.values.insert( + "event_id".into(), + RecallValue::Text("must-roll-back".into()), + ); + let mut forged_envelope = event_next.clone(); + forged_envelope.values.insert( + "invocation_id".into(), + RecallValue::Text("scheduler/other".into()), + ); + let mut wrong_type = event_next.clone(); + wrong_type + .values + .insert("row_count".into(), RecallValue::Text("nine".into())); + let mut missing = event_next.clone(); + missing.values.remove("event_id"); + let mut bad_json = event_next.clone(); + bad_json + .values + .insert("payload".into(), RecallValue::Text("not-json".into())); + let mut bad_timestamp = event_next.clone(); + bad_timestamp + .values + .insert("occurred_at".into(), RecallValue::Text("not-a-time".into())); + for records in [ + vec![event_next.clone(), forged], + vec![event_next.clone(), forged_envelope], + vec![event_next.clone(), wrong_type], + vec![event_next.clone(), missing], + vec![event_next.clone(), bad_json], + vec![event_next.clone(), bad_timestamp], + vec![ + event_next.clone(), + RecallRecord { + table: Table::Meta, + values: Default::default(), + }, + ], + vec![ + event_next.clone(), + RecallRecord { + table: Table::Conflicts, + values: Default::default(), + }, + ], + vec![event_next; 501], + ] { + assert!(matches!( + worker + .publish_recall(&RecallBatch { + invocation_id: "execution".into(), + records + }) + .await, + Err(WorkerError::Rejected(reqwest::StatusCode::BAD_REQUEST)) + )); + } + // A later authorization failure rolls back the preceding valid event, not just validation errors. + assert!(matches!( + worker + .publish_recall(&RecallBatch { + invocation_id: "foreign".into(), + records: vec![batch.records[0].clone(), reference] + }) + .await, + Err(WorkerError::Rejected(reqwest::StatusCode::BAD_REQUEST)) + )); + assert!( + read_recall_view(store, ("events", &format!("worker/{claim_id}/foreign")), 10) + .await + .expect("read rolled-back invocation") + .is_empty() + ); + assert_eq!( + read_recall_view(store, ("events", &invocation), 10) + .await + .expect("read unchanged invocation") + .len(), + 1 + ); + + native_session(store, worker, claim_id).await; + large_batches(store, worker, claim_id).await; + batch +} + +async fn native_session(store: &ControlStore, worker: &WorkerClient, claim_id: Uuid) { + let hosted = worker.recall_store(); + let summary = tokio::task::spawn_blocking(move || { + let session = start_recall_session(SessionConfig { + store: hosted, + envelope: RowEnvelope { invocation_id: "native-writer".into(), invocation_started_at: "2026-01-02T03:04:05Z".into(), producer_version: "test-version".into() }, + context: InvocationContext { command: Some("build".into()), git_dirty: Some(true), ..Default::default() }, + retention_days: 30, shutdown_timeout: Duration::from_secs(10), diagnostics_min_severity: "debug".into(), + max_output_bytes: None, heartbeat_interval: Duration::from_secs(60), setup_budget: Duration::from_secs(2), + maintenance: false, defer_context: false, maintenance_owner: "test-worker".into(), audit_evidence: Default::default(), + }); + session.append_output(OutputStream::Stdout, b"customers\0\xff\n"); + session.append_output(OutputStream::Stderr, b"warning\xc3\x28\n"); + session.publish_sql("select 7 as order_id", Some("statement-1"), Some("orders")); + session.publish_diagnostic(&json!({"occurred_at": "2026-01-02T03:04:05Z", "severity": "warning", "logger": "sqlbuild", "source": "test", "message": "orders warning", "fields": {"count": 3}}).to_string()).expect("queue diagnostic"); + session.publish_event(&json!({"event_id": "native-event", "occurred_at": "2026-01-02T03:04:05Z", "event_type": "statement_completed", "producer": "executor", "schema_version": 1, "payload": {"duration_ms": 2.5}}).to_string()).expect("queue event"); + session.close() + }).await.expect("native writer thread"); + assert!(summary.flush_complete, "{summary:?}"); + assert_eq!(summary.total().accepted, summary.total().delivered); + let invocation = format!("worker/{claim_id}/native-writer"); + let output = stored_bytes(store, &invocation).await; + assert_eq!(output.len(), 2); + assert!( + output + .iter() + .any(|(_, bytes)| bytes == b"customers\0\xff\n") + ); + assert!( + output + .iter() + .any(|(_, bytes)| bytes == b"warning\xc3\x28\n") + ); + assert_eq!( + read_recall_view(store, ("diagnostics", &invocation), 10) + .await + .expect("read diagnostic")[0]["message"], + json!("orders warning") + ); + assert_eq!( + read_recall_view(store, ("events", &invocation), 10) + .await + .expect("read native events") + .len(), + 1 + ); + assert_eq!( + read_recall_view(store, ("artifacts", &invocation), 10) + .await + .expect("read captured SQL")[0]["text"], + json!("select 7 as order_id") + ); + let delivered = read_recall_view(store, ("output_invocations", &invocation), 10) + .await + .expect("read delivery accounting"); + assert_eq!(delivered.len(), 1); + assert!( + delivered[0]["flush_complete"] == json!(1) || delivered[0]["flush_complete"] == json!(true) + ); +} + +async fn large_batches(store: &ControlStore, worker: &WorkerClient, claim_id: Uuid) { + use sqlbuild_recall::writer::models::SessionStore; + use std::time::Instant; + + let SessionStore::Hosted(opener) = worker.recall_store() else { + panic!("hosted store") + }; + tokio::task::spawn_blocking(move || { + let envelope = RowEnvelope { + invocation_id: "backlog".into(), + invocation_started_at: "2026-01-02T03:04:05Z".into(), + producer_version: "test".into(), + }; + let store = opener + .open(&envelope, Instant::now() + Duration::from_secs(2)) + .expect("open native HTTP store"); + // A replacement can expand one pending lifecycle event beyond the queue's batch count. + let events = (0..501) + .map(|index| { + event_object_row(&envelope, json!({ + "event_id": format!("backlog-{index}"), "occurred_at": "2026-01-02T03:04:05Z", + "event_type": "statement_completed", "producer": "executor", "schema_version": 1, + "payload": {"row_count": index} + }).as_object().expect("event object")).expect("encode backlog event") + }) + .collect::>(); + let expanded = store.insert(&events, Instant::now() + Duration::from_secs(30)); + // Six maximum-sized native artifacts fit the 32 MiB queue, but their JSON blobs exceed 64 MiB. + let artifacts = (0..6) + .flat_map(|index| { + let content = vec![b'a' + index; 4 * 1024 * 1024]; + artifact_rows( + &envelope, + &ArtifactInput { + digest: format!("sha256:{}", sha256_hex(&content)), + content, + kind: "evidence".into(), + media_type: "application/octet-stream".into(), + seen_at: "2026-01-02T03:04:05Z".into(), + ref_id: format!("backlog-{index}"), + role: "evidence".into(), + statement_id: None, + resource_id: None, + }, + ) + }) + .collect::>(); + let bytes = store.insert(&artifacts, Instant::now() + Duration::from_secs(30)); + assert!( + expanded.is_ok() && bytes.is_ok(), + "materialized rows: {expanded:?}; wire bytes: {bytes:?}" + ); + assert_eq!( + expanded.expect("publish expanded event batch").delivered, + [(Table::Events, 501)] + ); + assert_eq!( + bytes.expect("publish large artifact batch").delivered, + [(Table::Artifacts, 6), (Table::ArtifactRefs, 6)] + ); + }) + .await + .expect("join backlog writer"); + let invocation = format!("worker/{claim_id}/backlog"); + assert_eq!( + read_recall_view(store, ("events", &invocation), 1000) + .await + .expect("read full event backlog") + .len(), + 501 + ); + assert_eq!( + read_recall_view(store, ("artifacts", &invocation), 10) + .await + .expect("read large artifacts") + .len(), + 6 + ); +} diff --git a/crates/sqlbuild-orchestration/tests/api/statistics/eligibility/helpers.rs b/crates/sqlbuild-orchestration/tests/api/statistics/eligibility/helpers.rs new file mode 100644 index 0000000000..84a61e7b88 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/statistics/eligibility/helpers.rs @@ -0,0 +1,108 @@ +//! Partial namespace sampling cannot hide unsampled relations after a reader restart. + +use chrono::Utc; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::main::{ + ingest_statistics::ingest_statistics, read_person_statistics::read_person_statistics, + read_statistics_due::read_statistics_due, +}; +use crate::warehouse_metadata::models::{ + MetadataBatch, StatisticsDueRead, StatisticsQuery, WarehouseStateLocation, +}; + +pub(crate) async fn statistics_eligibility( + store: &ControlStore, + credentials: (&AuditContext, Uuid), + (location, session_id): (Uuid, Uuid), + record: &Value, +) { + let mut request = StatisticsDueRead { + session_id, + generation: 1, + location: WarehouseStateLocation { + database: None, + schema: "main".into(), + }, + after: None, + limit: 100, + }; + let first = read_statistics_due(store, (location, "development"), &request) + .await + .expect("due first page"); + assert_eq!(first.names.len(), 100); + let mut sample = record.clone(); + sample["name"] = json!(first.names[0]); + sample["observed_at"] = json!(Utc::now()); + sample["row_count"] = Value::Null; + sample["byte_count"] = json!(0); + let batch: MetadataBatch = serde_json::from_value( + json!({"kind":"statistics","session_id":session_id,"generation":1,"records":[sample]}), + ) + .expect("one sampled relation"); + assert!( + ingest_statistics(store, (location, "development"), &batch) + .await + .expect("partial sampling") + .applied + ); + let options = StatisticsQuery { + database: None, + schema: "main".into(), + name: first.names[0].clone(), + after_observed_at: Some( + serde_json::from_value(batch.records[0]["observed_at"].clone()).expect("observed_at"), + ), + after_event_id: Some(Uuid::nil()), + limit: None, + }; + let observed = read_person_statistics( + store, + credentials, + (location, "development", &options), + (DefaultRole::None, Default::default()), + ) + .await + .expect("unknown rows and zero bytes remain distinct"); + let current = observed.records.last().expect("current-hour sample"); + assert_eq!(current.record.row_count, None); + assert_eq!(current.record.byte_count, Some(0)); + assert_eq!( + current.record.byte_count_basis.as_deref(), + Some("synthetic_catalog_bytes") + ); + let remaining = read_statistics_due(store, (location, "development"), &request) + .await + .expect("restart from committed per-relation progress"); + assert_eq!(remaining.names, first.names[1..]); + assert_eq!(remaining.next_after, first.next_after); + request.after = remaining.next_after; + let second = read_statistics_due(store, (location, "development"), &request) + .await + .expect("next page"); + assert_eq!(second.names.len(), 100); + request.after = second.next_after; + let last = read_statistics_due(store, (location, "development"), &request) + .await + .expect("last page"); + assert_eq!(last.names.len(), 5); + assert!(last.next_after.is_none()); + request.after = None; + request.location.database = Some("orders_data".into()); + assert_eq!( + read_statistics_due(store, (location, "development"), &request) + .await + .expect("exact database") + .names, + ["orders_🧾_000"] + ); + request.generation = 2; + assert!( + read_statistics_due(store, (location, "development"), &request) + .await + .is_err() + ); +} diff --git a/crates/sqlbuild-orchestration/tests/api/statistics/helpers.rs b/crates/sqlbuild-orchestration/tests/api/statistics/helpers.rs new file mode 100644 index 0000000000..eb74eba6b0 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/statistics/helpers.rs @@ -0,0 +1,238 @@ +//! Samples are immutable hourly estimates, exact-scoped and independent of structural facts. + +use chrono::{Duration, Timelike, Utc}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::warehouse_metadata::main::{ + ingest_statistics::ingest_statistics, read_person_statistics::read_person_statistics, +}; +use crate::warehouse_metadata::models::{MetadataBatch, StatisticsQuery}; + +pub(crate) async fn statistics_reads( + store: &ControlStore, + credentials: (&AuditContext, Uuid), + (location, other, session_id): (Uuid, Uuid, Uuid), +) { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json"); + let data: Value = serde_json::from_str(&std::fs::read_to_string(path).expect("fixture")) + .expect("statistics fixture"); + let now = Utc::now() + .with_minute(0) + .expect("minute") + .with_second(0) + .expect("second") + .with_nanosecond(0) + .expect("nanosecond"); + let transaction = begin(store, TransactionMode::Read) + .await + .expect("facts before"); + let (transaction, before) = query( + transaction, + "SELECT COUNT(*) AS total FROM recall_events", + vec![], + ) + .await + .expect("fact count"); + commit(transaction).await.expect("facts read"); + let records: Vec = (0..data["retained_hours"].as_i64().expect("hours")) + .map(|index| { + let mut record = data["record"].clone(); + record["observed_at"] = json!(now - Duration::hours(205 - index)); + record + }) + .collect(); + for records in records.chunks(100) { + let batch: MetadataBatch = serde_json::from_value( + json!({"kind":"statistics","session_id":session_id,"generation":1,"records":records}), + ) + .expect("batch"); + assert!( + ingest_statistics(store, (location, "development"), &batch) + .await + .expect("ingest") + .applied + ); + assert!( + !ingest_statistics(store, (location, "development"), &batch) + .await + .expect("lost acknowledgement") + .applied + ); + } + let mut options = StatisticsQuery { + database: None, + schema: "main".into(), + name: "orders_🧾_000".into(), + after_observed_at: None, + after_event_id: None, + limit: Some(u32::MAX), + }; + let mut actual = Vec::new(); + for expected in [100, 100, 5] { + let page = read_person_statistics( + store, + credentials, + (location, "development", &options), + (DefaultRole::None, Default::default()), + ) + .await + .expect("page"); + assert_eq!(page.records.len(), expected); + options.after_observed_at = page.next_after.as_ref().map(|cursor| cursor.observed_at); + options.after_event_id = page.next_after.map(|cursor| cursor.event_id); + actual.extend(page.records); + } + assert!(options.after_event_id.is_none()); + let actual_times: Vec<_> = actual + .iter() + .map(|observation| observation.record.observed_at) + .collect(); + let expected_times: Vec<_> = (1..=205) + .rev() + .map(|hours| now - Duration::hours(hours)) + .collect(); + assert_eq!(actual_times, expected_times); + assert!( + actual + .iter() + .all(|observation| observation.record.row_count == Some(7) + && observation.record.byte_count == Some(4096)) + ); + let mut replacement: MetadataBatch = serde_json::from_value( + json!({"kind":"statistics","session_id":session_id,"generation":1,"records":[records[0]]}), + ) + .expect("replacement"); + replacement.records[0]["row_count"] = json!(999); + replacement.records[0]["observed_at"] = json!( + serde_json::from_value::>( + replacement.records[0]["observed_at"].clone() + ) + .expect("observed_at") + + Duration::seconds(3599) + ); + assert!( + !ingest_statistics(store, (location, "development"), &replacement) + .await + .expect("same hour first wins") + .applied + ); + for (database, schema, name) in [ + (Some("orders_data"), "main", "orders_🧾_000"), + (None, "landing", "orders_🧾_000"), + (None, "main", "Orders"), + ] { + let mut scoped = replacement.clone(); + scoped.records[0]["state_database"] = json!(database); + scoped.records[0]["state_schema"] = json!(schema); + scoped.records[0]["name"] = json!(name); + assert!( + ingest_statistics(store, (location, "development"), &scoped) + .await + .expect("separate exact identity") + .applied + ); + options.database = database.map(String::from); + options.schema = schema.into(); + options.name = name.into(); + assert_eq!( + read_person_statistics( + store, + credentials, + (location, "development", &options), + (DefaultRole::None, Default::default()) + ) + .await + .expect("exact scope") + .records + .len(), + 1 + ); + } + options.database = None; + options.schema = "main".into(); + options.name = "orders_🧾_000".into(); + options.limit = Some(1); + let first = read_person_statistics( + store, + credentials, + (location, "development", &options), + (DefaultRole::None, Default::default()), + ) + .await + .expect("first wins"); + assert_eq!(first.records[0].record.row_count, Some(7)); + for scope in [(other, "development"), (location, "preview")] { + assert!(matches!( + read_person_statistics( + store, + credentials, + (scope.0, scope.1, &options), + (DefaultRole::None, Default::default()) + ) + .await, + Err(ControlError::Denied { .. }) + )); + } + options.after_event_id = Some(Uuid::now_v7()); + assert!(matches!( + read_person_statistics( + store, + credentials, + (location, "development", &options), + (DefaultRole::None, Default::default()) + ) + .await, + Err(ControlError::Invalid(_)) + )); + for (generation, reader) in [(2, session_id), (1, Uuid::now_v7())] { + let mut fenced = replacement.clone(); + fenced.generation = generation; + fenced.session_id = reader; + assert!( + ingest_statistics(store, (location, "development"), &fenced) + .await + .is_err() + ); + } + let mut invalid = replacement.clone(); + invalid.records[0]["byte_count_basis"] = Value::Null; + assert!(matches!( + ingest_statistics(store, (location, "development"), &invalid).await, + Err(ControlError::Invalid(_)) + )); + let transaction = begin(store, TransactionMode::Read) + .await + .expect("facts after"); + let (transaction, after) = query( + transaction, + "SELECT COUNT(*) AS total FROM recall_events", + vec![], + ) + .await + .expect("fact count"); + commit(transaction).await.expect("facts read"); + assert_eq!( + before, after, + "statistics must not append structural change facts" + ); + crate::api::tests::statistics_eligibility::statistics_eligibility( + store, + credentials, + (location, session_id), + &data["record"], + ) + .await; + crate::api::tests::statistics_retention::statistics_retention( + store, + credentials, + (location, session_id), + &data, + ) + .await; +} diff --git a/crates/sqlbuild-orchestration/tests/api/statistics/retention/helpers.rs b/crates/sqlbuild-orchestration/tests/api/statistics/retention/helpers.rs new file mode 100644 index 0000000000..3f63670636 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/statistics/retention/helpers.rs @@ -0,0 +1,161 @@ +//! Bounded global expiration does not depend on catalog or namespace survival. + +use chrono::{Duration, Utc}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::main::prune_statistics::prune_statistics; +use crate::warehouse_metadata::main::{ + ingest_statistics::ingest_statistics, read_person_statistics::read_person_statistics, +}; +use crate::warehouse_metadata::models::{MetadataBatch, StatisticsQuery, StatisticsRetention}; + +pub(crate) async fn statistics_retention( + store: &ControlStore, + credentials: (&AuditContext, Uuid), + (location, session_id): (Uuid, Uuid), + data: &Value, +) { + let records: Vec = (0..data["expired_samples"].as_i64().expect("expired samples")) + .map(|index| { + let mut record = data["record"].clone(); + record["state_schema"] = json!("removed_namespace"); + record["name"] = json!(format!("inventory_{index}")); + record["observed_at"] = json!(Utc::now() - Duration::days(91)); + record + }) + .collect(); + for records in records.chunks(100) { + let batch: MetadataBatch = serde_json::from_value( + json!({"kind":"statistics","session_id":session_id,"generation":1,"records":records}), + ) + .expect("expired batch"); + assert!( + ingest_statistics(store, (location, "development"), &batch) + .await + .expect("expired sample") + .applied + ); + } + let options = StatisticsQuery { + database: None, + schema: "removed_namespace".into(), + name: "inventory_0".into(), + after_observed_at: None, + after_event_id: None, + limit: None, + }; + assert!( + read_person_statistics( + store, + credentials, + (location, "development", &options), + (DefaultRole::None, Default::default()) + ) + .await + .expect("expired rows hidden before cleanup") + .records + .is_empty() + ); + for expected in [100, 100, 5, 0] { + assert_eq!( + prune_statistics(store, StatisticsRetention::default()) + .await + .expect("global bounded expiration"), + expected + ); + } + let options = StatisticsQuery { + schema: "main".into(), + name: "orders_🧾_000".into(), + ..options + }; + assert_eq!( + read_person_statistics( + store, + credentials, + (location, "development", &options), + (DefaultRole::None, Default::default()) + ) + .await + .expect("retained points survive") + .records + .len(), + 100 + ); + configured_retention(store, credentials, (location, session_id), data).await; +} + +/// A shorter installation horizon hides and then deletes samples the default would keep. +async fn configured_retention( + store: &ControlStore, + credentials: (&AuditContext, Uuid), + (location, session_id): (Uuid, Uuid), + data: &Value, +) { + assert!(StatisticsRetention::days(0).is_err()); + let horizon = StatisticsRetention::days(5).expect("five-day horizon"); + let records: Vec = [10, 2] + .into_iter() + .map(|age| { + let mut record = data["record"].clone(); + record["name"] = json!("customers_horizon"); + record["observed_at"] = json!(Utc::now() - Duration::days(age)); + record + }) + .collect(); + let batch: MetadataBatch = serde_json::from_value( + json!({"kind":"statistics","session_id":session_id,"generation":1,"records":records}), + ) + .expect("horizon batch"); + assert!( + ingest_statistics(store, (location, "development"), &batch) + .await + .expect("horizon samples") + .applied + ); + let options = StatisticsQuery { + database: None, + schema: "main".into(), + name: "customers_horizon".into(), + after_observed_at: None, + after_event_id: None, + limit: None, + }; + let visible = async |retention| { + read_person_statistics( + store, + credentials, + (location, "development", &options), + (DefaultRole::None, retention), + ) + .await + .expect("horizon read") + .records + .len() + }; + assert_eq!(visible(StatisticsRetention::default()).await, 2); + assert_eq!(visible(horizon).await, 1); + assert_eq!( + prune_statistics(store, StatisticsRetention::default()) + .await + .expect("default keeps both"), + 0 + ); + let pruned = prune_statistics(store, horizon) + .await + .expect("configured expiration"); + assert!( + (2..100).contains(&pruned), + "the shorter horizon expires both the old sample and older retained hours" + ); + assert_eq!( + prune_statistics(store, horizon) + .await + .expect("configured expiration completes"), + 0 + ); + assert_eq!(visible(StatisticsRetention::default()).await, 1); +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs new file mode 100644 index 0000000000..c46382999f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_catalog_reads.rs @@ -0,0 +1,393 @@ +//! Real ingestion, scoped person grants and namespace keyset pages on both control stores. + +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::tests::test_types::WarehouseMetadataTestCase; +use crate::authorization::main::{ + assign_builtin_role::assign_builtin_role, bootstrap_user::bootstrap_user, +}; +use crate::credentials::main::{ + identities::create_principal::create_principal, sessions::login_password::login_password, + sessions::set_password::set_password, +}; +use crate::credentials::models::{ + AuditContext, BuiltinRole, DefaultRole, PasswordAuthority, PasswordLogin, RoleAssignmentRequest, +}; +use crate::credentials::tests::location_helpers::create_location; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::models::{LocationHealth, LocationHealthReport}; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::warehouse_metadata::main::{ + ingest_catalog::ingest_catalog, read_person_catalog::read_person_catalog, +}; +use crate::warehouse_metadata::models::{CatalogQuery, MetadataBatch}; + +#[test] +fn given_catalog_observations_when_people_page_then_live_grants_and_exact_namespaces_bound_every_result() + { + let test_cases = [ + WarehouseMetadataTestCase { + description: "SQLite catalog reads", + backend: Backend::Sqlite, + expected_observations: 205, + }, + WarehouseMetadataTestCase { + description: "PostgreSQL catalog reads", + backend: Backend::Postgres, + expected_observations: 205, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("local peer")), + }; + let admin = bootstrap_user(store, &context, "orders-admin") + .await + .expect("bootstrap"); + let person = create_principal(store, &context, "orders-reader") + .await + .expect("person"); + let mut sessions = Vec::new(); + for (id, name) in [(admin, "orders-admin"), (person, "orders-reader")] { + set_password( + store, + &context, + PasswordAuthority::Operator(id), + "orders-password-2026", + ) + .await + .expect("password"); + let PasswordLogin::Authenticated(session) = + login_password(store, &context, (name, "orders-password-2026")) + .await + .expect("login") + else { + panic!("authenticated"); + }; + sessions.push(session); + } + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json"); + let data: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("fixture")) + .expect("catalog inputs"); + let expected: Vec = (0..test_case.expected_observations) + .map(|index| format!("orders_🧾_{index:03}")) + .collect(); + for (scope, target) in [ + (location, "development"), + (location, "preview"), + (other, "development"), + ] { + let mut input = snapshot(scope); + input.target = target.into(); + publish_snapshot(store, &input).await.expect("snapshot"); + let session_id = Uuid::now_v7(); + report_location_health( + store, + scope, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: target.into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("reader"); + for (namespace, count) in [ + (&data["read_namespaces"][0], test_case.expected_observations), + (&data["read_namespaces"][1], 1), + (&data["read_namespaces"][2], 1), + ] { + let records: Vec = (0..count) + .map(|index| { + let mut record = data["record"].clone(); + record["name"] = json!(format!("orders_🧾_{index:03}")); + record["state_database"] = namespace["database"].clone(); + record["state_schema"] = namespace["schema"].clone(); + record + }) + .collect(); + for records in records.chunks(100) { + let batch: MetadataBatch = serde_json::from_value(json!({"kind":"catalog","session_id":session_id,"generation":1,"records":records, + "scan":{"scan_id":Uuid::now_v7(),"location":{"database":namespace["database"],"schema":namespace["schema"]},"complete":false}})).expect("batch"); + assert!( + ingest_catalog(store, (scope, target), &batch) + .await + .expect("real catalog ingestion") + .applied + ); + } + } + } + let credentials = (&context, sessions[1].identity.principal_id); + let mut options = CatalogQuery { + database: None, + schema: "main".into(), + after_observed_at: None, + after_event_id: None, + limit: Some(u32::MAX), + }; + assert!(matches!( + read_person_catalog( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await, + Err(ControlError::Denied { .. }) + )); + let mut assignment = RoleAssignmentRequest { + principal_id: person, + role: BuiltinRole::Viewer.into(), + location_id: Some(location), + target: Some("development".into()), + revoke: false, + }; + assign_builtin_role( + store, + (&context, sessions[0].identity.principal_id), + &assignment, + ) + .await + .expect("scoped viewer"); + let reader_session = crate::api::tests::fingerprint_reads_helpers::fingerprint_reads( + store, + credentials, + (location, other), + ) + .await; + crate::api::tests::statistics_helpers::statistics_reads( + store, + credentials, + (location, other, reader_session), + ) + .await; + let mut actual = Vec::new(); + for _ in 0..2 { + let page = read_person_catalog( + store, + credentials, + (location, "development", &options), + DefaultRole::None, + ) + .await + .expect("bounded page"); + assert_eq!(page.records.len(), 100, "{}", test_case.description); + let after = page.next_after.expect("more rows remain"); + options.after_observed_at = Some(after.observed_at); + options.after_event_id = Some(after.event_id); + actual.extend(page.records); + } + let last = read_person_catalog( + store, + credentials, + (location, "development", &options), + DefaultRole::None, + ) + .await + .expect("last page"); + assert_eq!(last.records.len(), 5); + assert!(last.next_after.is_none()); + actual.extend(last.records); + assert_eq!(actual.len(), test_case.expected_observations); + for row in &actual { + assert_eq!(json!(row.record.columns), data["record"]["columns"]); + } + let mut names = actual + .into_iter() + .map(|row| row.record.name) + .collect::>(); + names.sort(); + assert_eq!( + names, expected, + "ties and namespace exclusions preserve each exact name once" + ); + options.after_observed_at = None; + options.after_event_id = None; + options.limit = Some(0); + assert_eq!( + read_person_catalog( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await + .expect("minimum page") + .records + .len(), + 1 + ); + options.database = Some("orders_data".into()); + let exact = read_person_catalog( + store, + credentials, + (location, "development", &options), + DefaultRole::None, + ) + .await + .expect("exact database"); + assert_eq!(exact.records.len(), 1); + assert_eq!( + exact.records[0].record.state_database.as_deref(), + Some("orders_data") + ); + let after = exact.next_after.expect("full page needs continuation"); + options.after_observed_at = Some(after.observed_at); + options.after_event_id = Some(after.event_id); + let empty = read_person_catalog( + store, + credentials, + (location, "development", &options), + DefaultRole::None, + ) + .await + .expect("exact multiple ends with empty page"); + assert!(empty.records.is_empty()); + assert!(empty.next_after.is_none()); + options.after_observed_at = None; + options.after_event_id = None; + for scope in [(other, "development"), (location, "preview")] { + assert!(matches!( + read_person_catalog( + store, + credentials, + (scope.0, scope.1, &options), + DefaultRole::None + ) + .await, + Err(ControlError::Denied { .. }) + )); + } + options.after_event_id = Some(Uuid::now_v7()); + assert!(matches!( + read_person_catalog( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await, + Err(ControlError::Invalid(_)) + )); + crate::api::tests::node_status_helpers::node_status_reads( + store, + credentials, + (location, reader_session), + ) + .await; + assignment.revoke = true; + assign_builtin_role( + store, + (&context, sessions[0].identity.principal_id), + &assignment, + ) + .await + .expect("revoke"); + assert!(matches!( + crate::warehouse_metadata::main::read_person_statistics::read_person_statistics( + store, + credentials, + ( + location, + "development", + &crate::warehouse_metadata::models::StatisticsQuery { + database: None, + schema: "main".into(), + name: "orders_🧾_000".into(), + after_observed_at: None, + after_event_id: None, + limit: None, + } + ), + (DefaultRole::None, Default::default()), + ) + .await, + Err(ControlError::Denied { .. }) + )); + assert!(matches!( + crate::warehouse_metadata::main::read_person_node_status::read_person_node_status( + store, + credentials, + ( + location, + "development", + &crate::warehouse_metadata::models::NodeStatusQuery { + node_key: "model:orders".into() + } + ), + DefaultRole::None, + ) + .await, + Err(ControlError::Denied { .. }) + )); + let evidence = crate::warehouse_metadata::models::FingerprintQuery { + node_type: "model".into(), + node_name: "orders".into(), + database: None, + schema: Some("main".into()), + name: Some("orders".into()), + }; + assert!(matches!( + crate::warehouse_metadata::main::read_person_fingerprint::read_person_fingerprint( + store, + credentials, + (location, "development", &evidence), + DefaultRole::None, + ) + .await, + Err(ControlError::Denied { .. }) + )); + assert!( + matches!( + read_person_catalog( + store, + credentials, + (location, "development", &options), + DefaultRole::None + ) + .await, + Err(ControlError::Denied { .. }) + ), + "authorization precedes even invalid cursor handling" + ); + options.after_event_id = None; + assert_eq!( + read_person_catalog( + store, + credentials, + (location, "development", &options), + DefaultRole::Viewer + ) + .await + .expect("configured viewer default") + .records + .len(), + 1 + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_changes.rs b/crates/sqlbuild-orchestration/tests/api/test_changes.rs new file mode 100644 index 0000000000..c61c5fa997 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_changes.rs @@ -0,0 +1,426 @@ +//! Real SQLite broadcast and PostgreSQL LISTEN/NOTIFY feed independent HTTP observers. + +use crate::scheduler::tests::contracts::ticks::helpers::calendar_text; +use std::time::Duration; + +use chrono::Utc; +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::changes_helpers::{ + connect, expect_topics, next_topic, reject_snapshot_commit, reset_proxy, + restore_snapshot_commits, +}; +use crate::api::tests::helpers::{read_json, start_api, stop_api}; +use crate::api::tests::test_types::ChangesTestCase; +use crate::credentials::main::tokens::{issue_token::issue_token, revoke_token::revoke_token}; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::{change_trigger::change_trigger, list_triggers::list_triggers}; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; +use crate::runs::main::request_job_run::request_job_run; +use crate::runs::models::JobRunRequest; +use crate::runs::tests::helpers::snapshot; +use crate::scheduler::models::TickDecision; +use crate::scheduler::tests::contracts::helpers::{acquire_lease, evaluate_trigger}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::models::ControlStore; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_independent_http_observers_when_committed_state_changes_then_scoped_sse_refetches_real_state_and_stops_on_revocation() + { + let test_cases = [ + ChangesTestCase { + description: "SQLite committed live updates", + backend: Backend::Sqlite, + expected_generation: 2, + }, + ChangesTestCase { + description: "PostgreSQL replicated live updates", + backend: Backend::Postgres, + expected_generation: 2, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("isolated store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let audit = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let token = issue_token( + store, + &audit, + CredentialScope::Location { location_id }, + Utc::now() + chrono::Duration::hours(1), + ) + .await + .expect("location token"); + let mut input = snapshot(location_id); + input.documents.insert( + "jobs".into(), + json!({"orders": {"description":"Build orders", "selection":["orders"]}, + "daily": {"description":"Refresh orders daily", "selection":["orders"]}}), + ); + input.documents.insert("triggers".into(), json!({"daily_orders": {"description":"Refresh orders", "job_key":"daily", "starts_at":calendar_text(0), + "schedule":{"kind":"calendar", "times":[calendar_text(0),"2030-01-01T00:00:00Z"], "refresh_at":"2029-01-01T00:00:00Z"}}})); + publish_snapshot(store, &input) + .await + .expect("initial snapshot"); + let first = start_api(store).await; + let second = start_api(store).await; + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(30)) + .build() + .expect("HTTP client"); + let scope = (location_id, "development", token.bearer.as_str()); + let mut a = connect(&client, &first, scope).await; + let mut b = connect(&client, &second, scope).await; + let path = format!( + "{}/api/locations/{location_id}/targets/development", + first.url + ); + assert_eq!( + client + .get(format!("{path}/changes")) + .send() + .await + .expect("anonymous request") + .status(), + reqwest::StatusCode::UNAUTHORIZED + ); + assert_eq!( + client + .get(format!( + "{}/api/locations/{other}/targets/development/changes", + first.url + )) + .bearer_auth(&token.bearer) + .send() + .await + .expect("foreign subscription") + .status(), + reqwest::StatusCode::FORBIDDEN + ); + let mut foreign = input.clone(); + foreign.location_id = other; + publish_snapshot(store, &foreign) + .await + .expect("foreign publication"); + foreign.location_id = location_id; + foreign.target = "preview".into(); + publish_snapshot(store, &foreign) + .await + .expect("other target"); + input.generation = 2; + let response = client + .post(format!("{path}/snapshots")) + .bearer_auth(&token.bearer) + .json(&input) + .send() + .await + .expect("HTTP publish"); + assert_eq!(response.status(), reqwest::StatusCode::OK); + let snapshot_topics = [ + format!("snapshot:{location_id}"), + "node_status:development".into(), + ]; + expect_topics(&mut a, &snapshot_topics).await; + expect_topics(&mut b, &snapshot_topics).await; + let current = read_json( + &client, + ( + &format!( + "{}/api/locations/{location_id}/targets/development/snapshots", + second.url + ), + &token.bearer, + ), + ) + .await; + assert_eq!( + current["record"]["generation"], test_case.expected_generation, + "{}: notification observes committed state on the other replica", + test_case.description + ); + + reject_snapshot_commit(store).await; + input.generation = 3; + assert!( + publish_snapshot(store, &input).await.is_err(), + "late deferred constraint rejects COMMIT after all snapshot work" + ); + restore_snapshot_commits(store).await; + let current = read_json(&client, (&format!("{path}/snapshots"), &token.bearer)).await; + assert_eq!(current["record"]["generation"], 2); + let request = JobRunRequest { + run_id: Uuid::now_v7(), + location_id, + target: "development".into(), + job_key: "orders".into(), + reason: "manual verification".into(), + priority: 0, + window: None, + }; + assert!( + request_job_run(store, &audit, &request) + .await + .expect("queue job") + ); + let run_topics = [ + "runs".into(), + format!("run:{}", request.run_id), + "node_status:development".into(), + ]; + expect_topics(&mut a, &run_topics).await; + expect_topics(&mut b, &run_topics).await; + assert!( + !request_job_run(store, &audit, &request) + .await + .expect("idempotent retry") + ); + let trigger_id = list_triggers(store, location_id, "development") + .await + .expect("trigger")[0] + .trigger_id; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("acquire lease") + .expect("scheduler elected"); + let plan = evaluate_trigger(store, &lease, trigger_id) + .await + .expect("tick decision") + .expect("active trigger"); + let TickDecision::Requested { run_id } = + plan.actions.last().expect("latest tick").decision + else { + panic!("new scheduled run") + }; + let tick_topics = [ + "ticks".into(), + "runs".into(), + format!("run:{run_id}"), + "node_status:development".into(), + ]; + expect_topics(&mut a, &tick_topics).await; + expect_topics(&mut b, &tick_topics).await; + change_trigger( + store, + &audit, + &TriggerChangeRequest { + event_id: Uuid::now_v7(), + trigger_id, + actor: audit.actor.clone(), + reason: "pause verification".into(), + change: TriggerChange::Pause, + }, + ) + .await + .expect("pause"); + expect_topics(&mut a, &["ticks".into()]).await; + expect_topics(&mut b, &["ticks".into()]).await; + + assert!( + revoke_token(store, &audit, token.identity.token_id) + .await + .expect("revoke") + ); + assert_eq!( + next_topic(&mut a).await, + None, + "revocation closes an existing stream" + ); + assert_eq!( + next_topic(&mut b).await, + None, + "revocation reaches another replica" + ); + let fresh = issue_token( + store, + &audit, + CredentialScope::Location { location_id }, + Utc::now() + chrono::Duration::hours(1), + ) + .await + .expect("new token"); + let mut live = + connect(&client, &first, (location_id, "development", &fresh.bearer)).await; + let expiring = issue_token( + store, + &audit, + CredentialScope::Location { location_id }, + Utc::now() + chrono::Duration::seconds(1), + ) + .await + .expect("expiring token"); + let mut expires = connect( + &client, + &second, + (location_id, "development", &expiring.bearer), + ) + .await; + tokio::time::sleep( + (expiring.identity.expires_at - Utc::now()) + .to_std() + .expect("token is still active") + + Duration::from_millis(10), + ) + .await; + publish_snapshot(store, &input) + .await + .expect("committed change after expiry"); + expect_topics(&mut live, &snapshot_topics).await; + assert_eq!( + next_topic(&mut expires).await, + None, + "expired streams disclose no subsequent changes" + ); + // Reconnect starts with all, then REST supplies the current state rather than a replay. + let mut reconnected = connect( + &client, + &second, + (location_id, "development", &fresh.bearer), + ) + .await; + let current = read_json(&client, (&format!("{path}/snapshots"), &fresh.bearer)).await; + assert_eq!(current["record"]["generation"], 3); + let long_target = "p".repeat(9_000); + let mut long_stream = + connect(&client, &second, (location_id, &long_target, &fresh.bearer)).await; + input.target = long_target.clone(); + publish_snapshot(store, &input) + .await + .expect("target larger than PostgreSQL's NOTIFY payload limit"); + expect_topics( + &mut long_stream, + &[ + format!("snapshot:{location_id}"), + format!("node_status:{long_target}"), + ], + ) + .await; + stop_api(first).await; + assert_eq!( + next_topic(&mut live).await, + None, + "shutdown drains a live SSE body" + ); + stop_api(second).await; + assert_eq!(next_topic(&mut reconnected).await, None); + assert_eq!(next_topic(&mut long_stream).await, None); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_terminated_postgres_listeners_when_reconnecting_then_http_survives_and_observers_refetch() +{ + let test_cases = [ChangesTestCase { + description: "PostgreSQL listener interruption", + backend: Backend::Postgres, + expected_generation: 2, + }]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("isolated store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let audit = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let token = issue_token( + store, + &audit, + CredentialScope::Location { location_id }, + Utc::now() + chrono::Duration::hours(1), + ) + .await + .expect("token"); + let mut input = snapshot(location_id); + publish_snapshot(store, &input).await.expect("snapshot"); + let proxy = reset_proxy(store).await; + let first = start_api(&proxy.store).await; + let second = start_api(&proxy.store).await; + let client = reqwest::Client::new(); + let scope = (location_id, "development", token.bearer.as_str()); + let mut a = connect(&client, &first, scope).await; + let mut b = connect(&client, &second, scope).await; + let ControlStore::Postgres(pool) = store else { + panic!("PostgreSQL fixture") + }; + let terminated: Vec = sqlx_core::query_scalar::query_scalar( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid<>pg_backend_pid() AND query LIKE 'LISTEN%' AND query LIKE '%' || md5(current_schema()) || '%'", + ).fetch_all(pool).await.expect("interrupt only this fixture's LISTEN connections"); + assert_eq!(terminated, [true, true]); + let path = format!( + "{}/api/locations/{location_id}/targets/development/snapshots", + first.url + ); + let current = read_json(&client, (&path, &token.bearer)).await; + assert_eq!( + current["record"]["generation"], 1, + "HTTP still serves during listener recovery" + ); + expect_topics(&mut a, &["all".into()]).await; + expect_topics(&mut b, &["all".into()]).await; + proxy.reset.send_modify(|epoch| *epoch += 1); + expect_topics(&mut a, &["all".into()]).await; + expect_topics(&mut b, &["all".into()]).await; + let current = read_json(&client, (&path, &token.bearer)).await; + assert_eq!( + current["record"]["generation"], 1, + "HTTP survives a TCP reset" + ); + input.generation = 2; + publish_snapshot(store, &input) + .await + .expect("commit after re-listening"); + let topics = [ + format!("snapshot:{location_id}"), + "node_status:development".into(), + ]; + expect_topics(&mut a, &topics).await; + expect_topics(&mut b, &topics).await; + let mut reconnected = connect(&client, &second, scope).await; + let current = read_json(&client, (&path, &token.bearer)).await; + assert_eq!( + current["record"]["generation"], test_case.expected_generation, + "{}", + test_case.description + ); + stop_api(first).await; + stop_api(second).await; + assert_eq!(next_topic(&mut a).await, None); + assert_eq!(next_topic(&mut b).await, None); + assert_eq!(next_topic(&mut reconnected).await, None); + crate::store::tests::store_helpers::close_store(proxy.store).await; + proxy.shutdown.send(()).expect("stop database proxy"); + proxy.task.await.expect("database proxy stopped"); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_cli_login.rs b/crates/sqlbuild-orchestration/tests/api/test_cli_login.rs new file mode 100644 index 0000000000..9c3597d121 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_cli_login.rs @@ -0,0 +1,397 @@ +//! Real cross-replica exchange, atomic audit rollback and live password/session revocation. + +use futures_util::future::join_all; +use reqwest::{Client, StatusCode}; +use serde_json::{Value, json}; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::api::models::{ApiAuthentication, HumanLogin}; +use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; +use crate::api::tests::test_types::CliLoginTestCase; +use crate::credentials::main::{ + identities::create_principal::create_principal, sessions::login_password::login_password, + sessions::revoke_session::revoke_session, sessions::set_password::set_password, +}; +use crate::credentials::models::{AuditContext, PasswordAuthority, PasswordLogin}; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::logical_state::logical_state; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_browser_approved_cli_login_when_exchange_races_or_authority_changes_then_credentials_stay_single_use_atomic_and_live() + { + let test_cases = [ + CliLoginTestCase { + description: "SQLite single-use browser handoff", + backend: Backend::Sqlite, + expected_exchanges: 1, + }, + CliLoginTestCase { + description: "PostgreSQL single-use browser handoff", + backend: Backend::Postgres, + expected_exchanges: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/cli_login.json"); + let input: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("fixture")) + .expect("inputs"); + let password = input["password"].as_str().expect("password"); + let replacement = input["replacement"].as_str().expect("replacement"); + let origin = input["origin"].as_str().expect("origin"); + let host = origin.strip_prefix("https://").expect("HTTPS"); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("peer")), + }; + let person = create_principal(store, &context, "orders-reader") + .await + .expect("person"); + set_password( + store, + &context, + PasswordAuthority::Operator(person), + password, + ) + .await + .expect("password"); + let PasswordLogin::Authenticated(browser) = + login_password(store, &context, ("orders-reader", password)) + .await + .expect("browser login") + else { + panic!("browser credentials"); + }; + let authentication = || ApiAuthentication { + login: HumanLogin::Password(origin.into()), + ..ApiAuthentication::default() + }; + let first = start_api_with_authentication(store, authentication()).await; + let second = start_api_with_authentication(&replica, authentication()).await; + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert("host", host.parse().expect("host")); + headers.insert("origin", origin.parse().expect("origin")); + let client = Client::builder() + .default_headers(headers) + .build() + .expect("client"); + let secret = input["client_secret"] + .as_str() + .expect("synthetic exchange secret"); + let started = client + .post(format!("{}/api/auth/cli/start", first.url)) + .json(&json!({"client_hash":sha256_hex(secret.as_bytes())})) + .send() + .await + .expect("start"); + assert_eq!(started.status(), StatusCode::OK); + assert_eq!(started.headers()["cache-control"], "no-store"); + let flow: Value = started.json().await.expect("handoff"); + let code = flow["browser_code"].as_str().expect("browser code"); + let flow_id = code.split_once('.').expect("flow ID").0; + let exchange = json!({"flow_id":flow_id,"client_secret":secret}); + assert_eq!( + client + .post(format!("{}/api/auth/cli/claim", second.url)) + .json(&exchange) + .send() + .await + .expect("pending") + .status(), + StatusCode::ACCEPTED + ); + let approval = json!({"browser_code":code,"approve":true}); + let cookie = format!("__Host-sqb-session={}", browser.bearer); + assert_eq!( + client + .post(format!("{}/api/auth/cli/approve", second.url)) + .header("cookie", &cookie) + .json(&approval) + .send() + .await + .expect("missing CSRF") + .status(), + StatusCode::UNAUTHORIZED + ); + let cancelled: Value = client + .post(format!("{}/api/auth/cli/start", first.url)) + .json(&json!({"client_hash":sha256_hex(secret.as_bytes())})) + .send() + .await + .expect("cancel start") + .json() + .await + .expect("cancel flow"); + let cancelled_code = cancelled["browser_code"].as_str().expect("cancel code"); + assert_eq!( + client + .post(format!("{}/api/auth/cli/approve", second.url)) + .header("cookie", &cookie) + .header("x-sqb-csrf", &browser.csrf) + .json(&json!({"browser_code":cancelled_code,"approve":false})) + .send() + .await + .expect("cancel") + .status(), + StatusCode::NO_CONTENT + ); + assert_eq!(client.post(format!("{}/api/auth/cli/claim", first.url)) + .json(&json!({"flow_id":cancelled_code.split_once('.').expect("cancel ID").0,"client_secret":secret})) + .send().await.expect("cancel claim").status(), StatusCode::UNAUTHORIZED); + let expired: Value = client + .post(format!("{}/api/auth/cli/start", first.url)) + .json(&json!({"client_hash":sha256_hex(secret.as_bytes())})) + .send() + .await + .expect("expiry start") + .json() + .await + .expect("expiry flow"); + let expired_code = expired["browser_code"].as_str().expect("expiry code"); + let expired_id = + Uuid::parse_str(expired_code.split_once('.').expect("expiry ID").0).expect("ID"); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("expiry fence"); + let expired_at = transaction.now - chrono::Duration::seconds(1); + let (transaction, _) = query( + transaction, + "UPDATE sqb_cli_login_flows SET expires_at=$2 WHERE flow_id=$1", + vec![Parameter::Id(expired_id), Parameter::Time(expired_at)], + ) + .await + .expect("elapsed expiry"); + commit(transaction).await.expect("commit expiry"); + assert_eq!( + client + .post(format!("{}/api/auth/cli/approve", second.url)) + .header("cookie", &cookie) + .header("x-sqb-csrf", &browser.csrf) + .json(&json!({"browser_code":expired_code,"approve":true})) + .send() + .await + .expect("expired approval") + .status(), + StatusCode::UNAUTHORIZED + ); + assert_eq!( + client + .post(format!("{}/api/auth/cli/claim", first.url)) + .json(&json!({"flow_id":expired_id,"client_secret":secret})) + .send() + .await + .expect("expired claim") + .status(), + StatusCode::UNAUTHORIZED + ); + let approved = client + .post(format!("{}/api/auth/cli/approve", second.url)) + .header("cookie", &cookie) + .header("x-sqb-csrf", &browser.csrf) + .json(&approval) + .send() + .await + .expect("approve"); + assert_eq!(approved.status(), StatusCode::NO_CONTENT); + assert!(!approved.headers().contains_key("set-cookie")); + assert!(approved.bytes().await.expect("approval body").is_empty()); + let wrong = json!({"flow_id":flow_id,"client_secret":"not-the-cli-secret"}); + assert_eq!( + client + .post(format!("{}/api/auth/cli/claim", first.url)) + .json(&wrong) + .send() + .await + .expect("wrong exchange") + .status(), + StatusCode::UNAUTHORIZED + ); + let fault = TransactionFault { + description: "CLI exchange audit rollback", + table: "sqb_security_audit", + operation: "INSERT", + condition: "WHEN NEW.action='cli.login.exchanged'", + sqlite_body: "SELECT RAISE(ABORT, 'CLI exchange audit failed')", + postgres_body: "RAISE EXCEPTION 'CLI exchange audit failed'", + }; + install_fault(store, &fault).await; + let before = logical_state(store).await; + assert_eq!( + client + .post(format!("{}/api/auth/cli/claim", first.url)) + .json(&exchange) + .send() + .await + .expect("failed exchange") + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + assert_eq!( + logical_state(store).await, + before, + "failed exchange preserves every logical row" + ); + remove_fault(store, &fault).await; + let attempts = join_all((0..4).map(|index| { + let client = &client; + let exchange = &exchange; + let url = [&first.url, &second.url][index % 2]; + async move { + client + .post(format!("{url}/api/auth/cli/claim")) + .json(exchange) + .send() + .await + .expect("concurrent exchange") + } + })) + .await; + assert_eq!( + attempts + .iter() + .filter(|response| response.status() == StatusCode::OK) + .count(), + test_case.expected_exchanges, + "{}", + test_case.description + ); + assert_eq!( + attempts + .iter() + .filter(|response| response.status() == StatusCode::UNAUTHORIZED) + .count(), + 3 + ); + let issued: Value = attempts + .into_iter() + .find(|response| response.status() == StatusCode::OK) + .expect("one credential") + .json() + .await + .expect("CLI credential"); + assert_eq!(issued["identity"]["principal_id"], person.to_string()); + let authorization = format!("Session {}", issued["bearer"].as_str().expect("bearer")); + for url in [&first.url, &second.url] { + assert_eq!( + client + .get(format!("{url}/api/auth/session")) + .header("authorization", &authorization) + .send() + .await + .expect("CLI reload transport") + .status(), + StatusCode::OK + ); + assert_eq!( + client + .get(format!("{url}/api/control/audit")) + .header("authorization", &authorization) + .send() + .await + .expect("no implicit grant") + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + client + .post(format!("{url}/api/admin/roles")) + .header("authorization", &authorization) + .json(&json!({"principal_id": person, "role": "viewer", + "location_id": null, "target": null, "revoke": false})) + .send() + .await + .expect("CLI writes need no CSRF, only permission") + .status(), + StatusCode::FORBIDDEN + ); + } + assert_eq!( + client + .get(format!("{}/api/auth/session", first.url)) + .header("authorization", &authorization) + .header("cookie", &cookie) + .send() + .await + .expect("ambiguous transport") + .status(), + StatusCode::UNAUTHORIZED + ); + let started: Value = client + .post(format!("{}/api/auth/cli/start", first.url)) + .json(&json!({"client_hash":sha256_hex(secret.as_bytes())})) + .send() + .await + .expect("next start") + .json() + .await + .expect("next flow"); + let next_code = started["browser_code"].as_str().expect("next code"); + assert_eq!( + client + .post(format!("{}/api/auth/cli/approve", first.url)) + .header("cookie", &cookie) + .header("x-sqb-csrf", &browser.csrf) + .json(&json!({"browser_code":next_code,"approve":true})) + .send() + .await + .expect("next approve") + .status(), + StatusCode::NO_CONTENT + ); + let (transaction, _) = revoke_session( + begin(store, TransactionMode::Write).await.expect("write"), + &context, + (person, Some(browser.identity.session_id)), + ) + .await + .expect("revoke browser"); + commit(transaction).await.expect("commit revoke"); + assert_eq!(client.post(format!("{}/api/auth/cli/claim", second.url)) + .json(&json!({"flow_id":next_code.split_once('.').expect("next ID").0,"client_secret":secret})) + .send().await.expect("revoked authorizer").status(), StatusCode::UNAUTHORIZED); + set_password( + store, + &context, + PasswordAuthority::Operator(person), + replacement, + ) + .await + .expect("rotate password"); + for url in [&first.url, &second.url] { + assert_eq!( + client + .get(format!("{url}/api/auth/session")) + .header("authorization", &authorization) + .send() + .await + .expect("live password revocation") + .status(), + StatusCode::UNAUTHORIZED + ); + } + stop_api(first).await; + stop_api(second).await; + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs new file mode 100644 index 0000000000..0eecbe0f8a --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -0,0 +1,604 @@ +use std::collections::BTreeMap; +use std::time::Duration; + +use crate::api::errors::ClientError; +use crate::credentials::main::tokens::issue_token::issue_token; +use crate::credentials::main::tokens::revoke_token::revoke_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunExecution, RunStatus}; +use crate::runs::tests::helpers::request_run; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::next_remote_generation::next_remote_generation; +use crate::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; +use crate::snapshots::main::verify_execution::verify_execution; +use crate::snapshots::models::{SnapshotInput, SnapshotRecord}; +use crate::store::main::migrate_store::migrate_store; +use chrono::Utc; +use reqwest::StatusCode; +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::test_types::ApiContractTestCase; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retries_and_terminal_expiry_are_enforced() + { + let test_cases = [ + ApiContractTestCase { + description: "SQLite worker and location HTTP contract", + backend: Backend::Sqlite, + expected_status: RunStatus::Cancelled, + }, + ApiContractTestCase { + description: "PostgreSQL worker and location HTTP contract", + backend: Backend::Postgres, + expected_status: RunStatus::Cancelled, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("another location"); + let audit = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let expiry = Utc::now() + chrono::Duration::hours(1); + let location_token = issue_token( + store, + &audit, + CredentialScope::Location { location_id }, + expiry, + ) + .await + .expect("location credential"); + let api = start_api(store).await; + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .expect("HTTP client"); + let path = format!( + "{}/api/locations/{location_id}/targets/development/snapshots", + api.url + ); + let response = client + .post(&path) + .header("Content-Type", "application/json") + .body("invalid-json") + .send() + .await + .expect("unauthenticated request"); + assert_eq!( + response.status(), + StatusCode::UNAUTHORIZED, + "authentication precedes JSON parsing" + ); + assert_eq!(response.headers()["cache-control"], "no-store"); + assert_eq!(response.headers()["www-authenticate"], "Bearer"); + assert_eq!( + response + .json::() + .await + .expect("unaudited rejection"), + json!({"error": "valid bearer credential required"}), + "a response without a durable decision must not invent an ID" + ); + let openapi: serde_json::Value = client + .get(format!("{}/api/openapi.json", api.url)) + .send() + .await + .expect("OpenAPI response") + .json() + .await + .expect("OpenAPI JSON"); + assert_eq!( + openapi["components"]["securitySchemes"]["bearer"]["scheme"], + "bearer" + ); + assert_eq!( + openapi["components"]["securitySchemes"]["cliSession"]["name"], + "Authorization" + ); + assert_eq!( + openapi["paths"]["/api/control/runs"]["post"]["security"], + json!([{"session": []}, {"cliSession": []}]) + ); + assert_eq!( + openapi["paths"]["/api/auth/cli/approve"]["post"]["security"], + json!([{"session": []}, {"proxy": []}]), + "browser authorization cannot use a CLI session header" + ); + assert!(openapi["components"]["schemas"]["SnapshotInput"].is_object()); + assert!(openapi["components"]["schemas"]["RunChangeRequest"].is_object()); + assert!(openapi["components"]["schemas"]["RunExecution"].is_object()); + assert_eq!( + next_remote_generation( + &api.url, + &location_token.bearer, + location_id, + "development" + ) + .await + .expect("first generation"), + 1 + ); + let input = snapshot(location_id); + let published = publish_remote_snapshot(&api.url, &location_token.bearer, &input) + .await + .expect("publish retained native data"); + let metadata: SnapshotRecord = client + .get(format!("{path}/metadata")) + .bearer_auth(&location_token.bearer) + .send() + .await + .expect("metadata response") + .json() + .await + .expect("metadata record without project bodies"); + assert_eq!(metadata, published); + assert_eq!( + next_remote_generation( + &api.url, + &location_token.bearer, + location_id, + "development" + ) + .await + .expect("reconcile published generation"), + 2 + ); + assert!(matches!( + next_remote_generation(&api.url, &location_token.bearer, other, "development") + .await, + Err(ClientError::Rejected(StatusCode::FORBIDDEN)) + )); + let response = client + .get(&path) + .bearer_auth(&location_token.bearer) + .send() + .await + .expect("read published snapshot through API"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()["cache-control"], "no-store"); + drop(response); + assert!(matches!( + publish_remote_snapshot(&api.url, &location_token.bearer, &input).await, + Err(ClientError::Rejected(StatusCode::CONFLICT)) + )); + assert!(matches!( + publish_remote_snapshot(&api.url, &location_token.bearer, &snapshot(other)).await, + Err(ClientError::Rejected(StatusCode::FORBIDDEN)) + )); + let response = client + .post(&path) + .bearer_auth(&location_token.bearer) + .json(&input) + .send() + .await + .expect("stale publish"); + assert_eq!(response.status(), StatusCode::CONFLICT); + let response = client + .post(format!( + "{}/api/locations/{other}/targets/development/snapshots", + api.url + )) + .bearer_auth(&location_token.bearer) + .json(&snapshot(other)) + .send() + .await + .expect("cross-location publish"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let current: serde_json::Value = client + .get(&path) + .bearer_auth(&location_token.bearer) + .send() + .await + .expect("current snapshot") + .json() + .await + .expect("snapshot JSON"); + assert_eq!( + current["record"]["snapshot_id"], + published.snapshot_id.to_string() + ); + assert_eq!(current["nodes"]["orders"]["compiled_sql"], "select 1 as id"); + for image_id in [None, Some("sha256:unpublished-orders-image".into())] { + let mut mismatched = + request((location_id, published.snapshot_id), 1, 0, &["orders"]); + mismatched.image_id = image_id; + assert!(request_run(store, &mismatched).await.is_err()); + } + assert!( + list_runs(store, (location_id, "development"), 100) + .await + .expect("rejected requests do not queue") + .is_empty() + ); + queue( + store, + &request((location_id, published.snapshot_id), 1, 0, &["orders"]), + ) + .await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim") + .expect("work"); + let run_id = claim.request.run_id; + let worker = issue_token( + store, + &audit, + CredentialScope::Run { + run_id, + claim_id: claim.claim_id, + }, + expiry, + ) + .await + .expect("worker credential"); + let events_path = format!("{}/api/runs/{run_id}/events", api.url); + let heartbeat_path = format!("{}/api/runs/{run_id}/heartbeat", api.url); + let execution_path = format!("{}/api/runs/{run_id}/execution", api.url); + let response = client + .get(&execution_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("worker execution contract"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()["cache-control"], "no-store"); + let execution: RunExecution = response.json().await.expect("pinned execution"); + assert_eq!(execution.claim, claim); + assert_eq!(execution.environment_hash, input.environment_hash); + assert_eq!(execution.sqb_version, input.sqb_version); + assert_eq!( + execution.node_hashes.keys().collect::>(), + input.nodes.keys().collect::>(), + "dependency verification includes nodes outside the run selection" + ); + verify_execution(&input, &execution).expect("matching worker compilation"); + for field in ["target", "image_id", "environment_hash", "sqb_version"] { + let mut changed = json!(input); + changed[field] = json!("different"); + let changed: SnapshotInput = + serde_json::from_value(changed).expect("changed worker inputs"); + assert!(verify_execution(&changed, &execution).is_err(), "{field}"); + } + for field in [ + "kind", + "definition", + "raw_sql", + "compiled_sql", + "python_source", + ] { + let mut changed = json!(input); + changed["nodes"]["inventory"][field] = json!("different"); + let changed: SnapshotInput = + serde_json::from_value(changed).expect("changed dependency"); + assert!(verify_execution(&changed, &execution).is_err(), "{field}"); + } + let mut renamed = input.clone(); + let node = renamed.nodes.remove("inventory").expect("dependency"); + renamed.nodes.insert("renamed_inventory".into(), node); + assert!(verify_execution(&renamed, &execution).is_err()); + renamed.nodes.remove("renamed_inventory"); + assert!(verify_execution(&renamed, &execution).is_err()); + let mut display_only = input.clone(); + display_only.generation += 1; + display_only.git_author = Some("Example Author".into()); + display_only.git_dirty = true; + display_only + .documents + .insert("columns".into(), json!({"orders": ["id"]})); + verify_execution(&display_only, &execution) + .expect("display metadata does not change executable code"); + for (url, token) in [ + (execution_path.clone(), &location_token.bearer), + ( + format!("{}/api/runs/{}/execution", api.url, Uuid::now_v7()), + &worker.bearer, + ), + ] { + assert_eq!( + client + .get(url) + .bearer_auth(token) + .send() + .await + .expect("execution scope denial") + .status(), + StatusCode::FORBIDDEN + ); + } + let mut redeploy = input.clone(); + redeploy.generation += 1; + redeploy.environment_hash = "updated-orders-environment".into(); + redeploy.image_id = Some("sha256:updated-orders-image".into()); + redeploy.sqb_version = "99.0.0".into(); + redeploy + .nodes + .get_mut("orders") + .expect("orders") + .compiled_sql = Some("select 2 as id".into()); + assert_eq!( + client + .post(&path) + .bearer_auth(&location_token.bearer) + .json(&redeploy) + .send() + .await + .expect("publish redeploy") + .status(), + StatusCode::OK + ); + let still_pinned: RunExecution = client + .get(&execution_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("execution after redeploy") + .json() + .await + .expect("old execution contract"); + assert_eq!(still_pinned, execution); + let started = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Started, + }; + assert_eq!( + client + .post(&events_path) + .bearer_auth(&location_token.bearer) + .json(&started) + .send() + .await + .expect("location cannot act as worker") + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + client + .get(&path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("worker cannot read location snapshot") + .status(), + StatusCode::FORBIDDEN + ); + for applied in [true, false] { + let response = client + .post(&events_path) + .bearer_auth(&worker.bearer) + .json(&started) + .send() + .await + .expect("report started"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response + .json::() + .await + .expect("report response"), + json!({"applied": applied}) + ); + } + let wrong_claim = RunChangeRequest { + event_id: Uuid::now_v7(), + claim_id: Some(Uuid::now_v7()), + ..started.clone() + }; + assert_eq!( + client + .post(&events_path) + .bearer_auth(&worker.bearer) + .json(&wrong_claim) + .send() + .await + .expect("wrong claim") + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + client + .post(format!("{}/api/runs/{}/heartbeat", api.url, Uuid::now_v7())) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("other run") + .status(), + StatusCode::FORBIDDEN + ); + for (output_run, credential) in [ + (Uuid::now_v7(), &worker.bearer), + (run_id, &location_token.bearer), + ] { + for (endpoint, body) in [ + ("output", json!([])), + ( + "recall", + json!({"invocation_id": "execution", "records": []}), + ), + ] { + assert_eq!( + client + .post(format!("{}/api/runs/{output_run}/{endpoint}", api.url)) + .bearer_auth(credential) + .json(&body) + .send() + .await + .expect("publication scope denial") + .status(), + StatusCode::FORBIDDEN + ); + } + } + let mut forged = json!(crate::api::tests::output_helpers::chunk( + 0, + sqlbuild_recall::records::models::OutputStream::Stdout, + b"forged" + )); + forged["invocation_id"] = json!("scheduler/another-run"); + assert_eq!( + client + .post(format!("{}/api/runs/{run_id}/output", api.url)) + .bearer_auth(&worker.bearer) + .json(&json!([forged])) + .send() + .await + .expect("body cannot choose its output namespace") + .status(), + StatusCode::UNPROCESSABLE_ENTITY + ); + let cancellation = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::CancelRequested { + actor: "test-admin".into(), + reason: "Stop the orders build".into(), + }, + ..started.clone() + }; + assert_eq!( + client + .post(&events_path) + .bearer_auth(&worker.bearer) + .json(&cancellation) + .send() + .await + .expect("worker cannot impersonate admin") + .status(), + StatusCode::BAD_REQUEST + ); + let response = client + .post(&heartbeat_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("worker heartbeat"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response + .json::() + .await + .expect("heartbeat response"), + json!({"cancel_requested": false}) + ); + change_run(store, RunAuthority::Operator(&audit), &cancellation) + .await + .expect("operator cancellation"); + let response: serde_json::Value = client + .post(&heartbeat_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("cancel heartbeat") + .json() + .await + .expect("cancel response"); + assert_eq!(response, json!({"cancel_requested": true})); + let terminal = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::Finished { + outcome: crate::runs::models::RunOutcome::Cancelled, + reason: "worker acknowledged cancellation".into(), + exit_code: Some(143), + }, + ..started + }; + assert_eq!( + client + .post(&events_path) + .bearer_auth(&worker.bearer) + .json(&terminal) + .send() + .await + .expect("worker terminates") + .status(), + StatusCode::OK + ); + assert_eq!( + client + .post(&heartbeat_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("terminal credential rejected") + .status(), + StatusCode::UNAUTHORIZED + ); + assert_eq!( + client + .get(&execution_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("terminal execution credential rejected") + .status(), + StatusCode::UNAUTHORIZED + ); + assert_eq!( + list_runs(store, (location_id, "development"), 100) + .await + .expect("run projection")[0] + .status, + test_case.expected_status, + "{}", + test_case.description + ); + redeploy.generation += 1; + redeploy.image_id = None; + let local: SnapshotRecord = client + .post(&path) + .bearer_auth(&location_token.bearer) + .json(&redeploy) + .send() + .await + .expect("publish local snapshot without a container image") + .json() + .await + .expect("local snapshot response"); + let mut local_request = request((location_id, local.snapshot_id), 2, 0, &["orders"]); + local_request.image_id = None; + queue(store, &local_request).await; + revoke_token(store, &audit, location_token.identity.token_id) + .await + .expect("revoke location token"); + assert_eq!( + client + .get(&path) + .bearer_auth(&location_token.bearer) + .send() + .await + .expect("revocation effective") + .status(), + StatusCode::UNAUTHORIZED + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs new file mode 100644 index 0000000000..7d8a326aa6 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -0,0 +1,78 @@ +//! HTTP contracts compiled within the API owner. + +#[path = "changes/helpers.rs"] +mod changes_helpers; +#[path = "fingerprint_reads/helpers.rs"] +mod fingerprint_reads_helpers; +#[path = "helpers.rs"] +pub(crate) mod helpers; +#[path = "node_sql/helpers.rs"] +mod node_sql_helpers; +#[path = "node_status/helpers.rs"] +mod node_status_helpers; +#[path = "oidc/helpers.rs"] +mod oidc_helpers; +#[path = "oidc/test_types.rs"] +mod oidc_test_types; +#[path = "output/helpers.rs"] +mod output_helpers; +#[path = "person_sessions/helpers.rs"] +mod person_sessions_helpers; +#[path = "person_sessions/test_types.rs"] +mod person_sessions_test_types; +#[path = "recall/helpers.rs"] +mod recall_helpers; +#[path = "statistics/eligibility/helpers.rs"] +mod statistics_eligibility; +#[path = "statistics/helpers.rs"] +mod statistics_helpers; +#[path = "statistics/retention/helpers.rs"] +mod statistics_retention; +#[path = "test_catalog_reads.rs"] +mod test_catalog_reads; +#[path = "test_changes.rs"] +mod test_changes; +#[path = "test_cli_login.rs"] +mod test_cli_login; +#[path = "test_contract.rs"] +mod test_contract; +#[path = "test_custom_roles.rs"] +mod test_custom_roles; +#[path = "test_development.rs"] +mod test_development; +#[path = "test_external_sessions.rs"] +mod test_external_sessions; +#[path = "test_location_health.rs"] +mod test_location_health; +#[path = "test_location_work.rs"] +mod test_location_work; +#[path = "test_oidc.rs"] +mod test_oidc; +#[path = "test_passwords.rs"] +mod test_passwords; +#[path = "test_person_location_work.rs"] +mod test_person_location_work; +#[path = "test_person_sessions.rs"] +mod test_person_sessions; +#[path = "test_proxy.rs"] +mod test_proxy; +#[path = "test_reads.rs"] +mod test_reads; +#[path = "test_statistics_recovery.rs"] +mod test_statistics_recovery; +#[path = "test_types.rs"] +mod test_types; +#[path = "test_warehouse_metadata.rs"] +mod test_warehouse_metadata; +#[path = "test_worker.rs"] +mod test_worker; +#[path = "warehouse_metadata/catalog/helpers.rs"] +mod warehouse_catalog_helpers; +#[path = "warehouse_metadata/events/helpers.rs"] +mod warehouse_events_helpers; +#[path = "warehouse_metadata/helpers.rs"] +mod warehouse_metadata_helpers; +#[path = "warehouse_metadata/reconciliation/helpers.rs"] +mod warehouse_reconciliation_helpers; +#[path = "work/helpers.rs"] +mod work_helpers; diff --git a/crates/sqlbuild-orchestration/tests/api/test_custom_roles.rs b/crates/sqlbuild-orchestration/tests/api/test_custom_roles.rs new file mode 100644 index 0000000000..3f0e731bc5 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_custom_roles.rs @@ -0,0 +1,91 @@ +//! Custom role assignment works over HTTP with session CSRF and applies at once. + +use reqwest::header; +use serde_json::{Value, json}; + +use crate::api::models::{ApiAuthentication, HumanLogin}; +use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; +use crate::api::tests::test_types::CustomRoleApiTestCase; +use crate::credentials::models::Action; +use crate::credentials::tests::role_helpers::{SUPPORT_ROLE, granting_role, role_fixture}; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_http_sessions_when_custom_role_is_assigned_then_it_applies_at_once() { + let test_cases = [ + CustomRoleApiTestCase { + description: "SQLite custom role assignment over HTTP", + backend: Backend::Sqlite, + expected_statuses: [400, 200], + }, + CustomRoleApiTestCase { + description: "PostgreSQL custom role assignment over HTTP", + backend: Backend::Postgres, + expected_statuses: [400, 200], + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let roles = role_fixture(store).await; + let api = start_api_with_authentication( + store, + ApiAuthentication { + login: HumanLogin::Password("https://control.example.com".into()), + ..ApiAuthentication::default() + }, + ) + .await; + let client = reqwest::Client::new(); + let post = |path: &str, body: Value| { + let session = &roles.administrator; + client + .post(format!("{}{path}", api.url)) + .header(header::HOST, "control.example.com") + .header(header::ORIGIN, "https://control.example.com") + .header( + header::COOKIE, + format!("__Host-sqb-session={}", session.bearer), + ) + .header("x-sqb-csrf", &session.csrf) + .json(&body) + .send() + }; + let misspelt = post( + "/api/admin/roles", + json!({"principal_id": roles.member_id, "role": "Operator", + "location_id": roles.location_id, "target": "development"}), + ) + .await + .expect("misspelt role") + .status() + .as_u16(); + let assignment = post( + "/api/admin/roles", + json!({"principal_id": roles.member_id, "role": SUPPORT_ROLE, + "location_id": roles.location_id, "target": "development"}), + ) + .await + .expect("assign"); + let assignment_status = assignment.status().as_u16(); + assert_eq!( + [misspelt, assignment_status], + test_case.expected_statuses, + "{}", + test_case.description + ); + assert_eq!( + granting_role(store, &roles, Action::View).await.as_deref(), + Some(SUPPORT_ROLE) + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_development.rs b/crates/sqlbuild-orchestration/tests/api/test_development.rs new file mode 100644 index 0000000000..490d8861c7 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_development.rs @@ -0,0 +1,210 @@ +//! Development defaults share production transactions, identities and operator facts on both stores. + +use uuid::Uuid; + +use crate::api::models::{ApiAuthentication, DevelopmentScope, HumanLogin}; +use crate::credentials::main::locations::development_location::development_location; +use crate::credentials::models::AuditContext; +use crate::declarations::main::{ + change_trigger::change_trigger, list_triggers::list_triggers, + pause_development_triggers::pause_development_triggers, +}; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest, TriggerStatus}; +use crate::snapshots::main::{ + current_snapshot::current_snapshot, publish_snapshot::publish_snapshot, +}; +use crate::store::main::migrate_store::migrate_store; + +use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; +use crate::api::tests::test_types::DevelopmentTestCase; +use crate::credentials::tests::helpers::location_state; +use crate::scheduler::tests::contracts::ticks::helpers::snapshot; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_development_when_defaults_publish_then_identity_audit_and_trigger_pauses_commit_atomically() + { + let test_cases = [ + DevelopmentTestCase { + description: "SQLite atomic development defaults", + backend: Backend::Sqlite, + expected_status: TriggerStatus::Paused, + }, + DevelopmentTestCase { + description: "PostgreSQL atomic development defaults", + backend: Backend::Postgres, + expected_status: TriggerStatus::Paused, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "local-development".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let fault = TransactionFault { + description: "development audit failure", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected development audit failure')", + postgres_body: "RAISE EXCEPTION 'injected development audit failure'", + }; + install_fault(store, &fault).await; + assert!( + development_location(store, "/synthetic/orders", "development", &context) + .await + .is_err() + ); + assert_eq!(location_state(store).await, (0, 0)); + remove_fault(store, &fault).await; + let location = + development_location(store, "/synthetic/orders", "development", &context) + .await + .expect("create identity"); + assert_eq!( + development_location(store, "/synthetic/orders", "development", &context) + .await + .expect("retained identity"), + location + ); + let other = development_location(store, "/synthetic/orders", "preview", &context) + .await + .expect("target-specific identity"); + assert_ne!(location, other); + let api = start_api_with_authentication( + store, + ApiAuthentication { + login: HumanLogin::Development(DevelopmentScope { + location_id: location, + target: "development".into(), + triggers: false, + }), + ..ApiAuthentication::default() + }, + ) + .await; + let client = reqwest::Client::new(); + let url = format!( + "{}/api/locations/{location}/targets/development/snapshots", + api.url + ); + let mut input = snapshot(location); + install_fault(store, &fault).await; + assert_eq!( + client + .post(&url) + .json(&input) + .send() + .await + .expect("rejected publication") + .status(), + reqwest::StatusCode::INTERNAL_SERVER_ERROR + ); + assert!( + current_snapshot(store, location, "development") + .await + .expect("last good snapshot") + .is_none() + ); + assert!( + list_triggers(store, location, "development") + .await + .expect("no leaked trigger") + .is_empty() + ); + remove_fault(store, &fault).await; + client + .post(&url) + .json(&input) + .send() + .await + .expect("publication") + .error_for_status() + .expect("publication succeeds after rollback"); + let triggers = list_triggers(store, location, "development") + .await + .expect("paused projection"); + assert_eq!(triggers.len(), 1); + assert_eq!( + triggers[0].status, test_case.expected_status, + "{}", + test_case.description + ); + change_trigger( + store, + &context, + &TriggerChangeRequest { + event_id: Uuid::now_v7(), + trigger_id: triggers[0].trigger_id, + actor: context.actor.clone(), + reason: "Explicit development enable".into(), + change: TriggerChange::Enable, + }, + ) + .await + .expect("operator can enable retained triggers"); + input.generation = 2; + let additional = input.documents["triggers"]["daily_orders"].clone(); + input + .documents + .get_mut("triggers") + .expect("trigger document")["additional_orders"] = additional; + client + .post(&url) + .json(&input) + .send() + .await + .expect("reload") + .error_for_status() + .expect("atomic default pause on reload"); + let triggers = list_triggers(store, location, "development") + .await + .expect("reloaded triggers"); + assert_eq!( + triggers + .iter() + .map(|trigger| (trigger.name.as_str(), trigger.status)) + .collect::>(), + vec![ + ("additional_orders", TriggerStatus::Paused), + ("daily_orders", TriggerStatus::Running) + ] + ); + let mut other_input = snapshot(other); + other_input.target = "preview".into(); + publish_snapshot(store, &other_input) + .await + .expect("ordinary defaults stay enabled"); + pause_development_triggers(store, (location, "development")) + .await + .expect("restart pauses retained triggers"); + assert!( + list_triggers(store, location, "development") + .await + .expect("restart state") + .iter() + .all(|trigger| trigger.status == TriggerStatus::Paused) + ); + assert_eq!( + list_triggers(store, other, "preview") + .await + .expect("other scope unchanged")[0] + .status, + TriggerStatus::Running + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs new file mode 100644 index 0000000000..58aabc2518 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs @@ -0,0 +1,198 @@ +//! Per-request proxy identities keep live roles and final-admin protection across replicas. + +use std::collections::{BTreeMap, BTreeSet}; + +use reqwest::{Client, StatusCode, header}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::models::{ApiAuthentication, HumanLogin, TrustedProxyAuthentication}; +use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; +use crate::api::tests::test_types::ExternalSessionTestCase; +use crate::authorization::main::bootstrap_user::bootstrap_user; +use crate::credentials::main::{ + identities::create_principal::create_principal, locations::create_location::create_location, +}; +use crate::credentials::models::AuditContext; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_proxy_people_when_administrators_and_roles_change_then_live_authority_holds() { + let test_cases = [ + ExternalSessionTestCase { + description: "SQLite proxy administrator journey", + backend: Backend::Sqlite, + expected_viewer_statuses: [403, 200, 403], + }, + ExternalSessionTestCase { + description: "PostgreSQL proxy administrator journey", + backend: Backend::Postgres, + expected_viewer_statuses: [403, 200, 403], + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("database"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/proxy_identities.json"); + let inputs: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("DuckShop federation")) + .expect("inputs"); + let people = &inputs["administrator_journey"]; + let first_subject = people["bound"][0].as_str().expect("first bound person"); + let second_subject = people["bound"][1].as_str().expect("second bound person"); + let first = bootstrap_user(store, &context, first_subject) + .await + .expect("first admin"); + let second = create_principal(store, &context, second_subject) + .await + .expect("second person"); + let origin = inputs["origin"].as_str().expect("origin"); + let settings = TrustedProxyAuthentication { + origin: origin.into(), + trusted_peers: BTreeSet::from(["127.0.0.1".parse().expect("peer")]), + identities: BTreeMap::from([ + (first_subject.into(), first), + (second_subject.into(), second), + ]), + }; + let api = start_api_with_authentication( + store, + ApiAuthentication { + login: HumanLogin::TrustedProxy(settings.clone()), + ..Default::default() + }, + ) + .await; + let other = start_api_with_authentication( + &replica, + ApiAuthentication { + login: HumanLogin::TrustedProxy(settings), + ..Default::default() + }, + ) + .await; + let client = Client::builder() + .timeout(std::time::Duration::from_secs(5)) + .build() + .expect("client"); + let subjects = [first_subject, second_subject]; + let request = |index: usize, path: &str| { + client + .post(format!("{}{path}", other.url)) + .header(header::HOST, "control.example.com") + .header(header::ORIGIN, origin) + .header("x-sqb-user", subjects[index]) + }; + let read = |index: usize, path: &str| { + client + .get(format!("{}{path}", other.url)) + .header(header::HOST, "control.example.com") + .header("x-sqb-user", subjects[index]) + }; + let roles = |principal_id| json!({"principal_id":principal_id,"role":"admin","location_id":null,"target":null}); + let response = request(0, "/api/admin/roles") + .json(&roles(second)) + .send() + .await + .expect("admin grant"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.json::().await.expect("applied grant")["applied"], + true + ); + assert_eq!( + request(0, &format!("/api/admin/people/{first}")) + .json(&"revoke") + .send() + .await + .expect("SSO admin replacement") + .status(), + StatusCode::OK + ); + assert_eq!( + request(0, "/api/admin/roles") + .json(&roles(first)) + .send() + .await + .expect("live revoked admin") + .status(), + StatusCode::FORBIDDEN + ); + for operation in ["revoke", "disable"] { + let response = request(1, &format!("/api/admin/people/{second}")) + .json(&operation) + .send() + .await + .expect("final SSO admin"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let body = response.json::().await.expect("invariant denial"); + assert!(body["error"].as_str().expect("error").starts_with( + "cannot remove or disable the final installation administrator; decision " + )); + } + let location = create_location(store, "orders", &context) + .await + .expect("registered location"); + let metadata = format!("/api/control/locations/{location}/targets/development/runs"); + let no_role = read(0, &metadata) + .send() + .await + .expect("no ambient role") + .status(); + let viewer = json!({"principal_id":first,"role":"viewer","location_id":location,"target":"development"}); + assert_eq!( + request(1, "/api/admin/roles") + .json(&viewer) + .send() + .await + .expect("viewer grant") + .status(), + StatusCode::OK + ); + let granted = read(0, &metadata) + .send() + .await + .expect("live viewer") + .status(); + let mut revocation = viewer; + revocation["revoke"] = json!(true); + assert_eq!( + request(1, "/api/admin/roles") + .json(&revocation) + .send() + .await + .expect("viewer revocation") + .status(), + StatusCode::OK + ); + let revoked = read(0, &metadata) + .send() + .await + .expect("live viewer loss") + .status(); + assert_eq!( + [no_role, granted, revoked].map(|status| status.as_u16()), + test_case.expected_viewer_statuses, + "{}", + test_case.description + ); + stop_api(other).await; + stop_api(api).await; + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_health.rs b/crates/sqlbuild-orchestration/tests/api/test_location_health.rs new file mode 100644 index 0000000000..db5ebfe4fa --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_location_health.rs @@ -0,0 +1,198 @@ +use crate::credentials::main::tokens::issue_token::issue_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::platform::main::report_remote_location_health::report_remote_location_health; +use crate::platform::models::{LocationHealth, LocationHealthObservation, LocationHealthReport}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; +use chrono::Utc; +use reqwest::StatusCode; +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::test_types::LocationHealthTestCase; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::snapshot; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_scoped_location_observations_when_retried_or_read_then_history_and_authority_are_preserved() + { + let test_cases = [ + LocationHealthTestCase { + description: "SQLite health history", + backend: Backend::Sqlite, + expected_observations: 2, + }, + LocationHealthTestCase { + description: "PostgreSQL health history", + backend: Backend::Postgres, + expected_observations: 2, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let token = issue_token( + store, + &AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + CredentialScope::Location { location_id }, + Utc::now() + chrono::Duration::hours(1), + ) + .await + .expect("location token"); + let api = start_api(store).await; + let client = reqwest::Client::new(); + let url = format!("{}/api/locations/{location_id}/health", api.url); + let other_url = format!("{}/api/locations/{other}/health", api.url); + let heartbeat = LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: Uuid::now_v7(), + target: "preview #1".into(), + health: LocationHealth::Heartbeat {}, + }; + let before = Utc::now(); + assert!( + report_remote_location_health((&api.url, &token.bearer), location_id, &heartbeat) + .await + .expect("register") + ); + assert!( + !report_remote_location_health((&api.url, &token.bearer), location_id, &heartbeat) + .await + .expect("exact retry") + ); + let changed = LocationHealthReport { + target: "another target".into(), + ..heartbeat.clone() + }; + let response = client + .post(&url) + .bearer_auth(&token.bearer) + .json(&changed) + .send() + .await + .expect("changed retry"); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let response = client + .post(&other_url) + .bearer_auth(&token.bearer) + .json(&heartbeat) + .send() + .await + .expect("foreign observation"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let response = client + .get(&other_url) + .bearer_auth(&token.bearer) + .send() + .await + .expect("foreign history"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let response = client + .post(&url) + .header("Content-Type", "application/json") + .body("invalid") + .send() + .await + .expect("anonymous report"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + let response = client.post(&url).bearer_auth(&token.bearer).json(&json!({ + "event_id": Uuid::now_v7(), "session_id": heartbeat.session_id, "target": heartbeat.target, + "health": { "kind": "heartbeat", "unexpected": "must not disappear" }, + })).send().await.expect("strict health schema"); + assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); + let heartbeat = LocationHealthReport { + event_id: Uuid::now_v7(), + ..heartbeat + }; + assert!( + report_remote_location_health((&api.url, &token.bearer), location_id, &heartbeat) + .await + .expect("refresh existing instance liveness") + ); + let connection = LocationHealthReport { + event_id: Uuid::now_v7(), + health: LocationHealth::Connection { + generation: 1, + healthy: false, + }, + ..heartbeat.clone() + }; + let response = client + .post(&url) + .bearer_auth(&token.bearer) + .json(&connection) + .send() + .await + .expect("unpublished health"); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let mut input = snapshot(location_id); + input.target.clone_from(&heartbeat.target); + publish_snapshot(store, &input) + .await + .expect("published generation"); + assert!( + report_remote_location_health((&api.url, &token.bearer), location_id, &connection) + .await + .expect("connection observation") + ); + let response = client + .get(&url) + .bearer_auth(&token.bearer) + .send() + .await + .expect("health history"); + assert_eq!(response.headers()["cache-control"], "no-store"); + let observations: Vec = + response.json().await.expect("typed health history"); + assert_eq!( + observations.len(), + test_case.expected_observations, + "{}: rejected and duplicate reports append nothing", + test_case.description + ); + assert_eq!(observations[0].report, connection); + assert_eq!(observations[1].report, heartbeat); + let events = read_recall_events(store, 0, 100) + .await + .expect("read durable events"); + assert_eq!( + events + .iter() + .filter( + |event| event.get("event_type").and_then(serde_json::Value::as_str) + == Some("location_health") + ) + .count(), + 1, + "heartbeats only update liveness; connection observations remain durable facts" + ); + assert!( + observations + .iter() + .all(|observation| observation.received_at >= before + && observation.received_at <= Utc::now()) + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs new file mode 100644 index 0000000000..ac5bff2d86 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -0,0 +1,297 @@ +use crate::api::errors::ClientError; +use crate::credentials::main::tokens::issue_token::issue_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::platform::main::connect_location_work::connect_location_work; +use crate::platform::main::location_work::location_work; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::main::request_location_work::request_location_work; +use crate::platform::models::{ + LocationHealth, LocationHealthReport, LocationWork, LocationWorkOutcome, LocationWorkPoll, + LocationWorkRecord, LocationWorkRequest, LocationWorkResult, +}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use chrono::{Duration, Utc}; +use reqwest::StatusCode; +use uuid::Uuid; + +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::test_types::LocationWorkTestCase; +use crate::api::tests::work_helpers::execute; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::snapshot; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable_results_are_enforced() + { + let test_cases = [ + LocationWorkTestCase { + description: "SQLite outbound work", + backend: Backend::Sqlite, + expected_generation: 1, + }, + LocationWorkTestCase { + description: "PostgreSQL outbound work", + backend: Backend::Postgres, + expected_generation: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("fresh migration"); + migrate_store(store).await.expect("repeat migration"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let audit = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let request = LocationWorkRequest { + request_id: Uuid::now_v7(), + location_id, + target: "preview #1".into(), + reason: "Check orders connectivity".into(), + work: LocationWork::CheckConnection {}, + }; + assert!( + request_location_work(store, (&audit, None), &request) + .await + .is_err(), + "unpublished target" + ); + let mut input = snapshot(location_id); + input.target.clone_from(&request.target); + publish_snapshot(store, &input).await.expect("publish"); + let invalid_audit = AuditContext { + granting_role: String::new(), + ..audit.clone() + }; + assert!( + request_location_work(store, (&invalid_audit, None), &request) + .await + .is_err() + ); + assert!( + location_work(store, request.request_id) + .await + .expect("rolled back read") + .is_none() + ); + assert!( + request_location_work(store, (&audit, None), &request) + .await + .expect("enqueue") + ); + let original = location_work(store, request.request_id) + .await + .expect("read") + .expect("request"); + input.generation = 2; + publish_snapshot(store, &input).await.expect("republish"); + assert!( + !request_location_work(store, (&audit, None), &request) + .await + .expect("retry after redeploy") + ); + assert_eq!( + location_work(store, request.request_id) + .await + .expect("read"), + Some(original.clone()) + ); + let changed = LocationWorkRequest { + reason: "Different request".into(), + ..request.clone() + }; + assert!( + request_location_work(store, (&audit, None), &changed) + .await + .is_err() + ); + let changed_actor = AuditContext { + actor: "another-admin".into(), + ..audit.clone() + }; + assert!( + request_location_work(store, (&changed_actor, None), &request) + .await + .is_err() + ); + let token = issue_token( + store, + &audit, + CredentialScope::Location { location_id }, + Utc::now() + Duration::hours(1), + ) + .await + .expect("token"); + let api = start_api(store).await; + let worker = + connect_location_work((&api.url, &token.bearer), location_id).expect("work client"); + let poll = LocationWorkPoll { + session_id: Uuid::now_v7(), + target: request.target.clone(), + }; + assert!(worker.poll(&poll).await.is_err(), "register before polling"); + for session_id in [poll.session_id, Uuid::nil()] { + report_location_health( + store, + location_id, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: poll.target.clone(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("register instance"); + } + let wrong_target = LocationWorkPoll { + target: "another target".into(), + ..poll.clone() + }; + assert!(worker.poll(&wrong_target).await.is_err()); + let foreign = + connect_location_work((&api.url, &token.bearer), other).expect("foreign client"); + assert!(matches!( + foreign.poll(&poll).await, + Err(ClientError::Rejected(StatusCode::FORBIDDEN)) + )); + let client = reqwest::Client::new(); + let url = format!("{}/api/locations/{location_id}/work", api.url); + let response = client + .post(format!("{url}/poll")) + .header("Content-Type", "application/json") + .body("invalid") + .send() + .await + .expect("anonymous"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + let claim = worker.poll(&poll).await.expect("poll").expect("claim"); + assert_eq!( + claim.generation, test_case.expected_generation, + "{}: request remains pinned despite redeployment", + test_case.description + ); + assert_eq!(claim.request, request); + assert_eq!( + worker.poll(&poll).await.expect("lost poll response retry"), + Some(claim.clone()) + ); + let replacement = LocationWorkPoll { + session_id: Uuid::nil(), + ..poll.clone() + }; + assert!( + worker + .poll(&replacement) + .await + .expect("another instance") + .is_none() + ); + let stale = LocationWorkResult { + claim_id: claim.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::Connection { healthy: false }, + }; + execute( + store, + "UPDATE sqb_location_work SET claimed_until='2000-01-01T00:00:00Z'", + ) + .await; + assert!(matches!( + worker.complete(request.request_id, &stale).await, + Err(ClientError::Rejected(StatusCode::CONFLICT)) + )); + let reclaimed = worker + .poll(&replacement) + .await + .expect("reclaim") + .expect("new claim"); + assert_ne!(claim.claim_id, reclaimed.claim_id); + assert!(matches!( + worker.complete(request.request_id, &stale).await, + Err(ClientError::Rejected(StatusCode::CONFLICT)) + )); + let result = LocationWorkResult { + claim_id: reclaimed.claim_id, + session_id: replacement.session_id, + outcome: LocationWorkOutcome::Superseded {}, + }; + assert!( + worker + .complete(request.request_id, &result) + .await + .expect("complete") + ); + execute(store, "UPDATE sqb_location_work SET expires_at='2000-01-01T00:00:00Z', claimed_until='2000-01-01T00:00:00Z'").await; + assert!( + !worker + .complete(request.request_id, &result) + .await + .expect("lost acknowledgement retry after expiry") + ); + let changed_result = LocationWorkResult { + outcome: LocationWorkOutcome::Connection { healthy: true }, + ..result.clone() + }; + assert!( + worker + .complete(request.request_id, &changed_result) + .await + .is_err() + ); + assert!( + worker + .poll(&poll) + .await + .expect("completed excluded") + .is_none() + ); + let response = client + .get(format!("{url}/{}", request.request_id)) + .bearer_auth(&token.bearer) + .send() + .await + .expect("result read"); + assert_eq!(response.headers()["cache-control"], "no-store"); + let record: LocationWorkRecord = response.json().await.expect("typed result"); + assert_eq!(record.result_json, Some(result)); + assert!(record.completed_at.is_some()); + let next = LocationWorkRequest { + request_id: Uuid::now_v7(), + ..request + }; + request_location_work(store, (&audit, None), &next) + .await + .expect("another request"); + execute( + store, + "UPDATE sqb_location_work SET expires_at='2000-01-01T00:00:00Z'", + ) + .await; + assert!( + worker + .poll(&poll) + .await + .expect("expired excluded") + .is_none() + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_oidc.rs b/crates/sqlbuild-orchestration/tests/api/test_oidc.rs new file mode 100644 index 0000000000..53c10d0641 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_oidc.rs @@ -0,0 +1,404 @@ +//! Signed HTTPS provider exchanges exercise PKCE, browser binding and replica-safe one-use state. + +use std::collections::BTreeMap; + +use reqwest::{StatusCode, header}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::models::{ApiAuthentication, HumanLogin, OidcAuthentication}; +use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; +use crate::api::tests::oidc_helpers::{callback, login, start_provider, stop_provider}; +use crate::api::tests::oidc_test_types::{BrowserFlow, ProviderFault}; +use crate::api::tests::test_types::{OidcApiTestCase, OidcThrottleTestCase}; +use crate::credentials::main::identities::create_principal::create_principal; +use crate::credentials::models::AuditContext; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_oidc_codes_when_verified_across_replicas_then_only_bound_live_people_authenticate() { + let test_cases = [ + OidcApiTestCase { + description: "SQLite OIDC boundary", + backend: Backend::Sqlite, + expected_authentications: 5, + }, + OidcApiTestCase { + description: "PostgreSQL OIDC boundary", + backend: Backend::Postgres, + expected_authentications: 5, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let provider = start_provider().await; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/oidc_identities.json"); + let inputs: Value = serde_json::from_str( + &std::fs::read_to_string(path).expect("DuckShop OIDC fixture"), + ) + .expect("DuckShop OIDC cases"); + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let person = create_principal(store, &context, "orders-person") + .await + .expect("person"); + let settings = OidcAuthentication { + origin: inputs["origin"].as_str().expect("origin").into(), + issuer: provider.state.issuer.clone(), + client_id: "orders-control".into(), + client_secret_env: None, + certificate_authority: Some(provider.authority.clone()), + identities: BTreeMap::from([ + ("orders-person".into(), person), + ("absent-person".into(), Uuid::now_v7()), + ]), + }; + let api = start_api_with_authentication( + store, + ApiAuthentication { + login: HumanLogin::Oidc(settings.clone()), + ..Default::default() + }, + ) + .await; + let other = start_api_with_authentication( + &replica, + ApiAuthentication { + login: HumanLogin::Oidc(settings), + ..Default::default() + }, + ) + .await; + let flow = login(&provider, &api.url).await; + let mut wrong = BrowserFlow { + cookie: "__Host-sqb-oidc=wrong-browser".into(), + callback: flow.callback.clone(), + }; + assert_eq!( + callback(&provider, &other.url, &wrong).await.status(), + StatusCode::UNAUTHORIZED + ); + wrong.cookie = flow.cookie.clone(); + wrong + .callback + .query_pairs_mut() + .clear() + .append_pair("state", "wrong-state") + .append_pair("code", "wrong-code"); + assert_eq!( + callback(&provider, &other.url, &wrong).await.status(), + StatusCode::UNAUTHORIZED + ); + let response = callback(&provider, &other.url, &flow).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); + assert!( + response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .any(|value| value + .to_str() + .expect("cookie") + .contains("__Host-sqb-oidc=;")) + ); + let cookie = response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .map(|value| value.to_str().expect("cookie")) + .find(|value| value.starts_with("__Host-sqb-session=")) + .expect("SSO session cookie") + .split(';') + .next() + .expect("cookie pair") + .to_owned(); + let body = response + .json::() + .await + .expect("authenticated session"); + assert_eq!(body["principal_id"], json!(person)); + assert!(body.get("bearer").is_none()); + for url in [&api.url, &other.url] { + let session = provider + .client + .get(format!("{url}/api/auth/session")) + .header(header::HOST, "control.example.com") + .header(header::COOKIE, &cookie) + .send() + .await + .expect("OIDC session shared across replicas"); + assert_eq!(session.status(), StatusCode::OK); + assert_eq!( + session.json::().await.expect("live identity")["principal_id"], + json!(person) + ); + assert_eq!( + provider + .client + .get(format!( + "{url}/api/control/locations/{}/targets/development/runs", + Uuid::now_v7() + )) + .header(header::HOST, "control.example.com") + .header(header::COOKIE, &cookie) + .send() + .await + .expect("no implicit role") + .status(), + StatusCode::FORBIDDEN + ); + } + assert_eq!( + callback(&provider, &api.url, &flow).await.status(), + StatusCode::UNAUTHORIZED + ); + for fault in inputs["invalid_tokens"].as_array().expect("token cases") { + *provider.state.fault.lock().expect("fault") = + serde_json::from_value(fault.clone()).expect("token fault"); + let flow = login(&provider, &api.url).await; + let response = callback(&provider, &other.url, &flow).await; + assert_eq!(response.status(), StatusCode::UNAUTHORIZED, "{fault}"); + let body = response.text().await.expect("sanitized denial"); + assert!(!body.contains("orders-access")); + assert!(!body.contains("code_verifier")); + } + *provider.state.fault.lock().expect("fault") = ProviderFault::default(); + for subject in inputs["ineligible_subjects"].as_array().expect("subjects") { + *provider.state.subject.lock().expect("subject") = + subject.as_str().expect("subject name").into(); + let flow = login(&provider, &api.url).await; + assert_eq!( + callback(&provider, &other.url, &flow).await.status(), + StatusCode::UNAUTHORIZED, + "{subject}" + ); + } + *provider.state.subject.lock().expect("subject") = "orders-person".into(); + let flow = login(&provider, &api.url).await; + *provider.state.published_key.lock().expect("published key") = "customers".into(); + provider.state.fault.lock().expect("fault").signing_key = Some("customers".into()); + assert_eq!( + callback(&provider, &other.url, &flow).await.status(), + StatusCode::OK, + "a key rotated after authorization is fetched at callback" + ); + *provider.state.published_key.lock().expect("published key") = "orders".into(); + *provider.state.fault.lock().expect("fault") = ProviderFault::default(); + for (action, condition, expected_retry) in [ + ( + "oidc.flow.consumed", + "WHEN NEW.action='oidc.flow.consumed'", + StatusCode::OK, + ), + ( + "oidc.authenticated", + "WHEN NEW.action='oidc.authenticated'", + StatusCode::UNAUTHORIZED, + ), + ] { + let flow = login(&provider, &api.url).await; + let point = TransactionFault { + description: "OIDC audit publication failure", + table: "sqb_security_audit", + operation: "INSERT", + condition, + sqlite_body: "SELECT RAISE(ABORT, 'synthetic OIDC audit failure')", + postgres_body: "RAISE EXCEPTION 'synthetic OIDC audit failure'", + }; + install_fault(store, &point).await; + assert_eq!( + callback(&provider, &other.url, &flow).await.status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + remove_fault(store, &point).await; + let retry = callback(&provider, &api.url, &flow).await; + assert_eq!(retry.status(), expected_retry, "{action}"); + } + let flow = login(&provider, &api.url).await; + assert_eq!( + callback(&provider, &other.url, &flow).await.status(), + StatusCode::OK, + "fresh login recovers after an audited-identity publication failure" + ); + let flow = login(&provider, &api.url).await; + let (left, right) = tokio::join!( + callback(&provider, &api.url, &flow), + callback(&provider, &other.url, &flow) + ); + let mut statuses = [left.status().as_u16(), right.status().as_u16()]; + statuses.sort(); + assert_eq!(statuses, [200, 401], "one concurrent callback succeeds"); + let flow = login(&provider, &api.url).await; + let transaction = begin(store, TransactionMode::Write).await.expect("disable"); + let now = transaction.now; + let (transaction, _) = query( + transaction, + "UPDATE sqb_principals SET disabled_at=$1 WHERE principal_id=$2", + vec![Parameter::Time(now), Parameter::Id(person)], + ) + .await + .expect("live disable"); + commit(transaction).await.expect("disable commit"); + for url in [&api.url, &other.url] { + assert_eq!( + provider + .client + .get(format!("{url}/api/auth/session")) + .header(header::HOST, "control.example.com") + .header(header::COOKIE, &cookie) + .send() + .await + .expect("disabled person session") + .status(), + StatusCode::UNAUTHORIZED + ); + } + assert_eq!( + callback(&provider, &other.url, &flow).await.status(), + StatusCode::UNAUTHORIZED + ); + let flow = login(&provider, &api.url).await; + let transaction = begin(store, TransactionMode::Write).await.expect("expiry"); + let expired = transaction.now - chrono::Duration::minutes(10); + let (transaction, _) = query( + transaction, + "UPDATE sqb_oidc_login_flows SET expires_at=$1", + vec![Parameter::Time(expired)], + ) + .await + .expect("advance expiry"); + commit(transaction).await.expect("expiry commit"); + assert_eq!( + callback(&provider, &other.url, &flow).await.status(), + StatusCode::UNAUTHORIZED + ); + let machine = provider + .client + .get(format!( + "{}/api/locations/{}/health", + api.url, + Uuid::now_v7() + )) + .header(header::HOST, "control.example.com") + .header(header::COOKIE, &flow.cookie) + .send() + .await + .expect("machine boundary"); + assert_eq!(machine.status(), StatusCode::UNAUTHORIZED); + let transaction = begin(store, TransactionMode::Read).await.expect("evidence"); + let (transaction, rows) = query(transaction, + "SELECT (SELECT COUNT(*) FROM sqb_principals) AS principals, (SELECT COUNT(*) FROM sqb_sessions WHERE credential_id IS NULL) AS sessions, (SELECT COUNT(*) FROM sqb_access_tokens) AS tokens, (SELECT COUNT(*) FROM sqb_security_audit WHERE action='oidc.authenticated') AS authentications", + vec![]).await.expect("no provisioning"); + assert_eq!( + json!(rows[0]), + json!({"principals":1,"sessions":test_case.expected_authentications,"tokens":0,"authentications":test_case.expected_authentications}), + "{}", + test_case.description + ); + commit(transaction).await.expect("inspection commit"); + stop_api(other).await; + stop_api(api).await; + close_store(replica).await; + close_test_store(fixture).await; + stop_provider(provider).await; + }); + } +} + +#[test] +fn given_oidc_login_attempts_when_alternating_replicas_then_the_peer_limit_is_shared() { + let test_cases = [ + OidcThrottleTestCase { + description: "SQLite shared peer limit", + backend: Backend::Sqlite, + expected_allowed: 20, + }, + OidcThrottleTestCase { + description: "PostgreSQL shared peer limit", + backend: Backend::Postgres, + expected_allowed: 20, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + migrate_store(&fixture.store).await.expect("migrate"); + let replica = second_store(&fixture.store).await; + let provider = start_provider().await; + let settings = OidcAuthentication { + origin: "https://control.example.com".into(), + issuer: provider.state.issuer.clone(), + client_id: "orders-control".into(), + client_secret_env: None, + certificate_authority: Some(provider.authority.clone()), + identities: BTreeMap::new(), + }; + let api = start_api_with_authentication( + &fixture.store, + ApiAuthentication { + login: HumanLogin::Oidc(settings.clone()), + ..Default::default() + }, + ) + .await; + let other = start_api_with_authentication( + &replica, + ApiAuthentication { + login: HumanLogin::Oidc(settings), + ..Default::default() + }, + ) + .await; + let mut statuses = Vec::new(); + for url in [&api.url, &other.url].into_iter().cycle().take(22) { + statuses.push( + provider + .client + .get(format!("{url}/api/auth/oidc/login")) + .header(header::HOST, "control.example.com") + .send() + .await + .expect("login attempt") + .status(), + ); + } + assert_eq!( + &statuses[..test_case.expected_allowed], + vec![StatusCode::SEE_OTHER; test_case.expected_allowed], + "{}", + test_case.description + ); + assert_eq!( + &statuses[test_case.expected_allowed..], + [StatusCode::TOO_MANY_REQUESTS; 2] + ); + stop_api(other).await; + stop_api(api).await; + close_store(replica).await; + close_test_store(fixture).await; + stop_provider(provider).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_passwords.rs b/crates/sqlbuild-orchestration/tests/api/test_passwords.rs new file mode 100644 index 0000000000..bf5c13331c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_passwords.rs @@ -0,0 +1,403 @@ +//! Real HTTP sessions use secure cookies, exact origins and live cross-replica credentials. + +use reqwest::header::{COOKIE, HOST, ORIGIN, SET_COOKIE}; +use reqwest::{Client, StatusCode}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::models::{ + ApiAuthentication, AuthenticationDenial, HumanLogin, PasswordSessionResponse, +}; +use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; +use crate::api::tests::test_types::PasswordApiTestCase; +use crate::credentials::main::{ + identities::create_principal::create_principal, sessions::set_password::set_password, +}; +use crate::credentials::models::{AuditContext, DefaultRole, PasswordAuthority}; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::TransactionMode; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_password_http_sessions_when_changed_or_revoked_then_cookie_and_csrf_boundaries_hold_across_replicas() + { + let test_cases = [ + PasswordApiTestCase { + description: "SQLite browser session contract", + backend: Backend::Sqlite, + expected_lifetime: 3600, + }, + PasswordApiTestCase { + description: "PostgreSQL browser session contract", + backend: Backend::Postgres, + expected_lifetime: 3600, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/password_credentials.json", + ); + let cases: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("DuckShop fixture")) + .expect("session inputs"); + let username = cases["username"].as_str().expect("username"); + let maximum_password_bytes = cases["maximum_password_bytes"] + .as_u64() + .expect("password byte limit") as usize; + let password = cases["http_password_character"] + .as_str() + .expect("password character") + .repeat(maximum_password_bytes); + let replacement = cases["http_replacement_character"] + .as_str() + .expect("replacement character") + .repeat(maximum_password_bytes); + let password = password.as_str(); + let replacement = replacement.as_str(); + let origin = cases["origin"].as_str().expect("origin"); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let person = create_principal(store, &context, username) + .await + .expect("person"); + set_password( + store, + &context, + PasswordAuthority::Operator(person), + password, + ) + .await + .expect("set password"); + let api = start_api_with_authentication( + store, + ApiAuthentication { + login: HumanLogin::Password(origin.into()), + ..ApiAuthentication::default() + }, + ) + .await; + let other = start_api_with_authentication( + &replica, + ApiAuthentication { + login: HumanLogin::Password(origin.into()), + default_role: DefaultRole::Viewer, + }, + ) + .await; + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + HOST, + origin + .strip_prefix("https://") + .expect("HTTPS origin") + .parse() + .expect("host"), + ); + headers.insert(ORIGIN, origin.parse().expect("origin")); + let client = Client::builder() + .default_headers(headers) + .timeout(std::time::Duration::from_secs(5)) + .build() + .expect("browser transport"); + let response = client + .post(format!("{}/api/auth/password/login", api.url)) + .json(&json!({"username": username, "password": "x".repeat(16 * 1024)})) + .send() + .await + .expect("oversized request"); + assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); + let mut decisions = Vec::new(); + for invalid_origin in cases["invalid_origins"] + .as_array() + .expect("invalid origins") + { + let response = client + .post(format!("{}/api/auth/password/login", api.url)) + .header(ORIGIN, invalid_origin.as_str().expect("origin")) + .json(&json!({"username": username, "password": password})) + .send() + .await + .expect("origin probe"); + assert_eq!( + response.status(), + StatusCode::UNAUTHORIZED, + "{}", + test_case.description + ); + decisions.push( + response + .json::() + .await + .expect("denial decision") + .request_id, + ); + } + for (name, secret) in [ + (username, cases["wrong"].as_str().expect("wrong password")), + ("unknown-person", password), + ] { + let response = client + .post(format!("{}/api/auth/password/login", api.url)) + .json(&json!({"username": name, "password": secret})) + .send() + .await + .expect("credential probe"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + let denial: AuthenticationDenial = response.json().await.expect("generic denial"); + assert_eq!(denial.error, "authentication rejected"); + decisions.push(denial.request_id); + } + let response = client + .post(format!("{}/api/auth/password/login", api.url)) + .json(&json!({"username": username, "password": password})) + .send() + .await + .expect("login"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()["cache-control"], "no-store"); + let cookies: Vec<_> = response + .headers() + .get_all(SET_COOKIE) + .iter() + .map(|value| value.to_str().expect("cookie").to_owned()) + .collect(); + assert_eq!(cookies.len(), 2); + let session = cookies + .iter() + .find(|cookie| cookie.starts_with("__Host-sqb-session=")) + .expect("session cookie"); + assert!(session.contains("; HttpOnly")); + for cookie in &cookies { + assert!(cookie.contains("; Secure; SameSite=Strict;")); + assert!(cookie.contains("; Path=/;")); + assert!(cookie.contains(&format!("Max-Age={}", test_case.expected_lifetime))); + assert!(!cookie.to_lowercase().contains("domain=")); + } + let cookie = session.split(';').next().expect("cookie pair").to_owned(); + let bearer = cookie.split_once('=').expect("session bearer").1; + let body: Value = response.json().await.expect("login body"); + assert!(body.get("bearer").is_none()); + assert!(!body.to_string().contains(bearer)); + let issued: PasswordSessionResponse = + serde_json::from_value(body).expect("typed session"); + assert_eq!(issued.identity.principal_id, person); + assert!(!format!("{issued:?}").contains(&issued.csrf_token)); + let forged = format!( + "__Host-sqb-session={}.{}", + issued.identity.session_id, + "0".repeat(64) + ); + for invalid_cookie in [&forged, &format!("{cookie}; {cookie}")] { + let response = client + .get(format!("{}/api/auth/session", api.url)) + .header(COOKIE, invalid_cookie) + .send() + .await + .expect("invalid session cookie"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + decisions.push( + response + .json::() + .await + .expect("session decision") + .request_id, + ); + } + let response = client + .get(format!("{}/api/auth/session", other.url)) + .header(COOKIE, &cookie) + .send() + .await + .expect("second replica read"); + assert_eq!(response.status(), StatusCode::OK); + let location_id = Uuid::now_v7(); + for (server, expected_status) in + [(&api, StatusCode::FORBIDDEN), (&other, StatusCode::OK)] + { + let url = format!( + "{}/api/control/locations/{location_id}/targets/development/runs", + server.url + ); + assert_eq!( + client + .get(&url) + .send() + .await + .expect("unauthenticated read") + .status(), + StatusCode::UNAUTHORIZED, + "viewer default cannot replace authentication" + ); + let response = client + .get(url) + .header(COOKIE, &cookie) + .send() + .await + .expect("live server-owned default role"); + assert_eq!(response.status(), expected_status); + assert_eq!( + response + .json::() + .await + .expect("metadata response") + .as_array() + .map(Vec::len), + (expected_status == StatusCode::OK).then_some(0), + "only the configured viewer receives the empty metadata array" + ); + let response = client + .post(format!("{}/api/control/runs", server.url)) + .header(COOKIE, &cookie) + .header("x-sqb-csrf", &issued.csrf_token) + .json( + &json!({"run_id": Uuid::now_v7(), "location_id": location_id, + "target": "development", "job_key": "daily_orders", + "reason": "Build orders", "priority": 0}), + ) + .send() + .await + .expect("default role cannot execute"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + } + for csrf in cases["invalid_csrf"].as_array().expect("invalid CSRF") { + let response = client + .post(format!("{}/api/auth/logout", other.url)) + .header(COOKIE, &cookie) + .header("x-sqb-csrf", csrf.as_str().expect("CSRF value")) + .send() + .await + .expect("CSRF probe"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + decisions.push( + response + .json::() + .await + .expect("CSRF decision") + .request_id, + ); + } + let response = client + .get(format!( + "{}/api/locations/{}/targets/development/runs", + api.url, + Uuid::now_v7() + )) + .header(COOKIE, &cookie) + .send() + .await + .expect("session is not location authority"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + let response = client + .post(format!("{}/api/auth/password", other.url)) + .header(COOKIE, &cookie) + .header("x-sqb-csrf", &issued.csrf_token) + .json(&json!({"current_password": password, "new_password": replacement})) + .send() + .await + .expect("password replacement"); + assert_eq!(response.status(), StatusCode::NO_CONTENT); + for value in response.headers().get_all(SET_COOKIE) { + assert!( + value + .to_str() + .expect("expired cookie") + .contains("Max-Age=0") + ); + } + for server in [&api, &other] { + let response = client + .get(format!("{}/api/auth/session", server.url)) + .header(COOKIE, &cookie) + .send() + .await + .expect("old session check"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + decisions.push( + response + .json::() + .await + .expect("revocation decision") + .request_id, + ); + } + let response = client + .post(format!("{}/api/auth/password/login", api.url)) + .json(&json!({"username": username, "password": replacement})) + .send() + .await + .expect("replacement login"); + assert_eq!(response.status(), StatusCode::OK); + let cookie = response + .headers() + .get_all(SET_COOKIE) + .iter() + .map(|value| value.to_str().expect("cookie")) + .find(|value| value.starts_with("__Host-sqb-session=")) + .expect("new session") + .split(';') + .next() + .expect("cookie pair") + .to_owned(); + let renewed: PasswordSessionResponse = + response.json().await.expect("new session identity"); + assert_ne!(renewed.identity.session_id, issued.identity.session_id); + let response = client + .post(format!("{}/api/auth/logout", other.url)) + .header(COOKIE, &cookie) + .header("x-sqb-csrf", &renewed.csrf_token) + .send() + .await + .expect("logout"); + assert_eq!(response.status(), StatusCode::NO_CONTENT); + let response = client + .get(format!("{}/api/auth/session", api.url)) + .header(COOKIE, &cookie) + .send() + .await + .expect("logout across replicas"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + decisions.push( + response + .json::() + .await + .expect("logout decision") + .request_id, + ); + let (transaction, rows) = query(begin(store, TransactionMode::Read).await.expect("audit read"), + "SELECT event_id, action, ip_address, scope_json, before_json, after_json FROM sqb_security_audit", vec![]).await.expect("security evidence"); + commit(transaction).await.expect("audit read complete"); + for decision in decisions { + assert!( + rows.iter().all(|row| row["event_id"] != json!(decision)), + "anonymous denials stay out of the durable audit" + ); + } + let audit = serde_json::to_string(&rows).expect("audit"); + for secret in [password, replacement, bearer, &issued.csrf_token] { + assert!( + !audit.contains(secret), + "security audits exclude secret material" + ); + } + stop_api(other).await; + stop_api(api).await; + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs new file mode 100644 index 0000000000..552171a323 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_person_location_work.rs @@ -0,0 +1,180 @@ +//! Person connection tests use live built-ins and the same atomic work queue on both stores. + +use uuid::Uuid; + +use crate::api::tests::test_types::LocationWorkTestCase; +use crate::authorization::main::{ + assign_builtin_role::assign_builtin_role, bootstrap_user::bootstrap_user, +}; +use crate::credentials::main::{ + identities::create_principal::create_principal, sessions::login_password::login_password, + sessions::set_password::set_password, +}; +use crate::credentials::models::{ + AuditContext, BuiltinRole, PasswordAuthority, PasswordLogin, RoleAssignmentRequest, +}; +use crate::credentials::tests::location_helpers::create_location; +use crate::platform::main::{ + read_person_connection_test::read_person_connection_test, + request_location_work::request_location_work, +}; +use crate::platform::models::{LocationWork, LocationWorkRequest}; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_person_connection_tests_when_retried_and_revoked_then_live_scope_precedes_work_reads_and_writes() + { + let test_cases = [ + LocationWorkTestCase { + description: "SQLite person work", + backend: Backend::Sqlite, + expected_generation: 1, + }, + LocationWorkTestCase { + description: "PostgreSQL person work", + backend: Backend::Postgres, + expected_generation: 1, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: Some(std::net::Ipv4Addr::LOCALHOST.into()), + }; + let admin = bootstrap_user(store, &context, "orders-admin") + .await + .expect("bootstrap"); + let person = create_principal(store, &context, "orders-developer") + .await + .expect("person"); + let mut sessions = Vec::new(); + for (id, name) in [(admin, "orders-admin"), (person, "orders-developer")] { + set_password( + store, + &context, + PasswordAuthority::Operator(id), + "orders-password-2026", + ) + .await + .expect("password"); + let PasswordLogin::Authenticated(session) = + login_password(store, &context, (name, "orders-password-2026")) + .await + .expect("login") + else { + panic!("valid login") + }; + sessions.push(session); + } + let credentials = sessions[1].identity.principal_id; + let location = create_location(store, "orders", "test-owner") + .await + .expect("location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publish"); + let assignment = RoleAssignmentRequest { + principal_id: person, + role: BuiltinRole::Developer.into(), + location_id: Some(location), + target: Some("development".into()), + revoke: false, + }; + assign_builtin_role( + store, + (&context, sessions[0].identity.principal_id), + &assignment, + ) + .await + .expect("assign existing builtin"); + let request = LocationWorkRequest { + request_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + reason: "Check orders connectivity".into(), + work: LocationWork::CheckConnection {}, + }; + assert!( + request_location_work(store, (&context, Some(credentials)), &request) + .await + .expect("authorized enqueue") + ); + assert!( + !request_location_work(store, (&context, Some(credentials)), &request) + .await + .expect("idempotent retry") + ); + let result = read_person_connection_test( + store, + (&context, credentials), + (location, "development", request.request_id), + ) + .await + .expect("authorized read") + .expect("record"); + assert_eq!( + result.generation, test_case.expected_generation, + "{}", + test_case.description + ); + assert_eq!(result.requested_by, format!("principal:{person}")); + assert!(result.result_json.is_none()); + for (other, target) in [(Uuid::now_v7(), "development"), (location, "preview")] { + assert!(matches!( + read_person_connection_test( + store, + (&context, credentials), + (other, target, request.request_id) + ) + .await, + Err(ControlError::Denied { .. }) + )); + assert!( + read_person_connection_test( + store, + (&context, sessions[0].identity.principal_id), + (other, target, request.request_id) + ) + .await + .expect("installation permission still filters stored scope") + .is_none() + ); + } + assign_builtin_role( + store, + (&context, sessions[0].identity.principal_id), + &RoleAssignmentRequest { + revoke: true, + ..assignment + }, + ) + .await + .expect("live revoke"); + assert!(matches!( + request_location_work(store, (&context, Some(credentials)), &request).await, + Err(ControlError::Denied { .. }) + )); + assert!(matches!( + read_person_connection_test( + store, + (&context, credentials), + (location, "development", request.request_id) + ) + .await, + Err(ControlError::Denied { .. }) + )); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs new file mode 100644 index 0000000000..911fc980b2 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs @@ -0,0 +1,517 @@ +//! Existing human authentication mechanisms share live authority, never machine authority. + +use reqwest::{Method, StatusCode}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::tests::helpers::stop_api; +use crate::api::tests::oidc_helpers::{start_provider, stop_provider}; +use crate::api::tests::person_sessions_helpers::{ + browser_sessions, forged_request, queue_counts, request, +}; +use crate::api::tests::person_sessions_test_types::SessionRoleCase; +use crate::api::tests::test_types::LocationWorkTestCase; +use crate::authorization::main::{ + assign_builtin_role::assign_builtin_role, bootstrap_user::bootstrap_user, +}; +use crate::credentials::main::{ + identities::create_principal::create_principal, sessions::login_password::login_password, + sessions::set_password::set_password, +}; +use crate::credentials::models::{ + AuditContext, PasswordAuthority, PasswordLogin, RoleAssignmentRequest, +}; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::models::LocationWork; +use crate::runs::tests::helpers::{request as run_request, request_run}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; +use crate::sensors::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::SnapshotNode; +use crate::store::main::{begin::begin, commit::commit, migrate_store::migrate_store}; +use crate::store::models::TransactionMode; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_work_kind_remain_enforced() + { + let test_cases = [ + LocationWorkTestCase { + description: "SQLite browser session matrix", + backend: Backend::Sqlite, + expected_generation: 1, + }, + LocationWorkTestCase { + description: "PostgreSQL browser session matrix", + backend: Backend::Postgres, + expected_generation: 1, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("database"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let context = AuditContext { + actor: "control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: Some(std::net::Ipv4Addr::LOCALHOST.into()), + }; + let admin = bootstrap_user(store, &context, "orders-admin") + .await + .expect("administrator"); + let person = create_principal(store, &context, "orders-person") + .await + .expect("person"); + for id in [admin, person] { + set_password( + store, + &context, + PasswordAuthority::Operator(id), + "orders-password-2026", + ) + .await + .expect("password"); + } + let PasswordLogin::Authenticated(admin_session) = + login_password(store, &context, ("orders-admin", "orders-password-2026")) + .await + .expect("admin login") + else { + panic!("valid admin") + }; + let location = create_location(store, "orders", "test-owner") + .await + .expect("location"); + let mut input = snapshot(location); + input.documents.get_mut("triggers").expect("triggers")["health_orders"]["when"] = + json!({"kind":"custom", "function":"orders_ready", "code_hash":"orders_v1"}); + let sql_input: Value = serde_json::from_str( + &std::fs::read_to_string( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/node_sql.json"), + ) + .expect("SQL fixture"), + ) + .expect("SQL inputs"); + let orders = input.nodes.get_mut("orders").expect("orders node"); + orders.raw_sql = Some(sql_input["raw_sql"].as_str().expect("raw SQL").into()); + orders.compiled_sql = Some( + sql_input["compiled_sql"] + .as_str() + .expect("compiled SQL") + .into(), + ); + orders.python_source = Some( + sql_input["python_source"] + .as_str() + .expect("Python source") + .into(), + ); + input.nodes.insert( + "udf:orders".into(), + SnapshotNode { + kind: "udf".into(), + definition: json!({"name":"orders", "arguments":[], "returns":"INTEGER"}), + raw_sql: None, + compiled_sql: Some( + sql_input["function_sql"] + .as_str() + .expect("function SQL") + .into(), + ), + python_source: None, + }, + ); + let published = publish_snapshot(store, &input).await.expect("publication"); + assert_eq!( + published.generation, test_case.expected_generation, + "{}", + test_case.description + ); + let seed_run = run_request((location, published.snapshot_id), 100, 0, &["orders"]); + request_run(store, &seed_run) + .await + .expect("nonempty metadata control"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("election") + .expect("leader"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + let evaluation = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("fenced sensor request") + .expect("requested"); + let transaction = begin(store, TransactionMode::Read) + .await + .expect("inspect sensor"); + let (transaction, stored) = read_location_work(transaction, evaluation.evaluation_id) + .await + .expect("sensor work exists"); + assert!(matches!( + stored.expect("actual work").record.request_json.work, + LocationWork::EvaluateSensor { .. } + )); + commit(transaction).await.expect("close read"); + let provider = start_provider().await; + let sessions = browser_sessions(store, &provider, person).await; + assert_eq!(sessions.len(), 3, "password, proxy and OIDC all execute"); + let path = + format!("/api/control/locations/{location}/targets/development/connection-tests"); + let metadata = format!("/api/control/locations/{location}/targets/development/runs"); + let sql_path = + format!("/api/control/locations/{location}/targets/development/nodes/orders/sql"); + let inputs: Value = serde_json::from_str(&std::fs::read_to_string(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json")).expect("DuckShop matrix")).expect("JSON"); + let roles: Vec = + serde_json::from_value(inputs["session_roles"].clone()) + .expect("independent expectations"); + let mut expected_work = 1; + let mut expected_runs = 1; + let mut last_request = Value::Null; + for role_case in roles { + for role in role_case.role.iter().copied() { + assign_builtin_role( + &replica, + (&context, admin_session.identity.principal_id), + &RoleAssignmentRequest { + principal_id: person, + role: role.into(), + location_id: None, + target: None, + revoke: false, + }, + ) + .await + .expect("assign role through independent store"); + } + for session in &sessions { + let body = + json!({"request_id":Uuid::now_v7(), "reason":"Check orders connectivity"}); + last_request = body.clone(); + let connection = request( + &provider.client, + session, + (Method::POST, &path), + (&body, &session.csrf), + ) + .await; + assert_eq!(connection.status().as_u16(), role_case.expected_connection); + assert_eq!(connection.headers()["cache-control"], "no-store"); + let read = request( + &provider.client, + session, + (Method::GET, &format!("{path}/{}", evaluation.evaluation_id)), + (&Value::Null, &session.csrf), + ) + .await; + assert_eq!(read.status().as_u16(), role_case.expected_connection); + let read_body: Value = read.json().await.expect("sensor response"); + assert_eq!( + read_body.is_null(), + role_case.expected_connection == StatusCode::OK.as_u16(), + "authorized reads hide actual sensor work as null" + ); + let page = request( + &provider.client, + session, + (Method::GET, &metadata), + (&Value::Null, &session.csrf), + ) + .await; + assert_eq!(page.status().as_u16(), role_case.expected_metadata); + let page: Value = page.json().await.expect("metadata response"); + assert_eq!( + page.is_array(), + role_case.expected_metadata == StatusCode::OK.as_u16() + ); + assert!(!page.to_string().contains("select 1 as id")); + let sql = request( + &provider.client, + session, + (Method::GET, &sql_path), + (&Value::Null, ""), + ) + .await; + assert_eq!(sql.status().as_u16(), role_case.expected_sql); + assert_eq!(sql.headers()["cache-control"], "no-store"); + let sql: Value = sql.json().await.expect("SQL response"); + for (key, expected) in [ + ("snapshot_id", json!(published.snapshot_id)), + ("node_key", json!("orders")), + ("raw_sql", sql_input["raw_sql"].clone()), + ("compiled_sql", sql_input["compiled_sql"].clone()), + ] { + assert_eq!( + sql.get(key), + [None, Some(&expected)][usize::from(role_case.expected_sql == 200)] + ); + } + assert_eq!( + sql.get("decision_id").is_some(), + role_case.expected_sql != 200 + ); + assert!(sql.get("python_source").is_none()); + let unpublished = request( + &provider.client, + session, + (Method::GET, &sql_path.replace("development", "preview")), + (&Value::Null, ""), + ) + .await; + assert_eq!(unpublished.status().as_u16(), role_case.expected_sql); + assert_eq!( + unpublished + .json::() + .await + .expect("unpublished scope") + .is_null(), + role_case.expected_sql == 200 + ); + let job = json!({"run_id":Uuid::now_v7(), "location_id":location, "target":"development", "job_key":"orders", "reason":"Refresh orders", "priority":0}); + assert_eq!( + request( + &provider.client, + session, + (Method::POST, "/api/control/runs"), + (&job, &session.csrf) + ) + .await + .status() + .as_u16(), + role_case.expected_job + ); + for (endpoint, request_body) in + [(&path[..], &body), ("/api/control/runs", &job)] + { + assert_eq!( + forged_request( + &provider.client, + session, + (Method::POST, endpoint), + request_body + ) + .await + .status(), + StatusCode::UNAUTHORIZED + ); + } + for machine_path in [ + format!("/api/locations/{location}/targets/development/snapshots"), + format!( + "/api/locations/{location}/work/{}", + evaluation.evaluation_id + ), + format!( + "/api/locations/{location}/targets/development/runs/{}/recall/output_tail?invocation=scheduler/{}", + seed_run.run_id, seed_run.run_id + ), + format!( + "/api/locations/{location}/targets/development/runs/{}/recall/artifacts?invocation=scheduler/{}", + seed_run.run_id, seed_run.run_id + ), + ] { + assert_eq!( + request( + &provider.client, + session, + (Method::GET, &machine_path), + (&Value::Null, &session.csrf) + ) + .await + .status(), + StatusCode::UNAUTHORIZED + ); + } + } + expected_work += usize::from(role_case.expected_connection == 200) * sessions.len(); + expected_runs += usize::from(role_case.expected_job == 200) * sessions.len(); + assert_eq!( + queue_counts(store).await, + json!({"work":expected_work,"runs":expected_runs,"claims":0}) + ); + for role in role_case.role.iter().copied() { + assign_builtin_role( + &replica, + (&context, admin_session.identity.principal_id), + &RoleAssignmentRequest { + principal_id: person, + role: role.into(), + location_id: None, + target: None, + revoke: true, + }, + ) + .await + .expect("revoke through independent store"); + } + for session in &sessions { + assert_eq!( + request( + &provider.client, + session, + (Method::GET, &sql_path), + (&Value::Null, "") + ) + .await + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + request( + &provider.client, + session, + (Method::POST, &path), + (&last_request, &session.csrf) + ) + .await + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + request( + &provider.client, + session, + ( + Method::GET, + &format!( + "{path}/{}", + last_request["request_id"].as_str().expect("request ID") + ) + ), + (&Value::Null, &session.csrf) + ) + .await + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + request( + &provider.client, + session, + (Method::GET, &metadata), + (&Value::Null, &session.csrf) + ) + .await + .status(), + StatusCode::FORBIDDEN + ); + } + assert_eq!( + queue_counts(store).await, + json!({"work":expected_work,"runs":expected_runs,"claims":0}) + ); + } + assign_builtin_role( + &replica, + (&context, admin_session.identity.principal_id), + &RoleAssignmentRequest { + principal_id: person, + role: crate::credentials::models::BuiltinRole::Developer.into(), + location_id: Some(location), + target: Some("development".into()), + revoke: false, + }, + ) + .await + .expect("scoped developer"); + crate::api::tests::node_sql_helpers::node_sql_boundaries( + store, + (&provider.client, &sessions[0]), + (location, published.snapshot_id), + (&input, &sql_input), + ) + .await; + for session in &sessions { + let result = request( + &provider.client, + session, + ( + Method::GET, + &format!( + "{path}/{}", + last_request["request_id"].as_str().expect("accepted ID") + ), + ), + (&Value::Null, &session.csrf), + ) + .await; + assert_eq!(result.status(), StatusCode::OK); + assert_eq!( + result + .json::() + .await + .expect("actual connection record")["request_json"]["request_id"], + last_request["request_id"] + ); + for other in [ + path.replace("development", "preview"), + path.replace(&location.to_string(), &Uuid::now_v7().to_string()), + ] { + assert_eq!( + request( + &provider.client, + session, + (Method::POST, &other), + (&last_request, &session.csrf) + ) + .await + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + request( + &provider.client, + session, + ( + Method::GET, + &format!( + "{other}/{}", + last_request["request_id"].as_str().expect("ID") + ) + ), + (&Value::Null, &session.csrf) + ) + .await + .status(), + StatusCode::FORBIDDEN + ); + } + let sensor_work = request( + &provider.client, + session, + (Method::GET, &format!("{path}/{}", evaluation.evaluation_id)), + (&Value::Null, &session.csrf), + ) + .await; + assert_eq!(sensor_work.status(), StatusCode::OK); + assert_eq!( + sensor_work + .json::() + .await + .expect("hidden sensor work"), + Value::Null + ); + } + assert_eq!( + queue_counts(store).await, + json!({"work":expected_work,"runs":expected_runs,"claims":0}) + ); + for session in sessions { + stop_api(session.api).await; + } + stop_provider(provider).await; + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_proxy.rs b/crates/sqlbuild-orchestration/tests/api/test_proxy.rs new file mode 100644 index 0000000000..f9f762b681 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_proxy.rs @@ -0,0 +1,271 @@ +//! Explicit proxy bindings authenticate each request without sessions, provisioning or machine authority. + +use std::collections::{BTreeMap, BTreeSet}; + +use reqwest::{Client, StatusCode, header}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::models::{ + ApiAuthentication, AuthenticationDenial, HumanLogin, TrustedProxyAuthentication, +}; +use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; +use crate::api::tests::test_types::ProxyApiTestCase; +use crate::credentials::main::identities::create_principal::create_principal; +use crate::credentials::models::AuditContext; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_explicit_proxy_bindings_when_real_peers_and_live_people_are_checked_then_spoofing_and_implicit_grants_are_rejected() + { + let test_cases = [ + ProxyApiTestCase { + description: "SQLite proxy identity boundary", + backend: Backend::Sqlite, + trusted_peer: "::ffff:127.0.0.1", + expected_sessions: 0, + }, + ProxyApiTestCase { + description: "PostgreSQL proxy identity boundary", + backend: Backend::Postgres, + trusted_peer: "127.0.0.1", + expected_sessions: 0, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/proxy_identities.json"); + let inputs: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("DuckShop subjects")) + .expect("synthetic bindings"); + let value = |key: &str| inputs[key].as_str().expect("text input"); + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let person = create_principal(store, &context, value("person")) + .await + .expect("known person"); + let settings = TrustedProxyAuthentication { + origin: value("origin").into(), + trusted_peers: BTreeSet::from([test_case.trusted_peer.parse().expect("proxy IP")]), + identities: BTreeMap::from([ + (value("subject").into(), person), + (value("absent_subject").into(), Uuid::now_v7()), + ]), + }; + let api = start_api_with_authentication( + store, + ApiAuthentication { + login: HumanLogin::TrustedProxy(settings.clone()), + ..ApiAuthentication::default() + }, + ) + .await; + let other = start_api_with_authentication( + &replica, + ApiAuthentication { + login: HumanLogin::TrustedProxy(settings.clone()), + ..ApiAuthentication::default() + }, + ) + .await; + let untrusted = start_api_with_authentication( + store, + ApiAuthentication { + login: HumanLogin::TrustedProxy(TrustedProxyAuthentication { + trusted_peers: BTreeSet::from([value("untrusted_peer") + .parse() + .expect("untrusted IP")]), + ..settings + }), + ..ApiAuthentication::default() + }, + ) + .await; + let client = Client::builder() + .timeout(std::time::Duration::from_secs(5)) + .build() + .expect("HTTP client"); + let runs = format!( + "/api/control/locations/{}/targets/development/runs", + Uuid::now_v7() + ); + let request = |url: &str| { + client + .get(format!("{url}{runs}")) + .header(header::HOST, "control.example.com") + .header(header::ORIGIN, value("origin")) + }; + for url in [&api.url, &other.url] { + let response = request(url) + .header("x-sqb-user", value("subject")) + .send() + .await + .expect("per-request proxy authentication"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let forwarded = request(url) + .header("x-sqb-user", value("subject")) + .header(header::AUTHORIZATION, value("proxy_authorization")) + .send() + .await + .expect("proxy forwards its own authorization"); + assert_eq!( + forwarded.status(), + StatusCode::FORBIDDEN, + "{}: only a Session authorization is a CLI session", + test_case.description + ); + assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); + assert!(!response.headers().contains_key(header::SET_COOKIE)); + let rejected_write = client + .post(format!("{url}/api/control/runs")) + .header(header::HOST, "control.example.com") + .header(header::ORIGIN, "https://other.example.com") + .header("x-sqb-user", value("subject")) + .json(&json!({})) + .send() + .await + .expect("cross-origin write"); + assert_eq!(rejected_write.status(), StatusCode::UNAUTHORIZED); + } + let mut denials = Vec::new(); + for subject in [ + "", + value("unknown_subject"), + value("person"), + value("absent_subject"), + ] { + let response = request(&api.url) + .header("x-sqb-user", subject) + .send() + .await + .expect("unbound/ineligible subject"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + denials.push( + response + .json::() + .await + .expect("durable denial") + .request_id, + ); + } + for probe in [ + request(&api.url), + request(&api.url) + .header("x-sqb-user", value("subject")) + .header("x-sqb-user", value("subject")), + request(&api.url) + .header("x-sqb-user", value("subject")) + .headers(header::HeaderMap::from_iter([( + header::ORIGIN, + header::HeaderValue::from_static("https://other.example.com"), + )])), + request(&api.url) + .header("x-sqb-user", value("subject")) + .headers(header::HeaderMap::from_iter([( + header::HOST, + header::HeaderValue::from_static("other.example.com"), + )])), + request(&untrusted.url) + .header("x-sqb-user", value("subject")) + .header("x-forwarded-for", value("untrusted_peer")) + .header("forwarded", format!("for={}", value("untrusted_peer"))), + ] { + let response = probe.send().await.expect("spoofing probe"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + denials.push( + response + .json::() + .await + .expect("spoofing denial") + .request_id, + ); + } + let machine = client + .get(format!( + "{}/api/locations/{}/health", + api.url, + Uuid::now_v7() + )) + .header(header::HOST, "control.example.com") + .header("x-sqb-user", value("subject")) + .send() + .await + .expect("no location authority"); + assert_eq!(machine.status(), StatusCode::UNAUTHORIZED); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("disable person"); + let now = transaction.now; + let (transaction, _) = query( + transaction, + "UPDATE sqb_principals SET disabled_at=$1 WHERE principal_id=$2", + vec![Parameter::Time(now), Parameter::Id(person)], + ) + .await + .expect("live disabled fact"); + commit(transaction).await.expect("disable committed"); + for url in [&api.url, &other.url] { + let response = request(url) + .header("x-sqb-user", value("subject")) + .send() + .await + .expect("live person validation"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + denials.push( + response + .json::() + .await + .expect("disabled denial") + .request_id, + ); + } + let mut transaction = begin(store, TransactionMode::Read) + .await + .expect("inspect durable evidence"); + for decision in denials { + let rows; + (transaction, rows) = query( + transaction, + "SELECT event_id FROM sqb_security_audit WHERE event_id=$1", + vec![Parameter::Id(decision)], + ) + .await + .expect("denial audit"); + assert!( + rows.is_empty(), + "anonymous denials stay out of the durable audit" + ); + } + let (transaction, rows) = query(transaction, "SELECT (SELECT COUNT(*) FROM sqb_principals) AS principals, (SELECT COUNT(*) FROM sqb_sessions) AS sessions, (SELECT COUNT(*) FROM sqb_access_tokens) AS tokens", vec![]).await.expect("no implicit provisioning or personal tokens"); + assert_eq!( + json!(rows[0]), + json!({"principals": 1, "sessions": test_case.expected_sessions, "tokens": 0}), + "{}", + test_case.description + ); + commit(transaction).await.expect("inspection complete"); + stop_api(untrusted).await; + stop_api(other).await; + stop_api(api).await; + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_reads.rs new file mode 100644 index 0000000000..e3adc23031 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_reads.rs @@ -0,0 +1,507 @@ +//! Real HTTP reads keep both location and target scope, including guessed nested resource IDs. + +use crate::scheduler::tests::contracts::ticks::helpers::calendar_text; +use std::collections::BTreeMap; +use std::time::Duration; + +use crate::credentials::main::tokens::issue_token::issue_token; +use crate::credentials::main::tokens::revoke_token::revoke_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::declarations::main::change_trigger::change_trigger; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunOutcome}; +use crate::runs::tests::helpers::request_run; +use crate::scheduler::tests::contracts::helpers::{acquire_lease, evaluate_trigger}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::publish_recall; +use crate::worker::main::connect_worker::connect_worker; +use chrono::Utc; +use reqwest::StatusCode; +use serde_json::json; +use sqlbuild_recall::records::main::output_row::output_row; +use sqlbuild_recall::records::models::{OutputStream, RowEnvelope}; +use uuid::Uuid; + +use crate::api::tests::helpers::{read_json, start_api, stop_api}; +use crate::api::tests::output_helpers::chunk; +use crate::api::tests::recall_helpers::recall_contract; +use crate::api::tests::test_types::ApiReadTestCase; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{request, snapshot}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_published_locations_and_scheduled_work_when_read_through_http_then_resources_and_histories_remain_scoped() + { + let test_cases = [ + ApiReadTestCase { + description: "SQLite scoped HTTP reads", + backend: Backend::Sqlite, + expected_runs: 3, + expected_ticks: 1, + }, + ApiReadTestCase { + description: "PostgreSQL scoped HTTP reads", + backend: Backend::Postgres, + expected_runs: 3, + expected_ticks: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let mut input = snapshot(location_id); + input.documents.insert( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), + ); + input.documents.insert("triggers".into(), json!({"daily_orders": {"description": "Refresh orders", "job_key": "orders", "starts_at": calendar_text(0), + "schedule": {"kind": "calendar", "times": [calendar_text(0), calendar_text(1), calendar_text(2), "2030-01-01T00:00:00Z"], "refresh_at": "2029-01-01T00:00:00Z"}}})); + let published = publish_snapshot(store, &input) + .await + .expect("publish declarations"); + let trigger_id = list_triggers(store, location_id, "development") + .await + .expect("trigger")[0] + .trigger_id; + let mut foreign = input.clone(); + foreign.location_id = other; + let foreign_snapshot = publish_snapshot(store, &foreign) + .await + .expect("publish other location"); + let foreign_trigger = list_triggers(store, other, "development") + .await + .expect("foreign trigger")[0] + .trigger_id; + let foreign_run = request( + (other, foreign_snapshot.snapshot_id), + 1001, + 0, + &["inventory"], + ); + request_run(store, &foreign_run) + .await + .expect("queue foreign run"); + let mut another_target = input.clone(); + another_target.target = "validation".into(); + publish_snapshot(store, &another_target) + .await + .expect("publish second target"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + evaluate_trigger(store, &lease, trigger_id) + .await + .expect("evaluate due schedule"); + let mut manual_ids = Vec::new(); + for (id, node) in [(100, "customers"), (101, "products")] { + let run = request((location_id, published.snapshot_id), id, 0, &[node]); + manual_ids.push(run.run_id); + request_run(store, &run).await.expect("queue manual run"); + } + let change = TriggerChangeRequest { + event_id: Uuid::now_v7(), + trigger_id, + actor: "test-admin".into(), + reason: "Pause orders refresh".into(), + change: TriggerChange::Pause, + }; + let audit = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + change_trigger(store, &audit, &change) + .await + .expect("pause trigger"); + let expiry = Utc::now() + chrono::Duration::hours(1); + let token = issue_token( + store, + &audit, + CredentialScope::Location { location_id }, + expiry, + ) + .await + .expect("location token"); + let claim = claim_next_run( + store, + &lease, + &BTreeMap::new(), + Some((location_id, "development")), + ) + .await + .expect("claim") + .expect("work"); + let worker = issue_token( + store, + &audit, + CredentialScope::Run { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + expiry, + ) + .await + .expect("worker token"); + let api = start_api(store).await; + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .expect("HTTP client"); + let prefix = format!( + "{}/api/locations/{location_id}/targets/development", + api.url + ); + let run_path = format!("{prefix}/runs"); + let node_path = format!("{prefix}/runs/{}/nodes", manual_ids[0]); + let trigger_path = format!("{prefix}/triggers"); + let event_path = format!("{trigger_path}/{trigger_id}/events"); + let tick_path = format!("{trigger_path}/{trigger_id}/ticks"); + let invocation_path = format!("{prefix}/runs/{}/invocations", claim.request.run_id); + let recall_path = format!( + "{prefix}/runs/{}/recall/events?invocation=worker/{}/execution", + claim.request.run_id, claim.claim_id + ); + let paths = [ + &run_path, + &node_path, + &trigger_path, + &event_path, + &tick_path, + &invocation_path, + &recall_path, + ]; + let connection = connect_worker( + &api.url, + &worker.bearer, + (claim.request.run_id, claim.claim_id), + ) + .expect("worker HTTP connection"); + recall_contract(store, &connection, claim.claim_id).await; + let output = chunk(0, OutputStream::Stdout, b"orders worker output"); + connection + .publish_output(std::slice::from_ref(&output)) + .await + .expect("output-only namespace"); + let neighboring = RowEnvelope { + invocation_id: format!("worker/{}-other", claim.claim_id), + invocation_started_at: "2026-01-02T03:04:05Z".into(), + producer_version: "test".into(), + }; + publish_recall(store, &[output_row(&neighboring, &output)]) + .await + .expect("neighboring namespace"); + let invocations = read_json(&client, (&invocation_path, &token.bearer)).await; + let invocation_ids = invocations + .as_array() + .expect("run invocations") + .iter() + .map(|row| { + row["invocation_id"] + .as_str() + .expect("invocation ID") + .to_owned() + }) + .collect::>(); + assert_eq!( + invocation_ids, + vec![ + format!("scheduler/{}", claim.request.run_id), + format!("worker/{}", claim.claim_id), + format!("worker/{}/another-invocation", claim.claim_id), + format!("worker/{}/backlog", claim.claim_id), + format!("worker/{}/execution", claim.claim_id), + format!("worker/{}/native-writer", claim.claim_id), + ] + ); + let evidence = read_json(&client, (&recall_path, &token.bearer)).await; + assert_eq!(evidence.as_array().expect("worker events").len(), 1); + assert_eq!(evidence[0]["duration_ms"], json!(12.75)); + assert_eq!(evidence[0]["row_count"], json!(9)); + assert_eq!( + read_json(&client, (&format!("{recall_path}&limit=0"), &token.bearer)).await, + json!([]) + ); + for path in [ + recall_path.replace( + &format!("worker/{}/execution", claim.claim_id), + &format!("scheduler/{}", foreign_run.run_id), + ), + recall_path.replace( + &format!("worker/{}/execution", claim.claim_id), + &format!("worker/{}-other/execution", claim.claim_id), + ), + recall_path.replace( + &format!("/runs/{}/", claim.request.run_id), + &format!("/runs/{}/", manual_ids[0]), + ), + recall_path.replace( + &format!("/runs/{}/", claim.request.run_id), + &format!("/runs/{}/", foreign_run.run_id), + ), + recall_path.replace("/development/", "/validation/"), + ] { + assert_eq!( + client + .get(path) + .bearer_auth(&token.bearer) + .send() + .await + .expect("Recall scope denial") + .status(), + StatusCode::NOT_FOUND + ); + } + assert_eq!( + client + .get(recall_path.replace("/recall/events?", "/recall/unknown?")) + .bearer_auth(&token.bearer) + .send() + .await + .expect("unknown view") + .status(), + StatusCode::BAD_REQUEST + ); + let scheduler_path = format!( + "{prefix}/runs/{}/recall/events?invocation=scheduler/{}", + claim.request.run_id, claim.request.run_id + ); + let history = read_json(&client, (&scheduler_path, &token.bearer)).await; + assert!( + history + .as_array() + .expect("run history") + .iter() + .any(|row| row["event_type"] == "run_claimed") + ); + let runs = read_json(&client, (&run_path, &token.bearer)).await; + assert_eq!( + runs.as_array().expect("runs").len(), + test_case.expected_runs, + "{}", + test_case.description + ); + assert!( + runs.as_array() + .expect("runs") + .iter() + .all(|run| run["location_id"] == location_id.to_string() + && run["target"] == "development") + ); + let nodes = read_json(&client, (&node_path, &token.bearer)).await; + assert_eq!(nodes.as_array().expect("nodes").len(), 1); + assert_eq!(nodes[0]["node_key"], "customers"); + assert_eq!(nodes[0]["status"], "pending"); + let triggers = read_json(&client, (&trigger_path, &token.bearer)).await; + assert_eq!(triggers.as_array().expect("triggers").len(), 1); + assert_eq!(triggers[0]["trigger_id"], trigger_id.to_string()); + assert_eq!(triggers[0]["status"], "paused"); + let events = read_json(&client, (&event_path, &token.bearer)).await; + assert_eq!(events.as_array().expect("events").len(), 1); + assert_eq!(events[0]["request_json"], json!(change)); + let ticks = read_json(&client, (&tick_path, &token.bearer)).await; + assert_eq!( + ticks.as_array().expect("ticks").len(), + test_case.expected_ticks + ); + assert_eq!(ticks[0]["outcome"], "requested"); + assert_eq!(ticks[0]["scheduled_for"], calendar_text(2)); + for path in [&run_path, &event_path, &tick_path, &invocation_path] { + assert_eq!( + read_json(&client, (&format!("{path}?limit=1"), &token.bearer)) + .await + .as_array() + .expect("bounded history") + .len(), + 1 + ); + assert_eq!( + read_json(&client, (&format!("{path}?limit=0"), &token.bearer)).await, + json!([]) + ); + assert_eq!( + client + .get(format!("{path}?limit=invalid")) + .bearer_auth(&token.bearer) + .send() + .await + .expect("invalid limit") + .status(), + StatusCode::BAD_REQUEST + ); + } + for path in paths { + assert_eq!( + client + .get(path) + .send() + .await + .expect("unauthenticated read") + .status(), + StatusCode::UNAUTHORIZED + ); + assert_eq!( + client + .get(path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("wrong credential scope") + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + client + .get(path.replace(&location_id.to_string(), &other.to_string())) + .bearer_auth(&token.bearer) + .send() + .await + .expect("cross-location read") + .status(), + StatusCode::FORBIDDEN + ); + } + for path in [ + node_path.replace(&manual_ids[0].to_string(), &foreign_run.run_id.to_string()), + event_path.replace(&trigger_id.to_string(), &foreign_trigger.to_string()), + tick_path.replace(&trigger_id.to_string(), &foreign_trigger.to_string()), + invocation_path.replace( + &claim.request.run_id.to_string(), + &foreign_run.run_id.to_string(), + ), + node_path.replace("/development/", "/validation/"), + event_path.replace("/development/", "/validation/"), + tick_path.replace("/development/", "/validation/"), + invocation_path.replace("/development/", "/validation/"), + ] { + assert_eq!( + client + .get(path) + .bearer_auth(&token.bearer) + .send() + .await + .expect("nested ID cannot bypass scope") + .status(), + StatusCode::NOT_FOUND + ); + } + assert_eq!( + read_json( + &client, + ( + &run_path.replace("/development/", "/validation/"), + &token.bearer + ) + ) + .await, + json!([]) + ); + let validation = read_json( + &client, + ( + &trigger_path.replace("/development/", "/validation/"), + &token.bearer, + ), + ) + .await; + assert_eq!(validation.as_array().expect("other target").len(), 1); + assert_ne!(validation[0]["trigger_id"], trigger_id.to_string()); + let openapi: serde_json::Value = client + .get(format!("{}/api/openapi.json", api.url)) + .send() + .await + .expect("OpenAPI") + .json() + .await + .expect("OpenAPI JSON"); + for path in [ + "/api/locations/{location_id}/targets/{target}/runs", + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/nodes", + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/invocations", + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/recall/{view}", + "/api/locations/{location_id}/targets/{target}/triggers", + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/events", + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/ticks", + ] { + assert_eq!( + openapi["paths"][path]["get"]["security"], + json!([{"bearer": []}]) + ); + } + input.generation += 1; + input.documents.insert("triggers".into(), json!({})); + publish_snapshot(store, &input) + .await + .expect("remove declaration"); + assert_eq!( + read_json(&client, (&trigger_path, &token.bearer)).await, + json!([]) + ); + assert_eq!( + read_json(&client, (&event_path, &token.bearer)).await, + events + ); + assert_eq!(read_json(&client, (&tick_path, &token.bearer)).await, ticks); + change_run( + store, + RunAuthority::Scheduler(&lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Finished { + outcome: RunOutcome::Failed, + reason: "Stop scoped read fixture".into(), + exit_code: Some(1), + }, + }, + ) + .await + .expect("terminalize worker and release liveness"); + assert_eq!( + read_json(&client, (&invocation_path, &token.bearer)).await, + invocations, + "invocation discovery uses retained facts, not active liveness" + ); + assert_eq!( + read_json(&client, (&recall_path, &token.bearer)).await, + evidence, + "historical Recall remains authorized after the active claim is removed" + ); + revoke_token(store, &audit, token.identity.token_id) + .await + .expect("revoke location"); + for path in paths { + assert_eq!( + client + .get(path) + .bearer_auth(&token.bearer) + .send() + .await + .expect("revoked read") + .status(), + StatusCode::UNAUTHORIZED + ); + } + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_statistics_recovery.rs b/crates/sqlbuild-orchestration/tests/api/test_statistics_recovery.rs new file mode 100644 index 0000000000..a39529267c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_statistics_recovery.rs @@ -0,0 +1,106 @@ +//! A reset during optional cleanup stops neither the maintenance role nor the HTTP server. + +use std::time::Duration; + +use serde_json::Value; + +use crate::api::tests::changes_helpers::reset_proxy; +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::test_types::WorkerRedirectTestCase; +use crate::platform::main::run_maintenance::run_maintenance; +use crate::store::main::migrate_store::migrate_store; +use crate::store::models::ControlStore; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_statistics_cleanup_when_connection_resets_then_maintenance_and_http_survive() { + let test_cases = [WorkerRedirectTestCase { + description: "PostgreSQL cleanup TCP reset", + expected_status: reqwest::StatusCode::OK, + }]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(Backend::Postgres) + .await + .expect("isolated store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let ControlStore::Postgres(pool) = store else { + panic!("PostgreSQL case") + }; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json", + ); + let data: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("fixture")) + .expect("statistics fixture"); + let fault = &data["cleanup_interruption"]; + sqlx_core::raw_sql::raw_sql(fault["block_sql"].as_str().expect("block SQL")) + .execute(pool) + .await + .expect("pause actual cleanup"); + let proxy = reset_proxy(store).await; + let api = start_api(&proxy.store).await; + let (stop, shutdown) = tokio::sync::watch::channel(false); + let maintenance_store = proxy.store.clone(); + let maintenance = tokio::spawn(async move { + run_maintenance( + (&maintenance_store, uuid::Uuid::now_v7()), + (Default::default(), &[]), + (|_| {}, shutdown), + ) + .await + }); + tokio::time::timeout( + Duration::from_secs(6), + sqlx_core::raw_sql::raw_sql(fault["wait_sql"].as_str().expect("wait SQL")) + .execute(pool), + ) + .await + .expect("bounded active cleanup wait") + .expect("actual prune is active"); + proxy.reset.send_modify(|epoch| *epoch += 1); + tokio::time::sleep(Duration::from_millis(100)).await; + // A TCP reset leaves the backend sleeping with the shared database's write lock. + // Cancel only this fixture's query after injecting the reset, before dropping its trigger. + sqlx_core::raw_sql::raw_sql(fault["restore_sql"].as_str().expect("restore SQL")) + .execute(pool) + .await + .expect("remove fault"); + let response = reqwest::Client::new() + .get(format!("{}/api/openapi.json", api.url)) + .send() + .await + .expect("HTTP survives cleanup reset"); + assert_eq!( + response.status(), + test_case.expected_status, + "{}", + test_case.description + ); + let schema: Value = response.json().await.expect("OpenAPI after cleanup reset"); + assert!( + schema["paths"] + .get("/api/control/locations/{location_id}/targets/{target}/statistics") + .is_some() + ); + assert!( + !maintenance.is_finished(), + "{}: the maintenance role survives the cleanup reset", + test_case.description + ); + stop.send(true).expect("stop maintenance"); + tokio::time::timeout(Duration::from_secs(5), maintenance) + .await + .expect("maintenance stops promptly") + .expect("join maintenance") + .expect("maintenance stopped cleanly"); + stop_api(api).await; + close_store(proxy.store).await; + proxy.shutdown.send(()).expect("stop proxy"); + proxy.task.await.expect("proxy stopped"); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs new file mode 100644 index 0000000000..2fbd082dd2 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -0,0 +1,119 @@ +use crate::runs::models::RunStatus; +use crate::store::errors::ControlError; +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct ApiContractTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_status: RunStatus, +} + +pub(crate) struct DevelopmentTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_status: crate::declarations::models::TriggerStatus, +} + +pub(crate) struct WorkerRedirectTestCase { + pub(crate) description: &'static str, + pub(crate) expected_status: reqwest::StatusCode, +} + +pub(crate) struct WorkerDeadlineTestCase { + pub(crate) description: &'static str, + pub(crate) budget: std::time::Duration, + pub(crate) expected_kind: sqlbuild_recall::store::errors::StoreErrorKind, + pub(crate) expected_arrival: bool, +} + +pub(crate) struct ApiReadTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_runs: usize, + pub(crate) expected_ticks: usize, +} + +pub(crate) struct TestApi { + pub(crate) url: String, + pub(crate) shutdown: tokio::sync::oneshot::Sender<()>, + pub(crate) task: tokio::task::JoinHandle>, +} + +pub(crate) struct SseConnection { + pub(crate) response: reqwest::Response, + pub(crate) pending: Vec, +} + +pub(crate) struct ResetProxy { + pub(crate) store: crate::store::models::ControlStore, + pub(crate) reset: tokio::sync::watch::Sender, + pub(crate) shutdown: tokio::sync::oneshot::Sender<()>, + pub(crate) task: tokio::task::JoinHandle<()>, +} + +pub(crate) struct ChangesTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_generation: i64, +} + +pub(crate) struct LocationHealthTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_observations: usize, +} + +pub(crate) struct LocationWorkTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_generation: i64, +} + +pub(crate) struct WarehouseMetadataTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_observations: usize, +} + +pub(crate) struct PasswordApiTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_lifetime: i64, +} + +pub(crate) struct ProxyApiTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) trusted_peer: &'static str, + pub(crate) expected_sessions: i64, +} + +pub(crate) struct ExternalSessionTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_viewer_statuses: [u16; 3], +} + +pub(crate) struct OidcApiTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_authentications: usize, +} + +pub(crate) struct OidcThrottleTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_allowed: usize, +} + +pub(crate) struct CliLoginTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_exchanges: usize, +} + +pub(crate) struct CustomRoleApiTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_statuses: [u16; 2], +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs new file mode 100644 index 0000000000..ef1a0ad714 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -0,0 +1,371 @@ +//! A scoped HTTP reader mirrors immutable warehouse observations, including external runs. + +use chrono::{Duration, Utc}; +use reqwest::StatusCode; +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::changes_helpers::{connect, expect_topics}; +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::test_types::WarehouseMetadataTestCase; +use crate::credentials::main::tokens::issue_token::issue_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::credentials::tests::location_helpers::create_location; +use crate::platform::main::connect_location_work::connect_location_work; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::models::{LocationHealth, LocationHealthReport}; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::TransactionMode; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::warehouse_metadata::models::{MetadataBatch, WarehouseHistoryKind}; + +#[test] +fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicity_are_preserved() { + let test_cases = [ + WarehouseMetadataTestCase { + description: "SQLite warehouse history", + backend: Backend::Sqlite, + expected_observations: 4, + }, + WarehouseMetadataTestCase { + description: "PostgreSQL warehouse history", + backend: Backend::Postgres, + expected_observations: 4, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let input = snapshot(location); + publish_snapshot(store, &input).await.expect("snapshot"); + let session_id = Uuid::now_v7(); + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: "development".into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("registered reader"); + let token = issue_token( + store, + &AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + CredentialScope::Location { + location_id: location, + }, + Utc::now() + Duration::hours(1), + ) + .await + .expect("token"); + let api = start_api(store).await; + let client = reqwest::Client::new(); + let url = format!( + "{}/api/locations/{location}/targets/development/metadata", + api.url + ); + let record = json!({ + "event_id": "warehouse-observation-1", + "state_database": null, "state_schema": "main", "source_name": "raw_orders", + "target_database": null, "target_schema": "main", "target_name": "orders", + "run_id": "external-command-1", "strategy": "max_column", "value_kind": "integer", + "data_version": "18446744073709551617", "data_version_hash": "same-data-version", + "observed_at": "2026-01-01T10:00:00Z", + }); + let mut later = record.clone(); + later["event_id"] = json!("warehouse-observation-2"); + later["observed_at"] = json!("2026-01-01T11:00:00Z"); + let mut other_run = record.clone(); + other_run + .as_object_mut() + .expect("record") + .remove("event_id"); + other_run["run_id"] = json!("external-command-2"); + let mut distinct = record.clone(); + distinct["event_id"] = json!("warehouse-observation-3"); + let mut page = vec![record.clone(); 100]; + page[1] = later; + page[2] = other_run.clone(); + page[3] = distinct; + // Both missing and explicitly NULL identities use the same fixed legacy fallback. + other_run["event_id"] = json!(null); + page[4] = other_run; + let batch = json!({"kind": "source_freshness", "session_id": session_id, "generation": 1, "records": page}); + let outbound = connect_location_work((&api.url, &token.bearer), location) + .expect("pooled native location client"); + let typed: MetadataBatch = serde_json::from_value(batch.clone()).expect("page"); + let mut changes = + connect(&client, &api, (location, "development", &token.bearer)).await; + assert!( + outbound + .sync_metadata("development", &typed) + .await + .expect("native client sync") + .applied + ); + expect_topics(&mut changes, &["node_status:development".into()]).await; + assert!( + !outbound + .sync_metadata("development", &typed) + .await + .expect("native client replay") + .applied + ); + drop(changes); + let mut equivalent = batch.clone(); + equivalent["records"] = json!([record]); + equivalent["records"][0]["observed_at"] = json!("2026-01-01T11:00:00+01:00"); + assert_eq!( + client + .post(&url) + .bearer_auth(&token.bearer) + .json(&equivalent) + .send() + .await + .expect("equivalent timestamp") + .json::() + .await + .expect("reply"), + json!({"applied": false}) + ); + let mut stale = batch.clone(); + stale["generation"] = json!(2); + let mut invalid = batch.clone(); + invalid["records"][1]["source_name"] = json!(" "); + let mut invalid_id = batch.clone(); + invalid_id["records"][0]["event_id"] = json!(" "); + let mut oversized = batch.clone(); + oversized["records"] = json!(vec![record.clone(); 101]); + let mut unknown = batch.clone(); + unknown["records"][0]["status"] = json!("success"); + let foreign = url.replace(&location.to_string(), &other.to_string()); + let other_target = url.replace("/development/", "/production/"); + for (endpoint, body, expected) in [ + (&url, stale, StatusCode::CONFLICT), + (&url, invalid, StatusCode::BAD_REQUEST), + (&url, invalid_id, StatusCode::BAD_REQUEST), + (&url, oversized, StatusCode::BAD_REQUEST), + (&url, unknown, StatusCode::BAD_REQUEST), + (&foreign, batch.clone(), StatusCode::FORBIDDEN), + (&other_target, batch.clone(), StatusCode::BAD_REQUEST), + ] { + assert_eq!( + client + .post(endpoint) + .bearer_auth(&token.bearer) + .json(&body) + .send() + .await + .expect("rejected page") + .status(), + expected + ); + } + assert_eq!( + client + .post(&url) + .body("invalid") + .send() + .await + .expect("anonymous") + .status(), + StatusCode::UNAUTHORIZED + ); + let fault = TransactionFault { + description: "warehouse page commits all records or none", + table: "sqb_warehouse_freshness", + operation: "INSERT", + condition: "WHEN NEW.record_json->>'source_name'='inventory'", + sqlite_body: "SELECT RAISE(ABORT, 'injected metadata failure')", + postgres_body: "RAISE EXCEPTION 'injected metadata failure'", + }; + install_fault(store, &fault).await; + let mut new_record = record.clone(); + new_record["state_schema"] = json!("reporting"); + let mut inventory = new_record.clone(); + inventory["event_id"] = json!("inventory-observation-1"); + inventory["source_name"] = json!("inventory"); + let mut additional = batch.clone(); + additional["records"] = json!([new_record, inventory]); + assert_eq!( + client + .post(&url) + .bearer_auth(&token.bearer) + .json(&additional) + .send() + .await + .expect("faulted page") + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + remove_fault(store, &fault).await; + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await.expect("read"), + "SELECT record_json FROM sqb_warehouse_freshness ORDER BY observed_at,event_id", + vec![], + ) + .await + .expect("mirrored history"); + commit(transaction).await.expect("close read"); + assert_eq!( + rows.len(), + test_case.expected_observations, + "{}: no observation collapsed by data version and no partial page committed", + test_case.description + ); + for row in rows { + assert_eq!(row["record_json"]["data_version"], "18446744073709551617"); + assert_eq!(row["record_json"]["state_schema"], "main"); + } + assert_eq!( + client + .post(&url) + .bearer_auth(&token.bearer) + .json(&additional) + .send() + .await + .expect("retry after rollback") + .json::() + .await + .expect("reply"), + json!({"applied": true}) + ); + migrate_store(store) + .await + .expect("warm migration preserves observations"); + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await.expect("read"), + "SELECT COUNT(*) AS n FROM sqb_warehouse_freshness", + vec![], + ) + .await + .expect("retained history"); + commit(transaction).await.expect("close read"); + assert_eq!(rows[0]["n"], 6); + // The same warehouse rows in another target are separate observations, not replays. + let preview_session = Uuid::now_v7(); + let mut preview = input.clone(); + preview.target = "preview".into(); + publish_snapshot(store, &preview) + .await + .expect("preview snapshot"); + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: preview_session, + target: "preview".into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("preview reader"); + let mut preview_batch = batch.clone(); + preview_batch["session_id"] = json!(preview_session); + let preview_url = url.replace("/development/", "/preview/"); + assert_eq!( + client + .post(&preview_url) + .bearer_auth(&token.bearer) + .json(&preview_batch) + .send() + .await + .expect("separate target") + .json::() + .await + .expect("reply"), + json!({"applied": true}) + ); + let (transaction, rows) = query(begin(store, TransactionMode::Read).await.expect("read"), + "SELECT target, COUNT(*) AS n FROM sqb_warehouse_freshness GROUP BY target ORDER BY target", vec![]).await.expect("isolated history"); + commit(transaction).await.expect("close read"); + assert_eq!( + serde_json::to_value(rows).expect("rows"), + json!([{"target": "development", "n": 6}, {"target": "preview", "n": 4}]) + ); + crate::api::tests::warehouse_metadata_helpers::fingerprint_history( + store, + &outbound, + (session_id, preview_session), + (&client, &url, &token.bearer), + ) + .await; + crate::api::tests::warehouse_reconciliation_helpers::reconcile_history( + store, + (&client, &url, &token.bearer), + (session_id, preview_session), + ) + .await; + for event_case in [ + ( + WarehouseHistoryKind::AuditResults, + "audit_result_history", + "result_id", + ), + ( + WarehouseHistoryKind::JanitorEvents, + "janitor_event_history", + "event_id", + ), + ( + WarehouseHistoryKind::Migrations, + "migration_history", + "event_id", + ), + ( + WarehouseHistoryKind::OldNameViews, + "old_name_history", + "event_id", + ), + ( + WarehouseHistoryKind::Microbatches, + "microbatch_history", + "event_id", + ), + ] { + crate::api::tests::warehouse_events_helpers::event_history( + store, + &outbound, + (session_id, preview_session), + (&client, &url, &token.bearer, event_case), + ) + .await; + } + crate::api::tests::warehouse_catalog_helpers::catalog_history( + store, + &outbound, + (session_id, preview_session), + (&client, &url, &token.bearer), + ) + .await; + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs new file mode 100644 index 0000000000..0d21031a7c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -0,0 +1,478 @@ +use std::collections::BTreeMap; + +use crate::credentials::main::tokens::issue_token::issue_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{ + NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::worker::errors::WorkerError; +use crate::worker::main::connect_worker::connect_worker; +use crate::worker::main::start_worker::start_worker; +use axum::Router; +use axum::response::Redirect; +use axum::routing::get; +use chrono::Utc; +use uuid::Uuid; + +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::output_helpers::{chunk, output_contract}; +use crate::api::tests::recall_helpers::recall_contract; +use crate::api::tests::test_types::{ + ApiContractTestCase, TestApi, WorkerDeadlineTestCase, WorkerRedirectTestCase, +}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_and_reports_are_retryable() + { + let test_cases = [ + ApiContractTestCase { + description: "SQLite worker HTTP lifecycle", + backend: Backend::Sqlite, + expected_status: RunStatus::Success, + }, + ApiContractTestCase { + description: "PostgreSQL worker HTTP lifecycle", + backend: Backend::Postgres, + expected_status: RunStatus::Success, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let baseline = snapshot(location); + let publication = publish_snapshot(store, &baseline).await.expect("publish"); + queue( + store, + &request((location, publication.snapshot_id), 1, 0, &["orders"]), + ) + .await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("acquire") + .expect("leader"); + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim") + .expect("work"); + let identity = (claim.request.run_id, claim.claim_id); + let token = issue_token( + store, + &AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + CredentialScope::Run { + run_id: identity.0, + claim_id: identity.1, + }, + Utc::now() + chrono::Duration::hours(1), + ) + .await + .expect("worker token"); + let api = start_api(store).await; + let worker = connect_worker(&api.url, &token.bearer, identity).expect("worker client"); + assert!( + !format!("{worker:?}").contains(&token.bearer), + "{}", + test_case.description + ); + let wrong_claim = connect_worker(&api.url, &token.bearer, (identity.0, Uuid::now_v7())) + .expect("different local claim"); + assert!( + matches!( + wrong_claim.execution().await, + Err(WorkerError::Configuration(_)) + ), + "{}", + test_case.description + ); + let mut drifted = baseline.clone(); + drifted + .nodes + .get_mut("inventory") + .expect("unselected dependency") + .raw_sql = Some("select 2 as id".into()); + assert!( + matches!( + start_worker(&worker, &drifted).await, + Err(WorkerError::Verification(_)) + ), + "{}", + test_case.description + ); + let runs = list_runs(store, (location, "development"), 10) + .await + .expect("unstarted"); + assert_eq!( + runs[0].status, + RunStatus::Queued, + "{}", + test_case.description + ); + change_run( + store, + RunAuthority::Scheduler(&lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: identity.0, + claim_id: Some(identity.1), + change: RunChange::Launched { container_id: None }, + }, + ) + .await + .expect("record process launch"); + let session = start_worker(&worker, &baseline) + .await + .expect("verified start"); + output_contract(store, &worker, identity.1).await; + let recall_batch = recall_contract(store, &worker, identity.1).await; + assert_eq!(session.execution.claim, claim, "{}", test_case.description); + assert_eq!( + start_worker(&worker, &baseline) + .await + .expect("retry start") + .execution, + session.execution, + "{}", + test_case.description + ); + assert!( + !worker.heartbeat().await.expect("heartbeat"), + "{}", + test_case.description + ); + let started = Uuid::now_v7(); + assert!( + worker + .report( + started, + RunChange::NodeStarted { + node_key: "orders".into() + } + ) + .await + .expect("node start"), + "{}", + test_case.description + ); + assert!( + !worker + .report( + started, + RunChange::NodeStarted { + node_key: "orders".into() + } + ) + .await + .expect("retry node start"), + "{}", + test_case.description + ); + worker + .report( + Uuid::now_v7(), + RunChange::NodeFinished { + node_key: "orders".into(), + outcome: NodeOutcome::Success, + rows_affected: Some(7), + error_code: None, + skipped_because_of: None, + }, + ) + .await + .expect("node outcome"); + worker + .report( + Uuid::now_v7(), + RunChange::Finished { + outcome: RunOutcome::Success, + reason: "warehouse work completed".into(), + exit_code: Some(0), + }, + ) + .await + .expect("finish"); + assert!(matches!( + worker.publish_recall(&recall_batch).await, + Err(WorkerError::Rejected(reqwest::StatusCode::UNAUTHORIZED)) + )); + assert!(matches!( + worker + .publish_output(&[chunk( + 5, + sqlbuild_recall::records::models::OutputStream::Stdout, + b"late" + )]) + .await, + Err(WorkerError::Rejected(reqwest::StatusCode::UNAUTHORIZED)) + )); + let runs = list_runs(store, (location, "development"), 10) + .await + .expect("completed"); + assert_eq!( + runs[0].status, test_case.expected_status, + "{}", + test_case.description + ); + assert!( + matches!( + worker.heartbeat().await, + Err(WorkerError::Rejected(reqwest::StatusCode::UNAUTHORIZED)) + ), + "{}", + test_case.description + ); + queue( + store, + &request((location, publication.snapshot_id), 2, 0, &["orders"]), + ) + .await; + let cancelled_claim = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim another run") + .expect("queued work"); + let cancelled_identity = (cancelled_claim.request.run_id, cancelled_claim.claim_id); + let cancelled_token = issue_token( + store, + &AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + CredentialScope::Run { + run_id: cancelled_identity.0, + claim_id: cancelled_identity.1, + }, + Utc::now() + chrono::Duration::hours(1), + ) + .await + .expect("another worker credential"); + change_run( + store, + RunAuthority::Scheduler(&lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: cancelled_identity.0, + claim_id: Some(cancelled_identity.1), + change: RunChange::Launched { container_id: None }, + }, + ) + .await + .expect("another launched process"); + change_run( + store, + RunAuthority::Operator(&AuditContext { + actor: "test-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: cancelled_identity.0, + claim_id: Some(cancelled_identity.1), + change: RunChange::CancelRequested { + actor: "test-operator".into(), + reason: "Stop before execution".into(), + }, + }, + ) + .await + .expect("cancel before execution"); + let cancelled_worker = + connect_worker(&api.url, &cancelled_token.bearer, cancelled_identity) + .expect("cancelled worker connection"); + assert!( + matches!( + start_worker(&cancelled_worker, &baseline).await, + Err(WorkerError::Cancelled) + ), + "{}", + test_case.description + ); + let runs = list_runs(store, (location, "development"), 10) + .await + .expect("cancelled run has not started"); + let cancelled_run = runs + .iter() + .find(|run| run.run_id == cancelled_identity.0) + .expect("cancelled projection"); + assert_eq!( + cancelled_run.status, + RunStatus::Launched, + "{}", + test_case.description + ); + assert!( + cancelled_run.started_at.is_none(), + "{}", + test_case.description + ); + drop(session); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_credentialed_worker_when_the_server_redirects_then_the_redirect_is_not_followed() { + let test_cases = [WorkerRedirectTestCase { + description: "a run bearer must not be forwarded by an API redirect", + expected_status: reqwest::StatusCode::TEMPORARY_REDIRECT, + }]; + for test_case in test_cases { + run(async { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("isolated redirect server"); + let url = format!("http://{}", listener.local_addr().expect("address")); + let application = Router::new().route( + "/api/runs/{run_id}/execution", + get(|| async { Redirect::temporary("/must-not-receive-a-credential") }), + ); + let (shutdown, signal) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + axum::serve(listener, application) + .with_graceful_shutdown(async { + let _ = signal.await; + }) + .await + .map_err(ControlError::from) + }); + let api = TestApi { + url, + shutdown, + task, + }; + let client = connect_worker( + &api.url, + "synthetic-worker-bearer", + (Uuid::now_v7(), Uuid::now_v7()), + ) + .expect("worker client"); + let Err(WorkerError::Rejected(status)) = client.execution().await else { + panic!("{}", test_case.description); + }; + assert_eq!( + status, test_case.expected_status, + "{}", + test_case.description + ); + stop_api(api).await; + }); + } +} + +#[test] +fn given_a_slow_recall_server_when_the_delivery_deadline_expires_then_the_native_store_stops_waiting() + { + use sqlbuild_recall::records::models::RowEnvelope; + use sqlbuild_recall::store::errors::StoreErrorKind; + use sqlbuild_recall::writer::models::SessionStore; + use std::time::{Duration, Instant}; + + let test_cases = [ + WorkerDeadlineTestCase { + description: "HTTP wait expires within the close budget", + budget: Duration::from_millis(50), + expected_kind: StoreErrorKind::Busy, + expected_arrival: true, + }, + WorkerDeadlineTestCase { + description: "expired budget sends no request", + budget: Duration::ZERO, + expected_kind: StoreErrorKind::Busy, + expected_arrival: false, + }, + ]; + for test_case in test_cases { + run(async { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind slow server"); + let url = format!( + "http://{}", + listener.local_addr().expect("slow server address") + ); + let arrived = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let observed = arrived.clone(); + let application = Router::new().route( + "/api/runs/{run_id}/recall", + axum::routing::post(move || { + let arrived = arrived.clone(); + async move { + arrived.store(true, std::sync::atomic::Ordering::SeqCst); + tokio::time::sleep(Duration::from_secs(2)).await; + axum::Json(sqlbuild_recall::store::models::InsertOutcome::default()) + } + }), + ); + let task = tokio::spawn(async move { axum::serve(listener, application).await }); + let client = connect_worker( + &url, + "synthetic-worker-bearer", + (Uuid::now_v7(), Uuid::now_v7()), + ) + .expect("connect worker to slow server"); + let SessionStore::Hosted(opener) = client.recall_store() else { + panic!("hosted store") + }; + tokio::task::spawn_blocking(move || { + let envelope = RowEnvelope { + invocation_id: "deadline".into(), + invocation_started_at: "2026-01-02T03:04:05Z".into(), + producer_version: "test".into(), + }; + let store = opener + .open(&envelope, Instant::now() + Duration::from_secs(1)) + .expect("open HTTP store"); + let started = Instant::now(); + let error = store + .insert(&[], started + test_case.budget) + .expect_err("deadline rejects delivery"); + assert_eq!( + error.kind, test_case.expected_kind, + "{}", + test_case.description + ); + assert!( + started.elapsed() < Duration::from_secs(1), + "{}", + test_case.description + ); + }) + .await + .expect("join slow-server publication"); + assert_eq!( + observed.load(std::sync::atomic::Ordering::SeqCst), + test_case.expected_arrival, + "{}", + test_case.description + ); + task.abort(); + let _ = task.await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/helpers.rs new file mode 100644 index 0000000000..01aa5d9916 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/catalog/helpers.rs @@ -0,0 +1,216 @@ +//! Catalog scans replace their namespace atomically through real HTTP. + +use reqwest::{Client, StatusCode}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::platform::models::LocationWorkClient; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::close_store; +use crate::warehouse_metadata::models::MetadataBatch; + +pub(crate) async fn catalog_history( + store: &ControlStore, + outbound: &LocationWorkClient, + (session_id, preview_session): (Uuid, Uuid), + (client, url, bearer): (&Client, &str, &str), +) { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json"); + let fixture: Value = serde_json::from_str(&std::fs::read_to_string(path).expect("fixture")) + .expect("catalog fixture"); + let record = fixture["record"].clone(); + let mut batch = json!({"kind":"catalog","session_id":session_id,"generation":1,"records":[record], + "scan":{"scan_id":Uuid::now_v7(),"location":{"database":null,"schema":"main"},"complete":true}}); + let typed: MetadataBatch = serde_json::from_value(batch.clone()).expect("catalog batch"); + assert!( + outbound + .sync_metadata("development", &typed) + .await + .expect("native catalog client") + .applied + ); + assert!( + !outbound + .sync_metadata("development", &typed) + .await + .expect("lost response retry") + .applied + ); + let initial = current(store).await; + assert_eq!(initial.as_array().expect("current").len(), 1); + let mut unscanned = batch.clone(); + unscanned["scan"] = Value::Null; + for (field, value, status) in [ + ("generation", json!(2), StatusCode::CONFLICT), + ( + "session_id", + json!(preview_session), + StatusCode::BAD_REQUEST, + ), + ("records", json!([record, record]), StatusCode::BAD_REQUEST), + ] { + let mut invalid = batch.clone(); + invalid[field] = value; + post((client, url, bearer), &invalid, status).await; + } + post((client, url, bearer), &unscanned, StatusCode::BAD_REQUEST).await; + post( + ( + client, + &url.replace( + &outbound.location_id.to_string(), + &Uuid::now_v7().to_string(), + ), + bearer, + ), + &batch, + StatusCode::FORBIDDEN, + ) + .await; + batch["scan"]["scan_id"] = json!(Uuid::now_v7()); + assert_eq!( + post((client, url, bearer), &batch, StatusCode::OK).await["applied"], + false + ); + assert_eq!( + current(store).await, + initial, + "an unchanged scan changes nothing" + ); + batch["scan"]["scan_id"] = json!(Uuid::now_v7()); + batch["records"][0]["columns"][1]["data_type"] = json!("INTEGER"); + let fault = TransactionFault { + description: "catalog change rollback", + table: "sqb_warehouse_catalog", + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected catalog failure')", + postgres_body: "RAISE EXCEPTION 'injected catalog failure'", + }; + install_fault(store, &fault).await; + post( + (client, url, bearer), + &batch, + StatusCode::INTERNAL_SERVER_ERROR, + ) + .await; + remove_fault(store, &fault).await; + assert_eq!(current(store).await, initial, "a failed page rolls back"); + let replica = second_store(store).await; + let transaction = begin(store, TransactionMode::Write) + .await + .expect("held catalog writer"); + let publisher = outbound.clone(); + let typed: MetadataBatch = + serde_json::from_value(batch.clone()).expect("changed catalog batch"); + let mut publication = + tokio::spawn(async move { publisher.sync_metadata("development", &typed).await }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(100), &mut publication) + .await + .is_err() + ); + assert_eq!( + current(&replica).await, + initial, + "reader retains the previous committed cut" + ); + commit(transaction).await.expect("release catalog writer"); + assert!( + tokio::time::timeout(std::time::Duration::from_secs(20), publication) + .await + .expect("catalog publication deadline") + .expect("join publication") + .expect("serialized publication") + .applied + ); + close_store(replica).await; + assert_eq!( + current(store).await[0]["record_json"]["columns"][1]["data_type"], + "INTEGER" + ); + let kept = batch["records"][0].clone(); + let mut obsolete: Vec = Vec::new(); + for index in 0..205 { + let mut item = record.clone(); + item["name"] = json!(format!("obsolete-{index:03}")); + obsolete.push(item); + } + for records in obsolete.chunks(100) { + batch["scan"]["scan_id"] = json!(Uuid::now_v7()); + batch["scan"]["complete"] = json!(false); + batch["records"] = json!(records); + post((client, url, bearer), &batch, StatusCode::OK).await; + } + let before_deletion = current(store).await; + assert_eq!(before_deletion.as_array().expect("current").len(), 206); + batch["scan"]["scan_id"] = json!(Uuid::now_v7()); + batch["records"] = json!([kept]); + post((client, url, bearer), &batch, StatusCode::OK).await; + assert_eq!( + current(store).await, + before_deletion, + "an incomplete scan removes nothing" + ); + batch["scan"]["complete"] = json!(true); + batch["records"] = json!([]); + assert_eq!( + post((client, url, bearer), &batch, StatusCode::OK).await["applied"], + true + ); + let replaced = current(store).await; + assert_eq!( + replaced + .as_array() + .expect("current") + .iter() + .map(|row| row["record_json"]["name"].clone()) + .collect::>(), + vec![record["name"].clone()], + "a complete scan keeps exactly the relations it delivered" + ); + assert_eq!( + post((client, url, bearer), &batch, StatusCode::OK).await["applied"], + false + ); + assert_eq!( + current(store).await, + replaced, + "replaying the final page changes nothing" + ); +} + +async fn post( + (client, url, bearer): (&Client, &str, &str), + body: &Value, + expected: StatusCode, +) -> Value { + let response = client + .post(url) + .bearer_auth(bearer) + .json(body) + .send() + .await + .expect("catalog request"); + let status = response.status(); + let body: Value = response.json().await.expect("catalog reply"); + assert_eq!(status, expected, "{body}"); + body +} + +async fn current(store: &ControlStore) -> Value { + let (transaction, current) = query( + begin(store, TransactionMode::Read).await.expect("read"), + "SELECT event_id,record_json FROM sqb_warehouse_catalog ORDER BY event_id", + vec![], + ) + .await + .expect("projection"); + commit(transaction).await.expect("close read"); + json!(current) +} diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs new file mode 100644 index 0000000000..9a274f5b61 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/events/helpers.rs @@ -0,0 +1,239 @@ +//! Warehouse events retain their identities, physical scopes and atomic replay semantics. + +use reqwest::{Client, StatusCode}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::platform::models::LocationWorkClient; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::warehouse_metadata::models::{ + WarehouseHistoryKind, WarehouseSyncResponse, WarehouseWatermarkRead, +}; + +pub(crate) async fn event_history( + store: &ControlStore, + outbound: &LocationWorkClient, + (session_id, preview_session): (Uuid, Uuid), + (client, metadata_url, bearer, (kind, fixture_name, source_key)): ( + &Client, + &str, + &str, + (WarehouseHistoryKind, &str, &str), + ), +) { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join(format!( + "../../benchmarks/duckshop/edge_cases/orchestration/{fixture_name}.json" + )); + let fixture: Value = serde_json::from_str(&std::fs::read_to_string(path).expect("fixture")) + .expect("event history"); + let timestamp = fixture["timestamp_key"].as_str().unwrap_or("occurred_at"); + let record = fixture["record"].clone(); + let batch = json!({"kind": kind, "session_id": session_id, "generation": 1, "records": [record, record]}); + let url = metadata_url; + for expected in [true, false] { + assert_eq!( + sync_events((client, url, bearer), &batch).await.applied, + expected + ); + } + let before = mirrored_events(store, kind).await; + assert_eq!( + before.len(), + 1, + "duplicate deterministic event identities deduplicate" + ); + assert_eq!( + before[0]["record_json"], record, + "external run IDs, text and timestamps survive" + ); + for (field, value, status) in [ + ("generation", json!(2), StatusCode::CONFLICT), + ( + "session_id", + json!(preview_session), + StatusCode::BAD_REQUEST, + ), + ( + "records", + json!(vec![record.clone(); 101]), + StatusCode::BAD_REQUEST, + ), + ] { + let mut invalid = batch.clone(); + invalid[field] = value; + assert_eq!( + client + .post(url) + .bearer_auth(bearer) + .json(&invalid) + .send() + .await + .expect("fenced batch") + .status(), + status + ); + } + assert_eq!( + client + .post(url.replace( + &outbound.location_id.to_string(), + &Uuid::now_v7().to_string() + )) + .bearer_auth(bearer) + .json(&batch) + .send() + .await + .expect("other location") + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + client + .post(url) + .json(&batch) + .send() + .await + .expect("anonymous") + .status(), + StatusCode::UNAUTHORIZED + ); + let mut scoped = record.clone(); + scoped["state_schema"] = json!("reporting"); + let mut database = record.clone(); + database["state_database"] = json!("analytics"); + let scopes = json!({"kind":kind,"session_id":session_id, + "generation":1, "records":[scoped,database]}); + assert!(sync_events((client, url, bearer), &scopes).await.applied); + let preview = json!({"kind":kind,"session_id":preview_session, + "generation":1, "records":[record]}); + assert!( + sync_events( + (client, &url.replace("/development/", "/preview/"), bearer), + &preview + ) + .await + .applied + ); + assert_eq!(mirrored_events(store, kind).await.len(), 4); + let watermark: WarehouseWatermarkRead = serde_json::from_value(json!({"session_id":session_id, + "generation":1,"kind":kind,"location":{"database":null,"schema":"main"}})) + .expect("watermark"); + assert_eq!( + json!( + outbound + .metadata_watermark("development", &watermark) + .await + .expect("watermark") + ), + record[timestamp] + ); + let fault = TransactionFault { + description: "event page rolls back atomically", + table: kind.mirror_table(), + operation: "INSERT", + condition: "WHEN NEW.source_event_id='rejected-result'", + sqlite_body: "SELECT RAISE(ABORT, 'injected audit failure')", + postgres_body: "RAISE EXCEPTION 'injected audit failure'", + }; + let before = mirrored_events(store, kind).await; + let mut first = record.clone(); + first[source_key] = json!("accepted-result"); + let mut second = record.clone(); + second[source_key] = json!("rejected-result"); + let additional = + json!({"kind":kind,"session_id":session_id,"generation":1,"records":[first,second]}); + install_fault(store, &fault).await; + assert_eq!( + client + .post(url) + .bearer_auth(bearer) + .json(&additional) + .send() + .await + .expect("fault") + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + remove_fault(store, &fault).await; + assert_eq!( + mirrored_events(store, kind).await, + before, + "no partial page survives" + ); + let (transaction, _) = query( + begin(store, TransactionMode::Write).await.expect("write"), + &format!("DELETE FROM {}", kind.mirror_table()), + vec![], + ) + .await + .expect("empty mirror"); + commit(transaction).await.expect("wipe"); + assert!(sync_events((client, url, bearer), &batch).await.applied); + assert!(sync_events((client, url, bearer), &scopes).await.applied); + assert!( + sync_events( + (client, &url.replace("/development/", "/preview/"), bearer), + &preview + ) + .await + .applied + ); + assert_eq!( + mirrored_events(store, kind).await, + before, + "identical facts after rebuild" + ); + migrate_store(store).await.expect("warm migration"); + assert_eq!(mirrored_events(store, kind).await, before); + assert_eq!( + client + .post(url) + .bearer_auth(bearer) + .json(&additional) + .send() + .await + .expect("retry") + .json::() + .await + .expect("response"), + json!({"applied":true}) + ); +} + +async fn mirrored_events(store: &ControlStore, kind: WarehouseHistoryKind) -> Vec { + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await.expect("read"), + &format!( + "SELECT event_id,target,record_json FROM {} ORDER BY event_id", + kind.mirror_table() + ), + vec![], + ) + .await + .expect("mirror"); + commit(transaction).await.expect("close read"); + rows.into_iter().map(|row| json!(row)).collect() +} + +async fn sync_events( + (client, url, bearer): (&Client, &str, &str), + batch: &Value, +) -> WarehouseSyncResponse { + client + .post(url) + .bearer_auth(bearer) + .json(batch) + .send() + .await + .expect("event sync") + .error_for_status() + .expect("accepted page") + .json() + .await + .expect("sync response") +} diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs new file mode 100644 index 0000000000..3e9aa5a83f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/helpers.rs @@ -0,0 +1,540 @@ +//! Exact definitions survive semantic-hash reuse, page retries and mirror rebuilds. + +use reqwest::{Client, StatusCode}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::platform::models::{LocationHealthReport, LocationWorkClient}; +use crate::store::main::decompress::decompress; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::warehouse_metadata::models::{MetadataBatch, WarehouseWatermarkRead}; + +pub(crate) async fn fingerprint_history( + store: &ControlStore, + outbound: &LocationWorkClient, + (session_id, preview_session_id): (Uuid, Uuid), + (client, freshness_url, bearer): (&Client, &str, &str), +) { + let url = freshness_url.replace("source-freshness", "fingerprints"); + let variants_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/fingerprint_definition_variants.json", + ); + let variants: Value = + serde_json::from_str(&std::fs::read_to_string(variants_path).expect("DuckShop fixture")) + .expect("DuckShop semantic-equivalence case"); + let definition = variants["definitions"][0]["sql"].as_str().expect("SQL"); + let variant = variants["definitions"][1]["sql"] + .as_str() + .expect("variant SQL"); + // Independently established with the public native hash function; raw UTF-8 differs. + let semantic = variants["semantic_hash"].as_str().expect("semantic hash"); + let record = json!({ + "event_id": "warehouse-fingerprint-1", "state_database": null, "state_schema": "main", + "node_type": "model", "node_name": "orders", "target_database": null, + "target_schema": "main", "target_name": "orders", "run_id": "external-command-1", + "definition_hash": semantic, "version_hash": "version-1", "schema_fingerprint": "schema-1", + "definition": definition, "metadata_json": "{\"audit\":1}", "ts": "2026-01-01T10:00:00Z", + }); + let mut other = record.clone(); + other["event_id"] = json!("warehouse-fingerprint-2"); + other["run_id"] = json!("external-command-2"); + other["definition"] = json!(variant); + let mut legacy = record.clone(); + legacy.as_object_mut().expect("record").remove("event_id"); + legacy["run_id"] = json!("legacy-command"); + let mut legacy_null = legacy.clone(); + legacy_null["event_id"] = Value::Null; + legacy_null["ts"] = json!("2026-01-01T11:00:00+01:00"); + let mut page = vec![record.clone(); 100]; + page[1] = other; + page[2] = legacy; + page[3] = legacy_null.clone(); + legacy_null["ts"] = json!("2026-01-01T10:00:00.000001Z"); + page[4] = legacy_null; + let batch = + json!({"kind": "fingerprints", "session_id": session_id, "generation": 1, "records": page}); + let typed: MetadataBatch = serde_json::from_value(batch.clone()).expect("typed page"); + for expected in [true, false] { + assert_eq!( + outbound + .sync_metadata("development", &typed) + .await + .expect("sync") + .applied, + expected + ); + } + let before = mirrored_fingerprints(store).await; + assert_eq!( + before.len(), + 4, + "replay and absent/NULL legacy IDs deduplicate" + ); + let mut texts = Vec::new(); + let mut hashes = std::collections::BTreeSet::new(); + let mut source_ids = std::collections::BTreeSet::new(); + for row in &before { + assert_eq!(row["record_json"]["definition_hash"], semantic); + assert!( + row["record_json"].get("definition").is_none(), + "event holds a reference, not another body" + ); + hashes.insert(row["definition_text_hash"].as_str().expect("hash")); + source_ids.insert(row["source_event_id"].as_str().expect("source ID")); + texts + .push(String::from_utf8(decompress(&row["body"]).expect("decompress")).expect("UTF-8")); + } + texts.sort(); + assert_eq!(texts, [variant, definition, definition, definition]); + assert_eq!( + source_ids, + std::collections::BTreeSet::from([ + "warehouse-fingerprint-1", + "warehouse-fingerprint-2", + "39e801c87fb3856345ca8ff4b4581edcd4439d8688c5fb399ac331948a004287", + "ac87eafefb99f262e0304fb1f5ec891a820af7ff9cdf1e97f44306861ffbbb18", + ]) + ); + assert_eq!( + hashes, + std::collections::BTreeSet::from([ + variants["definitions"][0]["text_hash"] + .as_str() + .expect("hash"), + variants["definitions"][1]["text_hash"] + .as_str() + .expect("hash"), + ]) + ); + let mut stale = batch.clone(); + stale["generation"] = json!(2); + let mut oversized = batch.clone(); + oversized["records"] = json!(vec![record.clone(); 101]); + let mut blank = batch.clone(); + blank["records"][0]["event_id"] = json!(" "); + let mut foreign = batch.clone(); + foreign["session_id"] = json!(Uuid::now_v7()); + for (endpoint, body, status) in [ + (url.clone(), stale, StatusCode::CONFLICT), + (url.clone(), oversized, StatusCode::BAD_REQUEST), + (url.clone(), blank, StatusCode::BAD_REQUEST), + (url.clone(), foreign, StatusCode::BAD_REQUEST), + ( + url.replace( + &outbound.location_id.to_string(), + &Uuid::now_v7().to_string(), + ), + batch.clone(), + StatusCode::FORBIDDEN, + ), + ] { + assert_eq!( + client + .post(endpoint) + .bearer_auth(bearer) + .json(&body) + .send() + .await + .expect("rejected page") + .status(), + status + ); + } + assert_eq!( + client + .post(&url) + .json(&batch) + .send() + .await + .expect("anonymous") + .status(), + StatusCode::UNAUTHORIZED + ); + let fault = TransactionFault { + description: "fingerprint event and definition page roll back together", + table: "sqb_warehouse_fingerprints", + operation: "INSERT", + condition: "WHEN NEW.record_json->>'node_name'='inventory'", + sqlite_body: "SELECT RAISE(ABORT, 'injected fingerprint failure')", + postgres_body: "RAISE EXCEPTION 'injected fingerprint failure'", + }; + let mut first = record.clone(); + first["event_id"] = json!("new-fingerprint-1"); + first["definition"] = json!("select product_id from products"); + let mut second = first.clone(); + second["event_id"] = json!("new-fingerprint-2"); + second["node_name"] = json!("inventory"); + second["definition"] = json!("select inventory_id from inventory"); + let additional = json!({"kind": "fingerprints", "session_id": session_id, "generation": 1, "records": [first, second]}); + let bodies_before = source_bodies(store).await; + install_fault(store, &fault).await; + assert_eq!( + client + .post(&url) + .bearer_auth(bearer) + .json(&additional) + .send() + .await + .expect("faulted page") + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + remove_fault(store, &fault).await; + assert_eq!( + mirrored_fingerprints(store).await, + before, + "no partial events" + ); + assert_eq!( + source_bodies(store).await, + bodies_before, + "no orphan definitions" + ); + // Rebuild from an empty mirror, including its unshared exact-content bodies. + let (transaction, _) = query( + begin(store, TransactionMode::Write).await.expect("write"), + "DELETE FROM sqb_warehouse_fingerprints", + vec![], + ) + .await + .expect("wipe mirror"); + let (transaction, _) = query(transaction, + "DELETE FROM sqb_source_texts WHERE NOT EXISTS (SELECT 1 FROM sqb_snapshot_sources s WHERE s.text_hash=sqb_source_texts.text_hash)", + vec![]).await.expect("wipe unshared bodies"); + commit(transaction).await.expect("commit rebuild start"); + assert!( + outbound + .sync_metadata("development", &typed) + .await + .expect("resync") + .applied + ); + assert_eq!( + mirrored_fingerprints(store).await, + before, + "same identities, facts and exact SQL after resync" + ); + assert_eq!( + source_bodies(store).await, + bodies_before, + "same deduplicated body set" + ); + assert_eq!( + client + .post(&url) + .bearer_auth(bearer) + .json(&additional) + .send() + .await + .expect("retry faulted page") + .json::() + .await + .expect("reply"), + json!({"applied": true}) + ); + // Review ownership markers are supported warehouse rows with intentionally empty hashes/SQL. + let mut marker = record.clone(); + marker["event_id"] = json!("warehouse-review-marker"); + for (key, value) in variants["review_marker"] + .as_object() + .expect("marker") + .iter() + { + marker[key] = value.clone(); + } + let marker: MetadataBatch = serde_json::from_value(json!({"kind":"fingerprints", + "session_id": session_id, "generation": 1, "records": [marker], + })) + .expect("review marker page"); + assert!( + outbound + .sync_metadata("development", &marker) + .await + .expect("review marker sync") + .applied + ); + let rows = mirrored_fingerprints(store).await; + let marker = rows + .iter() + .find(|row| row["source_event_id"] == "warehouse-review-marker") + .expect("stored marker"); + assert_eq!(marker["record_json"]["definition_hash"], ""); + assert_eq!(decompress(&marker["body"]).expect("marker definition"), b""); + let bodies = source_bodies(store).await; + let mut scoped = record.clone(); + scoped["state_schema"] = json!("reporting"); + let mut other_database = record.clone(); + other_database["state_database"] = json!("analytics"); + let scoped: MetadataBatch = serde_json::from_value(json!({"kind":"fingerprints", + "session_id": session_id, "generation": 1, "records": [scoped, other_database], + })) + .expect("physical scopes"); + assert!( + outbound + .sync_metadata("development", &scoped) + .await + .expect("distinct physical scopes") + .applied + ); + let preview: MetadataBatch = serde_json::from_value(json!({"kind":"fingerprints", + "session_id": preview_session_id, "generation": 1, "records": [record], + })) + .expect("preview target"); + assert!( + outbound + .sync_metadata("preview", &preview) + .await + .expect("distinct target scope") + .applied + ); + let after = mirrored_fingerprints(store).await; + assert_eq!( + after.len(), + 10, + "identical source IDs in other physical/target scopes are independent" + ); + assert_eq!( + source_bodies(store).await, + bodies, + "scopes share identical definition bytes" + ); + migrate_store(store).await.expect("warm migration"); + assert_eq!( + mirrored_fingerprints(store).await, + after, + "warm migration preserves imported history" + ); + for (kind, database, schema, target, reader, expected) in [ + ( + "fingerprints", + None, + "main", + "development", + session_id, + Some("2026-01-01T10:00:00.000001Z"), + ), + ( + "fingerprints", + Some("analytics"), + "main", + "development", + session_id, + Some("2026-01-01T10:00:00Z"), + ), + ( + "fingerprints", + None, + "reporting", + "development", + session_id, + Some("2026-01-01T10:00:00Z"), + ), + ( + "fingerprints", + None, + "main", + "preview", + preview_session_id, + Some("2026-01-01T10:00:00Z"), + ), + ( + "source_freshness", + None, + "main", + "development", + session_id, + Some("2026-01-01T11:00:00Z"), + ), + ( + "source_freshness", + Some("analytics"), + "main", + "development", + session_id, + None, + ), + ( + "fingerprints", + None, + "missing", + "development", + session_id, + None, + ), + ] { + let request: WarehouseWatermarkRead = serde_json::from_value(json!({ + "kind": kind, "session_id": reader, "generation": 1, + "location": {"database": database, "schema": schema}, + })) + .expect("watermark request"); + assert_eq!( + serde_json::to_value( + outbound + .metadata_watermark(target, &request) + .await + .expect("durable scoped watermark") + ) + .expect("watermark"), + json!(expected) + ); + } + let watermark_url = format!("{freshness_url}/watermark"); + let request = json!({"kind": "fingerprints", "session_id":session_id,"generation":1, + "location":{"database":null,"schema":"main"}}); + for (endpoint, mut body, status, field, value) in [ + ( + watermark_url.clone(), + request.clone(), + StatusCode::CONFLICT, + "generation", + json!(2), + ), + ( + watermark_url.clone(), + request.clone(), + StatusCode::BAD_REQUEST, + "session_id", + json!(preview_session_id), + ), + ( + watermark_url.clone(), + request.clone(), + StatusCode::UNPROCESSABLE_ENTITY, + "kind", + json!("arbitrary_table"), + ), + ( + watermark_url.replace( + &outbound.location_id.to_string(), + &Uuid::now_v7().to_string(), + ), + request.clone(), + StatusCode::FORBIDDEN, + "generation", + json!(1), + ), + ] { + body[field] = value; + assert_eq!( + client + .post(endpoint) + .bearer_auth(bearer) + .json(&body) + .send() + .await + .expect("fenced watermark") + .status(), + status + ); + } + assert_eq!( + client + .post(watermark_url) + .json(&request) + .send() + .await + .expect("anonymous watermark") + .status(), + StatusCode::UNAUTHORIZED + ); + let health_url = format!( + "{}/health", + freshness_url + .split("/targets/") + .next() + .expect("location URL") + ); + for failure in [None, Some("watermark"), Some("read"), Some("delivery")] { + let report: LocationHealthReport = serde_json::from_value(json!({ + "event_id":Uuid::now_v7(), "session_id":session_id, "target":"development", + "health":{"kind":"warehouse_sync", "generation":1, + "location":{"database":null,"schema":"main"},"table":"fingerprints", + "failure":failure,"records_sent":17,"elapsed_millis":43}, + })) + .expect("bounded sync health"); + for applied in [true, false] { + assert_eq!( + outbound + .report_health(&report) + .await + .expect("report health"), + applied + ); + } + let observations: Vec = client + .get(&health_url) + .bearer_auth(bearer) + .send() + .await + .expect("health read") + .error_for_status() + .expect("authorized health") + .json() + .await + .expect("health observations"); + assert_eq!( + observations + .iter() + .filter(|row| row["report"] == json!(report)) + .count(), + 1, + "exact retries preserve one historical health observation" + ); + let mut invalid = json!(report); + invalid["event_id"] = json!(Uuid::now_v7()); + invalid["health"]["generation"] = json!(2); + assert_eq!( + client + .post(&health_url) + .bearer_auth(bearer) + .json(&invalid) + .send() + .await + .expect("unpublished sync health") + .status(), + StatusCode::BAD_REQUEST + ); + invalid["health"]["generation"] = json!(1); + invalid["health"]["failure"] = json!("unbounded exception text"); + assert_eq!( + client + .post(&health_url) + .bearer_auth(bearer) + .json(&invalid) + .send() + .await + .expect("closed failure vocabulary") + .status(), + StatusCode::UNPROCESSABLE_ENTITY + ); + } +} + +async fn mirrored_fingerprints(store: &ControlStore) -> Vec { + let (transaction, rows) = query(begin(store, TransactionMode::Read).await.expect("read"), + "SELECT f.event_id,f.source_event_id,f.definition_text_hash,f.record_json,t.body FROM sqb_warehouse_fingerprints f \ + JOIN sqb_source_texts t ON t.text_hash=f.definition_text_hash ORDER BY f.event_id", vec![]).await.expect("mirrored definitions"); + commit(transaction).await.expect("close read"); + rows.into_iter() + .map(|row| serde_json::to_value(row).expect("row")) + .collect() +} + +async fn source_bodies(store: &ControlStore) -> Vec { + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await.expect("read"), + "SELECT text_hash,body FROM sqb_source_texts ORDER BY text_hash", + vec![], + ) + .await + .expect("immutable bodies"); + commit(transaction).await.expect("close read"); + rows.into_iter() + .map(|row| serde_json::to_value(row).expect("row")) + .collect() +} diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/reconciliation/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/reconciliation/helpers.rs new file mode 100644 index 0000000000..8185e12a73 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_metadata/reconciliation/helpers.rs @@ -0,0 +1,242 @@ +//! HTTP acceptance for full scans: only a complete scan removes, and only in its own namespace. + +use reqwest::{Client, StatusCode}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub(crate) async fn reconcile_history( + store: &ControlStore, + (client, url, bearer): (&Client, &str, &str), + (session_id, preview_session_id): (Uuid, Uuid), +) { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/warehouse_reconciliation.json"); + let fixture: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("DuckShop reconciliation")) + .expect("fixture"); + for (kind, table, timestamp) in [ + ("source_freshness", "sqb_warehouse_freshness", "observed_at"), + ("fingerprints", "sqb_warehouse_fingerprints", "ts"), + ] { + let record = fixture[kind].clone(); + let mut legacy = record.clone(); + legacy["event_id"] = Value::Null; + legacy["run_id"] = json!("orders-legacy-build"); + legacy[timestamp] = json!("2025-12-31T20:00:00-04:00"); + let mut originals = vec![record.clone(), legacy.clone()]; + for index in 0..fixture["obsolete_count"].as_u64().expect("obsolete count") { + let mut obsolete = record.clone(); + obsolete["event_id"] = json!(format!("reconciliation-obsolete-{index}")); + originals.push(obsolete); + } + for (id, field, value) in [ + ( + "reconciliation-other-database", + "state_database", + "analytics", + ), + ( + "reconciliation-other-schema", + "state_schema", + "reconcile_inventory", + ), + ] { + let mut control = record.clone(); + control["event_id"] = json!(id); + control[field] = json!(value); + originals.push(control); + } + for records in originals.chunks(100) { + post( + (client, url, bearer), + &json!({"kind":kind,"session_id":session_id,"generation":1,"records":records}), + StatusCode::OK, + ) + .await; + } + let mut preview = record.clone(); + preview["event_id"] = json!("reconciliation-preview"); + post( + (client, &url.replace("/development/", "/preview/"), bearer), + &json!({"kind":kind,"session_id":preview_session_id,"generation":1,"records":[preview]}), + StatusCode::OK, + ) + .await; + let before = observations(store, table).await; + let mut page = json!({"kind":kind,"session_id":session_id,"generation":1,"records":[record.clone(),legacy], + "scan":{"scan_id":Uuid::now_v7(),"location":{"database":null,"schema":"reconcile_orders"},"complete":false}}); + assert_eq!( + post((client, url, bearer), &page, StatusCode::OK).await, + json!({"applied":false}) + ); + assert_eq!( + observations(store, table).await, + before, + "an incomplete scan cannot delete" + ); + let mut foreign = page.clone(); + foreign["scan"]["location"]["database"] = json!("analytics"); + post((client, url, bearer), &foreign, StatusCode::BAD_REQUEST).await; + page["records"] = json!([]); + page["scan"]["complete"] = json!(true); + assert_eq!( + post((client, url, bearer), &page, StatusCode::OK).await, + json!({"applied":true}) + ); + assert_eq!( + source_ids(store, table).await, + [ + "", + "reconciliation-keep", + "reconciliation-other-database", + "reconciliation-other-schema", + "reconciliation-preview" + ], + "the scanned rows and every other namespace and target survive" + ); + assert_eq!( + post((client, url, bearer), &page, StatusCode::OK).await, + json!({"applied":false}), + "a lost final response is retry-safe" + ); + let empty = json!({"kind":kind,"session_id":session_id,"generation":1,"records":[],"scan":{ + "scan_id":Uuid::now_v7(),"location":{"database":null,"schema":"reconcile_orders"},"complete":true}}); + assert_eq!( + post((client, url, bearer), &empty, StatusCode::OK).await, + json!({"applied":true}) + ); + assert_eq!( + source_ids(store, table).await, + [ + "reconciliation-other-database", + "reconciliation-other-schema", + "reconciliation-preview" + ], + "a complete empty scan clears only its namespace" + ); + for records in originals.chunks(100) { + post( + (client, url, bearer), + &json!({"kind":kind,"session_id":session_id,"generation":1,"records":records}), + StatusCode::OK, + ) + .await; + } + assert_eq!( + observations(store, table).await, + before, + "a resync restores exact identities and observations" + ); + retained_history( + (store, table), + (client, url, bearer), + (session_id, &fixture, &record, (kind, timestamp)), + ) + .await; + } +} + +async fn retained_history( + (store, table): (&ControlStore, &str), + (client, url, bearer): (&Client, &str, &str), + (session_id, fixture, record, (kind, timestamp)): (Uuid, &Value, &Value, (&str, &str)), +) { + let count = fixture["retained_count"].as_u64().expect("retained count"); + let records: Vec<_> = (0..count) + .map(|index| { + let mut record = record.clone(); + record["event_id"] = json!(format!("retained-{index}")); + record["state_database"] = json!("analytics"); + record["state_schema"] = json!("reconcile_retained"); + record[timestamp] = json!(format!( + "2026-01-01T10:{:02}:{:02}Z", + index / 120, + (index / 2) % 60 + )); + record + }) + .collect(); + for records in records.chunks(100) { + post( + (client, url, bearer), + &json!({"kind":kind,"session_id":session_id,"generation":1,"records":records}), + StatusCode::OK, + ) + .await; + } + let mut page = json!({"kind":kind,"session_id":session_id,"generation":1,"records":[],"scan":{ + "scan_id":Uuid::now_v7(),"location":{"database":"analytics","schema":"reconcile_retained"},"complete":false}}); + let pages = records.len().div_ceil(100); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + for (index, records) in records.chunks(100).enumerate() { + page["records"] = json!(records); + page["scan"]["complete"] = json!(index + 1 == pages); + let reply = post((client, url, bearer), &page, StatusCode::OK).await; + assert_eq!( + reply["applied"], false, + "retained observations never change" + ); + } + }) + .await + .expect("a retained-heavy scan has a strict total deadline"); + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await.expect("read retained"), + &format!("SELECT source_event_id FROM {table} WHERE state_database='analytics' AND state_schema='reconcile_retained' ORDER BY source_event_id"), + vec![], + ).await.expect("retained identities"); + commit(transaction).await.expect("close retained read"); + let mut expected: Vec<_> = (0..count) + .map(|index| format!("retained-{index}")) + .collect(); + expected.sort(); + assert_eq!( + rows.iter() + .map(|row| row["source_event_id"].as_str().expect("identity")) + .collect::>(), + expected + ); +} + +async fn post( + (client, url, bearer): (&Client, &str, &str), + body: &Value, + expected: StatusCode, +) -> Value { + let response = client + .post(url) + .bearer_auth(bearer) + .json(body) + .send() + .await + .expect("metadata request"); + let status = response.status(); + let body: Value = response.json().await.expect("metadata response"); + assert_eq!(status, expected, "{body}"); + body +} + +async fn observations(store: &ControlStore, table: &str) -> Vec { + let (transaction, rows) = query(begin(store, TransactionMode::Read).await.expect("read"), + &format!("SELECT event_id,target,source_event_id,record_json FROM {table} WHERE state_schema IN ('reconcile_orders','reconcile_inventory') ORDER BY event_id"), vec![]).await.expect("mirror facts"); + commit(transaction).await.expect("close read"); + rows +} + +async fn source_ids(store: &ControlStore, table: &str) -> Vec { + let mut ids: Vec = observations(store, table) + .await + .iter() + .map(|row| { + row["record_json"]["event_id"] + .as_str() + .unwrap_or_default() + .to_owned() + }) + .collect(); + ids.sort(); + ids +} diff --git a/crates/sqlbuild-orchestration/tests/api/work/helpers.rs b/crates/sqlbuild-orchestration/tests/api/work/helpers.rs new file mode 100644 index 0000000000..2debfcd9b6 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/work/helpers.rs @@ -0,0 +1,20 @@ +//! Fault injection changes only disposable test stores, never product time or lease semantics. + +use crate::store::models::ControlStore; + +pub(crate) async fn execute(store: &ControlStore, sql: &str) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!("SQLite store") + }; + rusqlite::Connection::open(path) + .expect("fault connection") + .execute_batch(sql) + .expect("inject SQLite fault"); + return; + }; + sqlx_core::raw_sql::raw_sql(sql) + .execute(pool) + .await + .expect("inject PostgreSQL fault"); +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs new file mode 100644 index 0000000000..000b7020b8 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs @@ -0,0 +1,131 @@ +use uuid::Uuid; + +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn location_state(store: &ControlStore) -> (i64, i64) { + let sql = "SELECT (SELECT COUNT(*) FROM sqb_locations), \ + (SELECT COUNT(*) FROM sqb_security_audit WHERE action='location.created')"; + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + return rusqlite::Connection::open(path) + .expect("inspect locations") + .query_row(sql, [], |row| Ok((row.get(0)?, row.get(1)?))) + .expect("location and audit counts"); + }; + sqlx_core::query_as::query_as(sql) + .fetch_one(pool) + .await + .expect("location and audit counts") +} + +pub(crate) async fn stored_hashes(store: &ControlStore) -> Vec { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + let connection = rusqlite::Connection::open(path).expect("inspect durable tokens"); + return connection + .prepare("SELECT secret_hash FROM sqb_access_tokens") + .expect("token hashes") + .query_map([], |row| row.get(0)) + .expect("read token hashes") + .collect::>() + .expect("hashes"); + }; + sqlx_core::query_scalar::query_scalar("SELECT secret_hash FROM sqb_access_tokens") + .fetch_all(pool) + .await + .expect("read durable token hashes") +} + +/// Committed security facts; every authority change writes exactly one in its own transaction. +pub(crate) async fn security_event_count( + store: &ControlStore, +) -> Result { + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await?, + "SELECT COUNT(*) AS revision FROM sqb_security_audit", + vec![], + ) + .await?; + commit(transaction).await?; + rows.first() + .and_then(|row| row.get("revision")) + .and_then(serde_json::Value::as_u64) + .ok_or_else(|| { + crate::store::errors::ControlError::Schema("security event count is invalid".into()) + }) +} + +/// Test-only clock skew: every audit row becomes older than the given number of days. +pub(crate) async fn age_audit(store: &ControlStore, days: i64) { + let transaction = begin(store, TransactionMode::Write) + .await + .expect("age audit"); + let then = transaction.now - chrono::Duration::days(days); + let (transaction, _) = query( + transaction, + "UPDATE sqb_security_audit SET occurred_at=$1", + vec![Parameter::Time(then)], + ) + .await + .expect("aged audit rows"); + commit(transaction).await.expect("commit aged audit"); +} + +/// Acknowledge every audit row up to and including `through` for one sink, in storage order. +pub(crate) async fn acknowledge_audit(store: &ControlStore, sink_id: &str, through: i64) { + let transaction = begin(store, TransactionMode::Write) + .await + .expect("acknowledge audit"); + let now = transaction.now; + let (transaction, _) = query( + transaction, + "INSERT INTO sqb_security_audit_deliveries (sink_id, storage_id, event_id, delivered_at) \ + SELECT $1, storage_id, event_id, $2 FROM sqb_security_audit WHERE storage_id<=$3", + vec![ + Parameter::Text(sink_id.into()), + Parameter::Time(now), + Parameter::Integer(through), + ], + ) + .await + .expect("delivery rows"); + commit(transaction).await.expect("commit deliveries"); +} + +/// Remaining audit event IDs with their storage IDs, and the delivery rows that remain. +pub(crate) async fn audit_rows(store: &ControlStore) -> (Vec<(Uuid, i64)>, i64) { + let transaction = begin(store, TransactionMode::Read) + .await + .expect("read audit"); + let (transaction, rows) = query( + transaction, + "SELECT event_id, storage_id FROM sqb_security_audit ORDER BY storage_id", + vec![], + ) + .await + .expect("audit rows"); + let (transaction, deliveries) = query( + transaction, + "SELECT COUNT(*) AS deliveries FROM sqb_security_audit_deliveries", + vec![], + ) + .await + .expect("delivery count"); + commit(transaction).await.expect("commit audit read"); + ( + rows.iter() + .map(|row| { + ( + serde_json::from_value(row["event_id"].clone()).expect("event ID"), + row["storage_id"].as_i64().expect("storage ID"), + ) + }) + .collect(), + deliveries[0]["deliveries"].as_i64().expect("deliveries"), + ) +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs new file mode 100644 index 0000000000..43e31882ef --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs @@ -0,0 +1,24 @@ +//! Shared location fixtures use the same audited identity operation as operator bootstrap. + +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; +use uuid::Uuid; + +pub(crate) async fn create_location( + store: &ControlStore, + name: &str, + actor: &str, +) -> Result { + crate::credentials::main::locations::create_location::create_location( + store, + name, + &AuditContext { + actor: actor.into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/passwords/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/passwords/test_contract.rs new file mode 100644 index 0000000000..99577a7089 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/passwords/test_contract.rs @@ -0,0 +1,284 @@ +//! Real-store password replacement, live eligibility and transactional security audit. + +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::credentials::main::{ + identities::create_principal::create_principal, sessions::set_password::set_password, + sessions::verify_password::verify_password, +}; +use crate::credentials::models::{AuditContext, PasswordAuthority}; +use crate::credentials::tests::helpers::security_event_count; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +use crate::credentials::tests::test_passwords::test_types::PasswordTestCase; + +#[path = "test_types.rs"] +mod test_types; + +#[test] +fn given_person_passwords_when_changed_then_replicas_verify_exact_bytes_and_audit_failure_rolls_back() + { + let test_cases = [ + PasswordTestCase { + description: "SQLite password lifecycle", + backend: Backend::Sqlite, + expected_audits: 5, + }, + PasswordTestCase { + description: "PostgreSQL password lifecycle", + backend: Backend::Postgres, + expected_audits: 5, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("fresh schema"); + let replica = second_store(store).await; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/password_credentials.json", + ); + let cases: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("DuckShop fixture")) + .expect("password cases"); + let password = cases["password"].as_str().expect("password"); + let replacement = cases["replacement"].as_str().expect("replacement"); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let person = create_principal(store, &context, "orders-reader") + .await + .expect("person"); + for id in [person, Uuid::now_v7()] { + assert!( + verify_password(&replica, id, password) + .await + .expect("missing credential") + .is_none() + ); + } + for weak in [ + "", + "12345678901", + cases["too_short"].as_str().expect("short Unicode"), + ] { + assert!( + set_password(store, &context, PasswordAuthority::Operator(person), weak) + .await + .is_err() + ); + } + assert!( + set_password( + store, + &context, + PasswordAuthority::Operator(person), + &"x".repeat(1025) + ) + .await + .is_err() + ); + set_password( + store, + &context, + PasswordAuthority::Operator(person), + password, + ) + .await + .expect("set password"); + let first = verify_password(&replica, person, password) + .await + .expect("verify on second store") + .expect("password proof"); + assert_eq!(first.principal_id, person, "{}", test_case.description); + let transaction = begin(&replica, TransactionMode::Read) + .await + .expect("inspect storage"); + let (transaction, rows) = query( + transaction, + "SELECT password_hash FROM sqb_password_credentials WHERE principal_id=$1", + vec![Parameter::Id(person)], + ) + .await + .expect("stored PHC"); + commit(transaction).await.expect("inspection done"); + let first_hash = rows[0]["password_hash"].as_str().expect("PHC").to_owned(); + assert!(first_hash.starts_with("$argon2id$v=19$m=19456,t=2,p=1$")); + assert!(!first_hash.contains(password)); + set_password( + store, + &context, + PasswordAuthority::Operator(person), + password, + ) + .await + .expect("same bytes, new salt"); + let second = verify_password(&replica, person, password) + .await + .expect("new version") + .expect("valid password"); + assert_ne!(first.credential_id, second.credential_id); + let (transaction, rows) = query( + begin(&replica, TransactionMode::Read).await.expect("read"), + "SELECT password_hash FROM sqb_password_credentials WHERE principal_id=$1", + vec![Parameter::Id(person)], + ) + .await + .expect("replacement salt"); + commit(transaction).await.expect("read done"); + assert_ne!( + rows[0]["password_hash"], + json!(first_hash), + "identical passwords receive independent salts" + ); + let fault = TransactionFault { + description: "password audit failure", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected audit failure')", + postgres_body: "RAISE EXCEPTION 'injected audit failure'", + }; + let revision = security_event_count(&replica) + .await + .expect("password revision"); + install_fault(store, &fault).await; + assert!( + set_password( + store, + &context, + PasswordAuthority::Operator(person), + replacement + ) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert_eq!( + security_event_count(&replica) + .await + .expect("audit rollback revision"), + revision + ); + assert_eq!( + verify_password(&replica, person, password) + .await + .expect("rolled back"), + Some(second) + ); + set_password( + store, + &context, + PasswordAuthority::Operator(person), + replacement, + ) + .await + .expect("replace password"); + assert_eq!( + security_event_count(&replica) + .await + .expect("replacement revision"), + revision + 1 + ); + for invalid in [ + password, + cases["wrong"].as_str().expect("wrong"), + &"x".repeat(1025), + ] { + assert!( + verify_password(&replica, person, invalid) + .await + .expect("rejected password") + .is_none() + ); + } + assert!( + verify_password(&replica, person, replacement) + .await + .expect("new password") + .is_some() + ); + for boundary in [ + cases["minimum_unicode"] + .as_str() + .expect("12 Unicode characters"), + &"x".repeat(1024), + ] { + set_password( + store, + &context, + PasswordAuthority::Operator(person), + boundary, + ) + .await + .expect("accepted boundary"); + assert!( + verify_password(&replica, person, boundary) + .await + .expect("boundary verified") + .is_some() + ); + } + migrate_store(&replica) + .await + .expect("restart preserves credentials"); + let (transaction, rows) = query(begin(&replica, TransactionMode::Read).await.expect("audit read"), + "SELECT action, before_json, after_json FROM sqb_security_audit WHERE action='password.changed'", vec![]) + .await.expect("durable audits"); + commit(transaction).await.expect("audit read done"); + assert_eq!( + rows.len(), + test_case.expected_audits, + "failed and rejected changes leave no audit or credential change" + ); + let audit = serde_json::to_string(&rows).expect("audit inspection"); + for secret in [password, replacement, &first_hash] { + assert!( + !audit.contains(secret), + "audit contains identities, never passwords or hashes" + ); + } + let (transaction, _) = query( + begin(store, TransactionMode::Write).await.expect("disable"), + "UPDATE sqb_principals SET disabled_at=$1 WHERE principal_id=$2", + vec![Parameter::Time(chrono::Utc::now()), Parameter::Id(person)], + ) + .await + .expect("disabled person"); + commit(transaction).await.expect("disable committed"); + assert!( + verify_password(&replica, person, &"x".repeat(1024)) + .await + .expect("live eligibility") + .is_none() + ); + assert!( + set_password( + store, + &context, + PasswordAuthority::Operator(person), + replacement + ) + .await + .is_err() + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/passwords/test_types.rs b/crates/sqlbuild-orchestration/tests/credentials/passwords/test_types.rs new file mode 100644 index 0000000000..0398f3ab72 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/passwords/test_types.rs @@ -0,0 +1,7 @@ +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct PasswordTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_audits: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/roles/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/roles/helpers.rs new file mode 100644 index 0000000000..fad141b126 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/roles/helpers.rs @@ -0,0 +1,152 @@ +//! Real people, sessions and published custom roles for custom role contracts. + +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::authorization::main::{ + assign_builtin_role::assign_builtin_role, authorize_person::authorize_person, + bootstrap_user::bootstrap_user, +}; +use crate::credentials::main::{ + identities::create_principal::create_principal, sessions::login_password::login_password, + sessions::set_password::set_password, +}; +use crate::credentials::models::{ + Action, AuditContext, DefaultRole, IssuedSession, PasswordAuthority, PasswordLogin, + PermissionScope, RoleAssignmentRequest, RoleKey, +}; +use crate::credentials::tests::location_helpers::create_location; +use crate::credentials::tests::test_types::RoleFixture; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::models::{ControlStore, TransactionMode}; + +pub(crate) const SUPPORT_ROLE: &str = "orders_support"; + +async fn signed_in( + store: &ControlStore, + context: &AuditContext, + (principal, name): (Uuid, &str), +) -> IssuedSession { + set_password( + store, + context, + PasswordAuthority::Operator(principal), + "orders-password-2026", + ) + .await + .expect("password"); + let PasswordLogin::Authenticated(session) = + login_password(store, context, (name, "orders-password-2026")) + .await + .expect("login") + else { + panic!("valid login"); + }; + session +} + +pub(crate) fn credentials(session: &IssuedSession) -> Uuid { + session.identity.principal_id +} + +async fn assign( + store: &ControlStore, + fixture: (&AuditContext, &IssuedSession), + request: RoleAssignmentRequest, +) { + assert!( + assign_builtin_role(store, (fixture.0, credentials(fixture.1)), &request) + .await + .expect("role assignment") + ); +} + +/// Publish one generation whose `roles` document grants the support role these actions. +pub(crate) async fn publish_support_role( + store: &ControlStore, + location_id: Uuid, + (generation, actions): (i64, Value), +) { + let mut input = snapshot(location_id); + input.generation = generation; + input.documents.insert( + "roles".into(), + json!({SUPPORT_ROLE: {"description": "Answer order questions", "actions": actions}}), + ); + publish_snapshot(store, &input) + .await + .expect("publish roles"); +} + +/// The first administrator, then a member holding the published support role. +pub(crate) async fn role_fixture(store: &ControlStore) -> RoleFixture { + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("local peer")), + }; + let first = bootstrap_user(store, &context, "orders-admin") + .await + .expect("bootstrap"); + let administrator = signed_in(store, &context, (first, "orders-admin")).await; + let member = create_principal(store, &context, "support-person") + .await + .expect("member"); + let member_session = signed_in(store, &context, (member, "support-person")).await; + let location_id = create_location(store, "orders", "orders-admin") + .await + .expect("location"); + publish_support_role(store, location_id, (1, json!(["view"]))).await; + assign( + store, + (&context, &administrator), + RoleAssignmentRequest { + principal_id: member, + role: RoleKey::Custom(SUPPORT_ROLE.into()), + location_id: Some(location_id), + target: None, + revoke: false, + }, + ) + .await; + RoleFixture { + context, + location_id, + administrator, + member: member_session, + member_id: member, + } +} + +/// The role that grants the member this action in the fixture location, or `None` when denied. +pub(crate) async fn granting_role( + store: &ControlStore, + fixture: &RoleFixture, + action: Action, +) -> Option { + let scope = PermissionScope { + action, + location_id: Some(fixture.location_id), + target: Some("development".into()), + nodes: Default::default(), + }; + let decision = authorize_person( + begin(store, TransactionMode::Write).await.expect("begin"), + (&fixture.context, credentials(&fixture.member)), + &scope, + DefaultRole::None, + ) + .await; + assert!( + decision + .as_ref() + .err() + .is_none_or(|error| matches!(error, ControlError::Denied { .. })), + "authorization either grants or denies" + ); + decision.ok().map(|(_, context)| context.granting_role) +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/sessions/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/sessions/test_contract.rs new file mode 100644 index 0000000000..7174d9e926 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/sessions/test_contract.rs @@ -0,0 +1,401 @@ +//! Concurrent login reservations, secret eligibility and atomic audit failures on both stores. + +use chrono::Duration; +use futures_util::future::join_all; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::credentials::main::{ + audit::reserve_authentication_attempt::reserve_authentication_attempt, + identities::create_principal::create_principal, + sessions::authenticate_session::authenticate_session, sessions::login_password::login_password, + sessions::revoke_session::revoke_session, sessions::set_password::set_password, + sessions::verify_password::verify_password, tokens::authenticate_token::authenticate_token, +}; +use crate::credentials::models::{ + AuditContext, AuthenticationAttempt, PasswordAuthority, PasswordLogin, SessionAccess, +}; +use crate::credentials::tests::test_sessions::test_types::SessionTestCase; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[path = "test_types.rs"] +mod test_types; + +#[test] +fn given_password_sessions_when_replicas_race_or_auditing_fails_then_throttles_and_credentials_remain_authoritative() + { + let test_cases = [ + SessionTestCase { + description: "SQLite session storage contract", + backend: Backend::Sqlite, + expected_sessions: 5, + expected_peer_attempts: 20, + }, + SessionTestCase { + description: "PostgreSQL session storage contract", + backend: Backend::Postgres, + expected_sessions: 5, + expected_peer_attempts: 20, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/password_credentials.json", + ); + let cases: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("DuckShop fixture")) + .expect("password inputs"); + let password = cases["password"].as_str().expect("password"); + let replacement = cases["replacement"].as_str().expect("replacement"); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("peer")), + }; + let person = create_principal(store, &context, "orders-reader") + .await + .expect("person"); + set_password( + store, + &context, + PasswordAuthority::Operator(person), + password, + ) + .await + .expect("password"); + let context = &context; + let stores = [store, &replica]; + let outcomes = join_all((0..12).map(|index| async move { + login_password(stores[index % 2], context, ("orders-reader", password)) + .await + .expect("concurrent login") + })) + .await; + assert_eq!( + outcomes + .iter() + .filter(|result| matches!(result, PasswordLogin::Authenticated(_))) + .count(), + test_case.expected_sessions, + "{}", + test_case.description + ); + assert_eq!( + outcomes + .iter() + .filter(|result| matches!( + result, + PasswordLogin::Denied { + throttled: true, + .. + } + )) + .count(), + 12 - test_case.expected_sessions + ); + let retained = outcomes + .into_iter() + .find(|result| matches!(result, PasswordLogin::Authenticated(_))) + .expect("successful session"); + let PasswordLogin::Authenticated(session) = retained else { + panic!("expected an authenticated session"); + }; + for secret in [&session.bearer, &session.csrf] { + assert!(!format!("{session:?}").contains(secret)); + } + assert!( + authenticate_token(&replica, &session.bearer) + .await + .expect("distinct bearer scope") + .is_none() + ); + assert_eq!( + authenticate_session(&replica, &session.bearer, SessionAccess::Read) + .await + .expect("live session"), + Some(session.identity.clone()) + ); + let another_peer = AuditContext { + ip_address: Some("127.0.0.2".parse().expect("other peer")), + ..context.clone() + }; + assert!(matches!( + login_password(&replica, &another_peer, ("orders-reader", password)) + .await + .expect("global identity throttle"), + PasswordLogin::Denied { + throttled: true, + .. + } + )); + let peer_results = join_all((0..30).map(|index| { + let name = format!("customers-{index}"); + let context = &another_peer; + async move { + reserve_authentication_attempt( + stores[index % 2], + context, + AuthenticationAttempt::Login(&name), + ) + .await + .expect("shared peer reservation") + .1 + } + })) + .await; + assert_eq!( + peer_results.iter().filter(|allowed| **allowed).count(), + test_case.expected_peer_attempts + ); + let (transaction, rows) = query( + begin(store, TransactionMode::Read) + .await + .expect("inspect sessions"), + "SELECT secret_hash, csrf_hash FROM sqb_sessions", + vec![], + ) + .await + .expect("persisted sessions"); + commit(transaction).await.expect("read complete"); + assert_eq!(rows.len(), test_case.expected_sessions); + let stored = serde_json::to_string(&rows).expect("stored secret hashes"); + assert!(!stored.contains(&session.bearer)); + assert!(!stored.contains(&session.csrf)); + let customer = create_principal(store, context, "customers-reader") + .await + .expect("second person"); + set_password( + store, + context, + PasswordAuthority::Operator(customer), + password, + ) + .await + .expect("second password"); + let creation_fault = TransactionFault { + description: "session audit failure", + table: "sqb_security_audit", + operation: "INSERT", + condition: "WHEN NEW.action='session.created'", + sqlite_body: "SELECT RAISE(ABORT,'injected session audit failure')", + postgres_body: "RAISE EXCEPTION 'injected session audit failure'", + }; + install_fault(store, &creation_fault).await; + assert!( + login_password(&replica, context, ("customers-reader", password)) + .await + .is_err() + ); + remove_fault(store, &creation_fault).await; + let (transaction, rows) = query( + begin(store, TransactionMode::Read) + .await + .expect("inspect rollback"), + "SELECT session_id FROM sqb_sessions WHERE principal_id=$1", + vec![Parameter::Id(customer)], + ) + .await + .expect("no unaudited session"); + commit(transaction).await.expect("read complete"); + assert!(rows.is_empty()); + let PasswordLogin::Authenticated(customer_session) = + login_password(store, context, ("customers-reader", password)) + .await + .expect("customer login") + else { + panic!("expected customer session"); + }; + let revocation_fault = TransactionFault { + description: "logout audit failure", + condition: "WHEN NEW.action='session.revoked'", + ..creation_fault + }; + install_fault(store, &revocation_fault).await; + assert!( + revoke_session( + begin(store, TransactionMode::Write) + .await + .expect("logout transaction"), + context, + (customer, Some(customer_session.identity.session_id)) + ) + .await + .is_err() + ); + remove_fault(store, &revocation_fault).await; + assert!( + authenticate_session(&replica, &customer_session.bearer, SessionAccess::Read) + .await + .expect("failed logout rolls back") + .is_some() + ); + let proof = verify_password(store, customer, password) + .await + .expect("verify version") + .expect("password proof"); + set_password( + &replica, + context, + PasswordAuthority::Operator(customer), + replacement, + ) + .await + .expect("operator reset"); + assert!( + authenticate_session(store, &customer_session.bearer, SessionAccess::Read) + .await + .expect("reset invalidates session") + .is_none() + ); + assert!( + set_password( + store, + context, + PasswordAuthority::Reauthenticated { + proof, + session_id: customer_session.identity.session_id + }, + password + ) + .await + .is_err() + ); + let proof = verify_password(store, customer, replacement) + .await + .expect("new version") + .expect("replacement proof"); + assert!( + set_password( + store, + context, + PasswordAuthority::Reauthenticated { + proof, + session_id: customer_session.identity.session_id + }, + password + ) + .await + .is_err() + ); + let PasswordLogin::Authenticated(customer_session) = + login_password(store, context, ("customers-reader", replacement)) + .await + .expect("fresh login after password reset") + else { + panic!("expected fresh session for logout"); + }; + let (transaction, revoked) = revoke_session( + begin(store, TransactionMode::Write) + .await + .expect("logout transaction"), + context, + (customer, Some(customer_session.identity.session_id)), + ) + .await + .expect("logout"); + assert_eq!(revoked, 1); + commit(transaction).await.expect("publish logout"); + let (transaction, revoked) = revoke_session( + begin(&replica, TransactionMode::Write) + .await + .expect("retry logout transaction"), + context, + (customer, Some(customer_session.identity.session_id)), + ) + .await + .expect("idempotent logout"); + assert_eq!(revoked, 0); + commit(transaction).await.expect("finish retry"); + let PasswordLogin::Authenticated(disabled_session) = + login_password(store, context, ("customers-reader", replacement)) + .await + .expect("login before disabling person") + else { + panic!("expected live session before disabling person"); + }; + let transaction = begin(&replica, TransactionMode::Write) + .await + .expect("disable person"); + let now = transaction.now; + let (transaction, _) = query( + transaction, + "UPDATE sqb_principals SET disabled_at=$1 WHERE principal_id=$2", + vec![Parameter::Time(now), Parameter::Id(customer)], + ) + .await + .expect("disabled person"); + commit(transaction).await.expect("disable saved"); + assert!( + authenticate_session(store, &disabled_session.bearer, SessionAccess::Read) + .await + .expect("disabled person's session rejected across replicas") + .is_none() + ); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("expire session"); + let expired = transaction.now - Duration::seconds(1); + let (transaction, _) = query( + transaction, + "UPDATE sqb_sessions SET expires_at=$1 WHERE session_id=$2", + vec![ + Parameter::Time(expired), + Parameter::Id(session.identity.session_id), + ], + ) + .await + .expect("expired session"); + commit(transaction).await.expect("expiry saved"); + assert!( + authenticate_session(&replica, &session.bearer, SessionAccess::Read) + .await + .expect("expired session rejected") + .is_none() + ); + let (transaction, rows) = query( + begin(store, TransactionMode::Read) + .await + .expect("session history"), + "SELECT revoked_at FROM sqb_sessions WHERE session_id=$1", + vec![Parameter::Id(customer_session.identity.session_id)], + ) + .await + .expect("revoked history retained"); + commit(transaction).await.expect("history complete"); + assert!(!rows[0]["revoked_at"].is_null()); + let (transaction, rows) = query( + begin(store, TransactionMode::Read) + .await + .expect("audit history"), + "SELECT action, after_json FROM sqb_security_audit WHERE event_id=$1", + vec![Parameter::Id(Uuid::new_v5( + &customer_session.identity.session_id, + b"revoked", + ))], + ) + .await + .expect("one revocation audit"); + commit(transaction).await.expect("audit complete"); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0]["action"], json!("session.revoked")); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/sessions/test_types.rs b/crates/sqlbuild-orchestration/tests/credentials/sessions/test_types.rs new file mode 100644 index 0000000000..7892e45278 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/sessions/test_types.rs @@ -0,0 +1,8 @@ +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct SessionTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_sessions: usize, + pub(crate) expected_peer_attempts: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_administrators.rs b/crates/sqlbuild-orchestration/tests/credentials/test_administrators.rs new file mode 100644 index 0000000000..0c76ec1d78 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_administrators.rs @@ -0,0 +1,343 @@ +//! Installation bootstrap and final-admin transitions serialize across real database replicas. + +use futures_util::future::join_all; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::authorization::main::{ + bootstrap_user::bootstrap_user, change_administrator::change_administrator, +}; +use crate::credentials::main::{ + identities::create_principal::create_principal, sessions::login_password::login_password, + sessions::set_password::set_password, +}; +use crate::credentials::models::{ + AdministratorChange, AuditContext, PasswordAuthority, PasswordLogin, +}; +use crate::credentials::tests::helpers::security_event_count; +use crate::credentials::tests::test_types::AdministratorTestCase; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_installation_administrators_when_replicas_race_then_bootstrap_is_once_only_and_final_admin_survives() + { + let test_cases = [ + AdministratorTestCase { + description: "SQLite once-only bootstrap and final admin", + backend: Backend::Sqlite, + expected_administrators: 1, + }, + AdministratorTestCase { + description: "PostgreSQL once-only bootstrap and final admin", + backend: Backend::Postgres, + expected_administrators: 1, + }, + ]; + for test_case in test_cases { + for change in [AdministratorChange::Revoke, AdministratorChange::Disable] { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("database"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("trusted peer")), + }; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json"); + let inputs: Value = serde_json::from_str( + &std::fs::read_to_string(path).expect("DuckShop lifecycle"), + ) + .expect("inputs"); + let names = [ + inputs["first_person"].as_str().expect("first person"), + inputs["second_person"].as_str().expect("second person"), + ]; + let stores = [store, &replica]; + let results = join_all( + (0..2).map(|index| bootstrap_user(stores[index], &context, names[index])), + ) + .await; + assert_eq!( + results.iter().filter(|result| result.is_ok()).count(), + test_case.expected_administrators, + "{}", + test_case.description + ); + let (winner, first) = results + .into_iter() + .enumerate() + .find_map(|(index, result)| result.ok().map(|id| (index, id))) + .expect("one bootstrap winner"); + let other = 1 - winner; + let second = create_principal(store, &context, names[other]) + .await + .expect("ordinary role-free person"); + let password = inputs["password"].as_str().expect("password input"); + let mut sessions = Vec::new(); + for (id, name) in [(first, names[winner]), (second, names[other])] { + set_password(store, &context, PasswordAuthority::Operator(id), password) + .await + .expect("owner provisioning"); + let PasswordLogin::Authenticated(session) = + login_password(store, &context, (name, password)) + .await + .expect("login") + else { + panic!("valid login"); + }; + sessions.push(session); + } + let first_credentials = sessions[0].identity.principal_id; + let second_credentials = sessions[1].identity.principal_id; + assert!( + change_administrator( + store, + (&context, second_credentials), + (second, AdministratorChange::Grant), + ) + .await + .is_err(), + "a valid person session is not administrator authority" + ); + assert!( + change_administrator(store, (&context, first_credentials), (first, change)) + .await + .is_err(), + "cannot remove sole administrator" + ); + assert!( + change_administrator( + store, + (&context, first_credentials), + (second, AdministratorChange::Grant), + ) + .await + .expect("authenticated administrator grant") + ); + let revision = security_event_count(store) + .await + .expect("revision before duplicate grant"); + assert!( + !change_administrator( + store, + (&context, first_credentials), + (second, AdministratorChange::Grant), + ) + .await + .expect("idempotent grant") + ); + assert_eq!( + security_event_count(store) + .await + .expect("revision after duplicate grant"), + revision + ); + let (left, right) = tokio::join!( + change_administrator(store, (&context, first_credentials), (first, change)), + change_administrator( + &replica, + (&context, second_credentials), + (second, change) + ) + ); + assert_eq!( + [&left, &right] + .iter() + .filter(|result| matches!(result, Ok(true))) + .count(), + 1, + "concurrent self-removal leaves one administrator" + ); + let (transaction, rows) = query(begin(store, TransactionMode::Read).await.expect("inspect remaining admins"), + "SELECT COUNT(*) AS count FROM sqb_role_assignments a JOIN sqb_principals p USING(principal_id) WHERE a.role_key='admin' AND p.disabled_at IS NULL",vec![]).await.expect("count remaining admins"); + assert_eq!(rows[0]["count"], json!(1)); + commit(transaction) + .await + .expect("finish remaining-admin read"); + let (transaction, audit) = query(begin(store,TransactionMode::Read).await.expect("read grant audit"), + "SELECT actor,granting_role FROM sqb_security_audit WHERE action='role.assigned' AND actor<> 'local-control-store-owner'",vec![]).await.expect("grant audit"); + assert_eq!(audit.len(), 1); + assert_eq!(audit[0]["actor"], json!(format!("principal:{first}"))); + assert_eq!(audit[0]["granting_role"], json!("admin")); + commit(transaction).await.expect("finish audit read"); + // Simulate an operator's out-of-band loss: it must never reopen bootstrap. + let (transaction, _) = query( + begin(store, TransactionMode::Write) + .await + .expect("simulate out-of-band loss"), + "UPDATE sqb_principals SET disabled_at=$1", + vec![Parameter::Time(chrono::Utc::now())], + ) + .await + .expect("disable all people outside supported API"); + commit(transaction).await.expect("commit simulated loss"); + assert!( + bootstrap_user(&replica, &context, "replacement-admin") + .await + .is_err() + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } + } +} + +#[test] +fn given_bootstrap_and_admin_changes_when_durable_publication_fails_then_all_authority_rolls_back() +{ + let test_cases = [ + AdministratorTestCase { + description: "SQLite publication failures roll back authority", + backend: Backend::Sqlite, + expected_administrators: 0, + }, + AdministratorTestCase { + description: "PostgreSQL publication failures roll back authority", + backend: Backend::Postgres, + expected_administrators: 0, + }, + ]; + for test_case in test_cases { + { + let (table, operation) = ("sqb_security_audit", "INSERT"); + run(async { + let fixture = open_test_store(test_case.backend).await.expect("database"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("trusted peer")), + }; + let fault = TransactionFault { + description: "fail administrator publication", + table, + operation, + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'publication unavailable')", + postgres_body: "RAISE EXCEPTION 'publication unavailable'", + }; + install_fault(store, &fault).await; + assert!( + bootstrap_user(store, &context, "orders-admin") + .await + .is_err() + ); + remove_fault(store, &fault).await; + let (transaction,rows) = query(begin(store,TransactionMode::Read).await.expect("inspect rollback"),"SELECT (SELECT COUNT(*) FROM sqb_principals) AS people, (SELECT COUNT(*) FROM sqb_role_assignments) AS assignments",vec![]).await.expect("count rolled back bootstrap"); + assert_eq!( + serde_json::to_value(&rows[0]).expect("serializable row"), + json!({"people":0,"assignments":0}) + ); + assert_eq!( + rows[0]["assignments"], + json!(test_case.expected_administrators), + "{}", + test_case.description + ); + commit(transaction).await.expect("finish rollback read"); + assert_eq!( + security_event_count(store) + .await + .expect("rolled back revision"), + 0 + ); + let first = bootstrap_user(store, &context, "orders-admin") + .await + .expect("retry after rollback"); + let second = create_principal(store, &context, "customers-admin") + .await + .expect("second person"); + set_password( + store, + &context, + PasswordAuthority::Operator(first), + "orders-password-2026", + ) + .await + .expect("owner password provisioning"); + let PasswordLogin::Authenticated(session) = + login_password(store, &context, ("orders-admin", "orders-password-2026")) + .await + .expect("password login") + else { + panic!("login"); + }; + let credentials = session.identity.principal_id; + let revision = security_event_count(store) + .await + .expect("revision before failed grant"); + install_fault(store, &fault).await; + assert!( + change_administrator( + store, + (&context, credentials), + (second, AdministratorChange::Grant), + ) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert_eq!( + security_event_count(store) + .await + .expect("revision after failed grant"), + revision + ); + assert!( + change_administrator( + store, + (&context, credentials), + (second, AdministratorChange::Grant), + ) + .await + .expect("retry grant"), + "failed grant did not persist" + ); + set_password( + store, + &context, + PasswordAuthority::Operator(second), + "customers-password-2026", + ) + .await + .expect("provision alternative administrator"); + install_fault(store, &fault).await; + assert!( + change_administrator( + store, + (&context, credentials), + (first, AdministratorChange::Disable), + ) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert!( + change_administrator( + store, + (&context, credentials), + (first, AdministratorChange::Disable), + ) + .await + .expect("retry disabling person"), + "failed disable preserved live session and enabled person" + ); + close_test_store(fixture).await; + }); + } + } +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs new file mode 100644 index 0000000000..247721fd30 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs @@ -0,0 +1,522 @@ +//! Real durable audit pages retain clock-independent order, live permissions and atomic writes. + +use axum::{ + Json, Router, + extract::State, + http::{HeaderMap, StatusCode}, + routing::post, +}; +use chrono::{DateTime, Utc}; +use serde_json::{Value, json}; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicU16, Ordering}, +}; +use uuid::Uuid; + +use crate::authorization::main::{ + assign_builtin_role::assign_builtin_role, bootstrap_user::bootstrap_user, + read_security_audit::read_security_audit, +}; +use crate::credentials::main::{ + audit::append_audit::append_audit, identities::create_principal::create_principal, + sessions::login_password::login_password, sessions::set_password::set_password, +}; +use crate::credentials::models::{ + AuditContext, BuiltinRole, PasswordAuthority, PasswordLogin, RoleAssignmentRequest, +}; +use crate::credentials::tests::test_types::{AuditTestCase, AuditWebhookReceiver}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, migrate_store::migrate_store}; +use crate::store::models::TransactionMode; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::webhooks::_helpers::delivery::deliver_webhook; +use crate::webhooks::main::connect_webhook::connect_webhook; +use crate::webhooks::models::WebhookSource; + +#[test] +fn given_security_audit_pages_when_clock_and_authority_change_then_commit_order_and_atomicity_hold() +{ + let test_cases = [ + AuditTestCase { + description: "SQLite durable audit pages", + backend: Backend::Sqlite, + expected_page_length: 2, + }, + AuditTestCase { + description: "PostgreSQL durable audit pages", + backend: Backend::Postgres, + expected_page_length: 2, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("database"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("local peer")), + }; + let admin = bootstrap_user(store, &context, "orders-admin") + .await + .expect("bootstrap"); + let person = create_principal(store, &context, "orders-person") + .await + .expect("person"); + let mut sessions = Vec::new(); + for (principal, name) in [(admin, "orders-admin"), (person, "orders-person")] { + set_password( + store, + &context, + PasswordAuthority::Operator(principal), + "orders-password-2026", + ) + .await + .expect("password"); + let PasswordLogin::Authenticated(session) = + login_password(store, &context, (name, "orders-password-2026")) + .await + .expect("login") + else { + panic!("valid login"); + }; + sessions.push(session); + } + let admin_credentials = sessions[0].identity.principal_id; + let ordinary_credentials = sessions[1].identity.principal_id; + for role in [ + BuiltinRole::Viewer, + BuiltinRole::Operator, + BuiltinRole::Developer, + ] { + assert!( + assign_builtin_role( + store, + (&context, sessions[0].identity.principal_id), + &RoleAssignmentRequest { + principal_id: person, + role: role.into(), + location_id: None, + target: None, + revoke: false, + } + ) + .await + .expect("role assignment") + ); + assert!(matches!( + read_security_audit(&replica, (&context, ordinary_credentials), (0, 100)).await, + Err(ControlError::Denied { .. }) + )); + } + let existing = read_security_audit(store, (&context, admin_credentials), (0, 100)) + .await + .expect("existing security facts"); + let upgraded = read_security_audit(&replica, (&context, admin_credentials), (0, 100)) + .await + .expect("replica audit"); + assert_eq!( + upgraded + .iter() + .map(|event| event.event_id) + .collect::>(), + existing + .iter() + .map(|event| event.event_id) + .collect::>(), + "every replica pages security facts in the same commit order" + ); + assert_eq!( + upgraded + .iter() + .map(|event| event.storage_id) + .collect::>(), + (1..=upgraded.len() as i64).collect::>() + ); + let cursor = upgraded.last().expect("populated history").storage_id; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/security_audit_order.json", + ); + let inputs: Value = serde_json::from_str( + &std::fs::read_to_string(path).expect("DuckShop clock fixture"), + ) + .expect("clock inputs"); + let mut ids = Vec::new(); + for (index, timestamp) in inputs["occurred_at"] + .as_array() + .expect("clock steps") + .iter() + .enumerate() + { + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("append clock fact"); + transaction.now = timestamp + .as_str() + .expect("timestamp") + .parse::>() + .expect("clock value"); + let id = Uuid::now_v7(); + ids.push(id); + let transaction = append_audit( + transaction, + &context, + (id, "audit.cursor_fixture"), + ( + json!({"observation":index}), + json!(null), + json!({"recorded":true}), + ), + ) + .await + .expect("durable clock fact"); + commit(transaction).await.expect("commit clock fact"); + } + let first = read_security_audit(&replica, (&context, admin_credentials), (cursor, 2)) + .await + .expect("first bounded page"); + assert_eq!( + first.len(), + test_case.expected_page_length, + "{}", + test_case.description + ); + assert_eq!( + first.iter().map(|event| event.event_id).collect::>(), + ids[..2] + ); + assert!(first[0].occurred_at > first[1].occurred_at); + assert_eq!(first[0].storage_id, cursor + 1); + assert_eq!(first[1].storage_id, cursor + 2); + assert_eq!(first[1].actor, context.actor); + assert_eq!(first[1].granting_role, context.granting_role); + assert_eq!(first[1].request_id, context.request_id); + assert_eq!(first[1].ip_address.as_deref(), Some("127.0.0.1")); + assert_eq!(first[1].scope_json, json!({"observation":1})); + assert_eq!(first[1].before_json, json!(null)); + assert_eq!(first[1].after_json, json!({"recorded":true})); + let last = read_security_audit( + store, + (&context, admin_credentials), + (first[1].storage_id, 2), + ) + .await + .expect("strict continuation"); + assert_eq!( + last.iter().map(|event| event.event_id).collect::>(), + ids[2..] + ); + let cursor = last[0].storage_id; + assert!( + read_security_audit(store, (&context, admin_credentials), (cursor, 100)) + .await + .expect("empty continuation") + .is_empty() + ); + for bounds in [(-1, 1), (0, 0), (0, 101)] { + assert!(matches!( + read_security_audit(store, (&context, admin_credentials), bounds).await, + Err(ControlError::Invalid(_)) + )); + } + let fault = TransactionFault { + description: "audit append fails atomically", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'audit unavailable')", + postgres_body: "RAISE EXCEPTION 'audit unavailable'", + }; + install_fault(store, &fault).await; + assert!( + create_principal(store, &context, "orders-rollback") + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert!( + read_security_audit(&replica, (&context, admin_credentials), (cursor, 100)) + .await + .expect("failed audit did not publish") + .is_empty() + ); + create_principal(store, &context, "orders-rollback") + .await + .expect("failed principal did not persist"); + let after = read_security_audit(store, (&context, admin_credentials), (cursor, 100)) + .await + .expect("successful retry"); + assert_eq!(after.len(), 1); + assert_eq!(after[0].storage_id, cursor + 1); + let cursor = after[0].storage_id; + let (left, right) = tokio::join!( + create_principal(store, &context, "orders-concurrent"), + create_principal(&replica, &context, "customers-concurrent"), + ); + assert_ne!( + left.expect("first concurrent append"), + right.expect("second concurrent append") + ); + let concurrent = + read_security_audit(store, (&context, admin_credentials), (cursor, 100)) + .await + .expect("serialized writers"); + assert_eq!( + concurrent + .iter() + .map(|event| event.storage_id) + .collect::>(), + vec![cursor + 1, cursor + 2] + ); + assert!( + concurrent + .iter() + .all(|event| event.action == "principal.created") + ); + assert!( + assign_builtin_role( + store, + (&context, sessions[0].identity.principal_id), + &RoleAssignmentRequest { + principal_id: person, + role: BuiltinRole::Admin.into(), + location_id: None, + target: None, + revoke: false, + } + ) + .await + .expect("second administrator") + ); + assert!( + assign_builtin_role( + store, + (&context, sessions[1].identity.principal_id), + &RoleAssignmentRequest { + principal_id: admin, + role: BuiltinRole::Admin.into(), + location_id: None, + target: None, + revoke: true, + } + ) + .await + .expect("live role revocation") + ); + assert!(matches!( + read_security_audit(&replica, (&context, admin_credentials), (0, 100)).await, + Err(ControlError::Denied { .. }) + )); + assert!( + !read_security_audit(&replica, (&context, ordinary_credentials), (0, 100)) + .await + .expect("replacement administrator reads audit") + .is_empty() + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_committed_security_facts_when_export_retries_then_only_successful_receipts_advance_commit_order() + { + let test_cases = [ + AuditTestCase { + description: "SQLite durable audit export", + backend: Backend::Sqlite, + expected_page_length: 100, + }, + AuditTestCase { + description: "PostgreSQL durable audit export", + backend: Backend::Postgres, + expected_page_length: 100, + }, + ]; + let scenario: Value = serde_json::from_str( + &std::fs::read_to_string(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/security_audit_delivery.json", + )) + .expect("delivery fixture"), + ) + .expect("delivery scenario"); + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("database"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "principal:orders-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("peer")), + }; + let mut expected = Vec::new(); + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("audit action"); + for index in 0..scenario["events"].as_i64().expect("count") { + // Deliberately decreasing event clocks must not change delivery order. + transaction.now = "2026-01-01T00:00:00Z" + .parse::>() + .expect("clock") + - chrono::TimeDelta::seconds(index); + let id = Uuid::now_v7(); + expected.push(json!({"storage_id":index+1,"event_id":id,"occurred_at":transaction.now, + "actor":context.actor,"granting_role":context.granting_role,"request_id":context.request_id, + "ip_address":"127.0.0.1","action":scenario["action"],"scope_json":scenario["scope"], + "before_json":scenario["before"],"after_json":scenario["after"]})); + transaction = append_audit( + transaction, + &context, + (id, scenario["action"].as_str().expect("action")), + ( + scenario["scope"].clone(), + scenario["before"].clone(), + scenario["after"].clone(), + ), + ) + .await + .expect("append"); + } + commit(transaction).await.expect("commit source actions"); + let requests = Arc::new(Mutex::new(Vec::::new())); + let status = Arc::new(AtomicU16::new(503)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("listener"); + let endpoint = format!( + "http://audit:sample-password@{}/audit?token=sample-token", + listener.local_addr().expect("address") + ); + let app = Router::new() + .route( + "/audit", + post( + |State(AuditWebhookReceiver { requests, status }): State< + AuditWebhookReceiver, + >, + headers: HeaderMap, + Json(event): Json| async move { + assert_eq!( + headers + .get("Idempotency-Key") + .expect("key") + .to_str() + .expect("text"), + event["event_id"].as_str().expect("event id") + ); + requests.lock().expect("requests").push(event); + ( + StatusCode::from_u16(status.load(Ordering::SeqCst)) + .expect("status"), + [(axum::http::header::LOCATION, "/audit")], + ) + }, + ), + ) + .with_state(AuditWebhookReceiver { + requests: requests.clone(), + status: status.clone(), + }); + let (stop, shutdown) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + axum::serve(listener, app) + .with_graceful_shutdown(async { + drop(shutdown.await); + }) + .await + }); + let sink = + connect_webhook(&endpoint, WebhookSource::SecurityAudit).expect("audit sink"); + assert!(!format!("{sink:?}").contains("sample-password")); + assert!(deliver_webhook(store, &sink).await.is_err()); + assert_eq!( + requests.lock().expect("requests").len(), + 1, + "failure stops this page" + ); + status.store(307, Ordering::SeqCst); + assert!( + deliver_webhook(store, &sink).await.is_err(), + "never redirect audit facts" + ); + assert_eq!(requests.lock().expect("requests").len(), 2); + status.store(204, Ordering::SeqCst); + let fault = TransactionFault { + description: "lost security export acknowledgement", + table: "sqb_security_audit_deliveries", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'test acknowledgement failure')", + postgres_body: "RAISE EXCEPTION 'test acknowledgement failure'", + }; + install_fault(store, &fault).await; + assert!(deliver_webhook(store, &sink).await.is_err()); + remove_fault(store, &fault).await; + let replica = second_store(store).await; + let restarted = + connect_webhook(&endpoint, WebhookSource::SecurityAudit).expect("restart"); + assert_eq!( + deliver_webhook(&replica, &restarted) + .await + .expect("first bounded page"), + test_case.expected_page_length, + "{}", + test_case.description + ); + assert_eq!( + deliver_webhook(&replica, &restarted) + .await + .expect("remaining page"), + 5 + ); + assert_eq!( + deliver_webhook(&replica, &restarted) + .await + .expect("durable cursor"), + 0 + ); + // Identical endpoint, independent platform stream: security cursor grants no authority. + let platform = + connect_webhook(&endpoint, WebhookSource::Platform).expect("platform sink"); + assert_eq!( + deliver_webhook(&replica, &platform) + .await + .expect("independent cursor"), + 0 + ); + { + let received = requests.lock().expect("requests"); + assert_eq!(received.len(), expected.len() + 3); + assert_eq!(received[0], expected[0]); + assert_eq!(received[1], expected[0]); + assert_eq!( + received[2], expected[0], + "lost acknowledgement retains full identity" + ); + assert_eq!( + received[3..], + expected, + "every field survives; storage order beats clocks" + ); + } + stop.send(()).expect("stop receiver"); + server + .await + .expect("receiver task") + .expect("receiver stopped"); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit_retention.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit_retention.rs new file mode 100644 index 0000000000..186b36662b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit_retention.rs @@ -0,0 +1,167 @@ +//! Audit expiry deletes only rows every sink received and no retained run's retry still reads. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::main::audit::prune_audit::prune_audit; +use crate::credentials::models::{AuditContext, AuditRetention}; +use crate::credentials::tests::helpers::{acknowledge_audit, age_audit, audit_rows}; +use crate::credentials::tests::location_helpers::create_location; +use crate::credentials::tests::test_types::AuditRetentionTestCase; +use crate::runs::main::request_job_run::request_job_run; +use crate::runs::models::JobRunRequest; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::{begin::begin, commit::commit, migrate_store::migrate_store}; +use crate::store::models::TransactionMode; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_expired_audit_rows_when_pruned_then_undelivered_rows_survive_and_retries_still_match() { + let test_cases = [ + AuditRetentionTestCase { + description: "SQLite keeps undelivered and newest rows", + backend: Backend::Sqlite, + expected_retained: 2, + }, + AuditRetentionTestCase { + description: "PostgreSQL keeps undelivered and newest rows", + backend: Backend::Postgres, + expected_retained: 2, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "orders-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let location_id = create_location(store, "orders", "orders-admin") + .await + .expect("location"); + let mut input = snapshot(location_id); + input.documents.insert( + "jobs".into(), + json!({"daily_orders": { + "description": "Build orders", "selection": ["orders"], "settings": {}, + "concurrency_groups": [] + }}), + ); + publish_snapshot(store, &input).await.expect("publish job"); + let transaction = begin(store, TransactionMode::Write).await.expect("begin"); + let transaction = append_audit( + transaction, + &context, + (Uuid::now_v7(), "audit.retention_fixture"), + ( + json!({"location_id": location_id}), + json!(null), + json!({"recorded": true}), + ), + ) + .await + .expect("append"); + commit(transaction).await.expect("commit"); + let request = JobRunRequest { + run_id: Uuid::now_v7(), + location_id, + target: "development".into(), + job_key: "daily_orders".into(), + reason: "Refresh orders".into(), + priority: 1, + window: None, + }; + assert!( + request_job_run(store, &context, &request) + .await + .expect("request") + ); + let (rows, _) = audit_rows(store).await; + let through = rows.last().expect("job request row").1; + let undelivered = Uuid::now_v7(); + let newest = Uuid::now_v7(); + for event_id in [undelivered, newest] { + let transaction = begin(store, TransactionMode::Write).await.expect("begin"); + let transaction = append_audit( + transaction, + &context, + (event_id, "audit.retention_fixture"), + (json!({}), json!(null), json!({"recorded": true})), + ) + .await + .expect("append"); + commit(transaction).await.expect("commit"); + } + age_audit(store, 401).await; + acknowledge_audit(store, "orders-audit-sink", through).await; + let sinks = ["orders-audit-sink".to_string()]; + assert!(AuditRetention::days(0).is_err()); + assert_eq!( + prune_audit(store, AuditRetention::days(500).expect("horizon"), &sinks) + .await + .expect("longer horizon") + .0, + 0 + ); + let two_sinks = [sinks[0].clone(), "inventory-audit-sink".to_string()]; + assert_eq!( + prune_audit(store, AuditRetention::default(), &two_sinks) + .await + .expect("second sink has received nothing") + .0, + 0 + ); + let total = rows.len() + 2; + let (deleted, retained) = prune_audit(store, AuditRetention::default(), &sinks) + .await + .expect("prune"); + assert_eq!( + retained, test_case.expected_retained, + "{}", + test_case.description + ); + assert_eq!(deleted, total - 2); + let (remaining, deliveries) = audit_rows(store).await; + let ids: Vec = remaining.iter().map(|(id, _)| *id).collect(); + assert_eq!(ids, [undelivered, newest]); + assert_eq!(deliveries, 0, "deliveries of pruned rows go with them"); + assert!( + !request_job_run(store, &context, &request) + .await + .expect("retry still finds its original request after the audit is pruned") + ); + let transaction = begin(store, TransactionMode::Write).await.expect("begin"); + let transaction = append_audit( + transaction, + &context, + (Uuid::now_v7(), "audit.retention_fixture"), + (json!({}), json!(null), json!({"recorded": true})), + ) + .await + .expect("append after prune"); + commit(transaction).await.expect("commit"); + let (after, _) = audit_rows(store).await; + assert_eq!( + after.last().expect("new row").1, + remaining.last().expect("newest").1 + 1, + "storage IDs are never reused" + ); + assert_eq!( + prune_audit(store, AuditRetention::default(), &sinks) + .await + .expect("idempotent"), + (0, test_case.expected_retained) + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs new file mode 100644 index 0000000000..3e346698de --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs @@ -0,0 +1,378 @@ +use std::collections::BTreeMap; + +use crate::credentials::main::identities::create_principal::create_principal; +use crate::credentials::main::locations::create_location::create_location; +use crate::credentials::main::tokens::authenticate_token::authenticate_token; +use crate::credentials::main::tokens::issue_token::issue_token; +use crate::credentials::main::tokens::revoke_token::revoke_token; +use crate::credentials::main::tokens::rotate_token::rotate_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use chrono::{Duration, Utc}; +use uuid::Uuid; + +use crate::credentials::tests::helpers::{location_state, security_event_count, stored_hashes}; +use crate::credentials::tests::test_types::{CredentialTestCase, TransactionFault}; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::terminal_helpers::finish; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::TransactionMode; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_them_and_audit_failure_rolls_back() + { + let test_cases = [ + CredentialTestCase { + description: "SQLite credential lifecycle and atomic audit", + backend: Backend::Sqlite, + expected_hash_length: 64, + }, + CredentialTestCase { + description: "PostgreSQL credential lifecycle and atomic audit", + backend: Backend::Postgres, + expected_hash_length: 64, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + assert_eq!( + security_event_count(&replica) + .await + .expect("initial revision"), + 0 + ); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("loopback")), + }; + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("predecessor"); + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/token_rotation.sql"); + let sql = std::fs::read_to_string(path).expect("DuckShop historical token"); + for statement in sql.split(';').map(str::trim).filter(|sql| !sql.is_empty()) { + (transaction, _) = query(transaction, statement, vec![]) + .await + .expect("synthetic historical token"); + } + commit(transaction).await.expect("predecessor committed"); + assert!( + rotate_token( + store, + &context, + Uuid::parse_str("00000000-0000-4000-8000-000000000032") + .expect("synthetic token identity") + ) + .await + .is_err(), + "archived-location credentials cannot be rotated" + ); + assert_eq!(stored_hashes(&replica).await.len(), 1); + let fault = TransactionFault { + description: "security audit failure", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected audit failure')", + postgres_body: "RAISE EXCEPTION 'injected audit failure'", + }; + let insertion_fault = TransactionFault { + description: "replacement issuance failure", + table: "sqb_access_tokens", + ..fault + }; + let rotation_audit_fault = TransactionFault { + description: "rotation linkage audit failure", + condition: "WHEN NEW.action='token.rotated'", + ..fault + }; + install_fault(store, &fault).await; + assert!( + create_principal(store, &context, "orders-ci") + .await + .is_err() + ); + assert!(create_location(store, "orders", &context).await.is_err()); + assert_eq!(location_state(&replica).await, (1, 0)); + remove_fault(store, &fault).await; + create_principal(store, &context, "orders-ci") + .await + .expect("principal creation and audit rolled back together"); + let location_id = create_location(store, "orders", &context) + .await + .expect("location"); + assert_eq!(location_state(&replica).await, (2, 1)); + let scopes = [CredentialScope::Location { location_id }]; + let expiry = Utc::now() + Duration::hours(1); + for scope in scopes { + let issued = issue_token(store, &context, scope, expiry) + .await + .expect("issue scoped token"); + assert_eq!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("replica verifies"), + Some(issued.identity.clone()) + ); + assert!( + !format!("{issued:?}").contains(&issued.bearer), + "debug output redacts secrets" + ); + let replacement = ["0", "1"][usize::from(issued.bearer.ends_with('0'))]; + let tampered = + format!("{}{replacement}", &issued.bearer[..issued.bearer.len() - 1]); + for invalid in [ + &tampered, + "", + "not-a-token", + &format!("{}.{}", Uuid::now_v7(), "0".repeat(64)), + ] { + assert!( + authenticate_token(&replica, invalid) + .await + .expect("invalid credential") + .is_none() + ); + } + let revision = security_event_count(&replica) + .await + .expect("before rotation"); + let count = stored_hashes(store).await.len(); + for failure in [&fault, &insertion_fault, &rotation_audit_fault] { + install_fault(store, failure).await; + assert!( + rotate_token(store, &context, issued.identity.token_id) + .await + .is_err() + ); + remove_fault(store, failure).await; + assert_eq!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("old credential survives rollback"), + Some(issued.identity.clone()) + ); + assert_eq!(stored_hashes(&replica).await.len(), count); + assert_eq!( + security_event_count(&replica) + .await + .expect("rollback revision"), + revision + ); + } + let (first, second) = tokio::join!( + rotate_token(store, &context, issued.identity.token_id), + rotate_token(&replica, &context, issued.identity.token_id), + ); + assert_ne!( + first.is_ok(), + second.is_ok(), + "concurrent rotations have exactly one winner" + ); + let rotated = first.or(second).expect("rotation winner"); + assert_eq!(rotated.identity.scope, issued.identity.scope); + assert_eq!(rotated.identity.expires_at, issued.identity.expires_at); + assert_ne!(rotated.identity.token_id, issued.identity.token_id); + assert_ne!(rotated.bearer, issued.bearer); + assert!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("old token revoked atomically") + .is_none() + ); + assert_eq!( + authenticate_token(store, &rotated.bearer) + .await + .expect("replacement authenticates"), + Some(rotated.identity.clone()) + ); + assert_eq!(stored_hashes(&replica).await.len(), count + 1); + let (transaction, audits) = query(begin(store, TransactionMode::Read).await.expect("read audit"), + "SELECT before_json, after_json FROM sqb_security_audit WHERE action='token.rotated'", vec![]).await.expect("rotation audit"); + commit(transaction).await.expect("audit read"); + let linked = audits + .iter() + .filter(|audit| { + audit["before_json"]["token_id"] + == serde_json::json!(issued.identity.token_id) + }) + .collect::>(); + assert_eq!(linked.len(), 1, "one committed rotation"); + let audit = linked[0]; + assert_eq!(audit["before_json"], serde_json::json!(issued.identity)); + assert_eq!(audit["after_json"], serde_json::json!(rotated.identity)); + let issued = rotated; + let revision = security_event_count(&replica) + .await + .expect("before revocation"); + { + let failure = &fault; + install_fault(store, failure).await; + assert!( + revoke_token(store, &context, issued.identity.token_id) + .await + .is_err() + ); + remove_fault(store, failure).await; + assert!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("failed revocation rolled back") + .is_some() + ); + assert_eq!( + security_event_count(&replica) + .await + .expect("rollback revision"), + revision + ); + } + assert!( + revoke_token(store, &context, issued.identity.token_id) + .await + .expect("revoke") + ); + assert_eq!( + security_event_count(&replica) + .await + .expect("committed revision"), + revision + 1 + ); + assert!( + !revoke_token(&replica, &context, issued.identity.token_id) + .await + .expect("retry revoke") + ); + assert_eq!( + security_event_count(store) + .await + .expect("idempotent revision"), + revision + 1 + ); + assert!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("revoked token") + .is_none() + ); + assert!( + issue_token(store, &context, scope, Utc::now() - Duration::seconds(1)) + .await + .is_err() + ); + let stored = stored_hashes(store).await; + assert!( + stored + .iter() + .all(|hash| hash.len() == test_case.expected_hash_length + && hash != &issued.bearer), + "{}", + test_case.description + ); + } + for scope in [ + CredentialScope::Location { + location_id: Uuid::now_v7(), + }, + CredentialScope::Run { + run_id: Uuid::now_v7(), + claim_id: Uuid::now_v7(), + }, + ] { + assert!( + issue_token(store, &context, scope, expiry).await.is_err(), + "inactive scopes cannot acquire credentials" + ); + } + let published = publish_snapshot(store, &snapshot(location_id)) + .await + .expect("snapshot"); + queue( + store, + &request((location_id, published.snapshot_id), 1, 0, &["orders"]), + ) + .await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim") + .expect("work"); + let scope = CredentialScope::Run { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }; + let issued = issue_token(store, &context, scope, expiry) + .await + .expect("mint claimed-run token"); + let rotated = rotate_token(store, &context, issued.identity.token_id) + .await + .expect("rotate claimed-run credential"); + assert_eq!(rotated.identity.scope, issued.identity.scope); + assert_eq!(rotated.identity.expires_at, issued.identity.expires_at); + assert!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("old run credential revoked") + .is_none() + ); + let issued = rotated; + assert_eq!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("worker token"), + Some(issued.identity.clone()) + ); + assert!( + issue_token( + store, + &context, + CredentialScope::Run { + run_id: claim.request.run_id, + claim_id: Uuid::now_v7() + }, + expiry + ) + .await + .is_err() + ); + finish(store, &lease, &claim).await; + assert!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("terminal token expires immediately") + .is_none() + ); + assert!( + issue_token(store, &context, scope, expiry).await.is_err(), + "a terminal claim cannot mint a new token" + ); + assert!( + rotate_token(store, &context, issued.identity.token_id) + .await + .is_err(), + "a terminal claim cannot regain authority through rotation" + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs new file mode 100644 index 0000000000..d8c431a139 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs @@ -0,0 +1,26 @@ +//! Credential contracts compiled within the credential owner. + +#[path = "helpers.rs"] +pub(crate) mod helpers; +#[path = "locations/helpers.rs"] +pub(crate) mod location_helpers; +#[path = "roles/helpers.rs"] +pub(crate) mod role_helpers; +#[path = "test_administrators.rs"] +mod test_administrators; +#[path = "test_audit.rs"] +mod test_audit; +#[path = "test_audit_retention.rs"] +mod test_audit_retention; +#[path = "test_contract.rs"] +mod test_contract; +#[path = "test_custom_roles.rs"] +mod test_custom_roles; +#[path = "passwords/test_contract.rs"] +mod test_passwords; +#[path = "test_recovery.rs"] +mod test_recovery; +#[path = "sessions/test_contract.rs"] +mod test_sessions; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_custom_roles.rs b/crates/sqlbuild-orchestration/tests/credentials/test_custom_roles.rs new file mode 100644 index 0000000000..59c3915a79 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_custom_roles.rs @@ -0,0 +1,73 @@ +//! Published custom roles take effect at once: widening, narrowing and removal alike. + +use serde_json::{Value, json}; + +use crate::credentials::models::Action; +use crate::credentials::tests::role_helpers::{granting_role, publish_support_role, role_fixture}; +use crate::credentials::tests::test_types::CustomRoleTestCase; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_published_custom_roles_when_republished_then_each_definition_applies_at_once() { + let test_cases = [ + CustomRoleTestCase { + description: "SQLite applies each publication", + backend: Backend::Sqlite, + expected_after_removal: json!({"view": null, "job.run": null}), + }, + CustomRoleTestCase { + description: "PostgreSQL applies each publication", + backend: Backend::Postgres, + expected_after_removal: json!({"view": null, "job.run": null}), + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/custom_role_publication.json", + ); + let case: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("role fixture")) + .expect("role case"); + let roles = role_fixture(store).await; + let grants = || async { + json!({ + "view": granting_role(store, &roles, Action::View).await, + "job.run": granting_role(store, &roles, Action::JobRun).await, + }) + }; + let publications = case["publications"].as_array().expect("publications"); + for (generation, publication) in (2..).zip(publications) { + publish_support_role( + store, + roles.location_id, + (generation, publication["actions"].clone()), + ) + .await; + assert_eq!( + grants().await, + publication["expected"], + "{}: {}", + test_case.description, + publication["actions"] + ); + } + let mut removed = snapshot(roles.location_id); + removed.generation = 9; + publish_snapshot(store, &removed) + .await + .expect("publish without roles"); + assert_eq!(grants().await, test_case.expected_after_removal); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_recovery.rs b/crates/sqlbuild-orchestration/tests/credentials/test_recovery.rs new file mode 100644 index 0000000000..d36fb9dbce --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_recovery.rs @@ -0,0 +1,195 @@ +//! grant-admin works online, is atomic and audited, and cannot reopen first-admin initialization. + +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::authorization::main::{bootstrap_user::bootstrap_user, grant_admin::grant_admin}; +use crate::credentials::main::{ + sessions::authenticate_bound_person::authenticate_bound_person, + sessions::authenticate_session::authenticate_session, sessions::login_password::login_password, + sessions::set_password::set_password, +}; +use crate::credentials::models::{AuditContext, PasswordAuthority, PasswordLogin, SessionAccess}; +use crate::credentials::tests::helpers::security_event_count; +use crate::credentials::tests::test_types::AdministratorRecoveryTestCase; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_lost_administrator_access_when_grant_admin_runs_online_then_authority_and_audit_commit_atomically() + { + let test_cases = [ + AdministratorRecoveryTestCase { + description: "SQLite audited grant-admin", + backend: Backend::Sqlite, + expected_recoveries: 1, + }, + AdministratorRecoveryTestCase { + description: "PostgreSQL audited grant-admin", + backend: Backend::Postgres, + expected_recoveries: 1, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("database"); + migrate_store(&fixture.store).await.expect("migrate"); + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("trusted local peer")), + }; + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json", + ); + let inputs: Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("DuckShop recovery")) + .expect("inputs"); + let name = inputs["first_person"].as_str().expect("person name"); + let password = inputs["password"].as_str().expect("password"); + let principal = bootstrap_user(&fixture.store, &context, name) + .await + .expect("first admin"); + assert!( + bootstrap_user(&fixture.store, &context, "customers-reader") + .await + .is_err(), + "bootstrap closes once any person exists" + ); + set_password( + &fixture.store, + &context, + PasswordAuthority::Operator(principal), + password, + ) + .await + .expect("password"); + let PasswordLogin::Authenticated(session) = + login_password(&fixture.store, &context, (name, password)) + .await + .expect("login") + else { + panic!("valid password"); + }; + let external_session = authenticate_bound_person(&fixture.store, &context, principal) + .await + .expect("verified OIDC identity") + .expect("SSO session"); + let (transaction, _) = query( + begin(&fixture.store, TransactionMode::Write) + .await + .expect("simulate loss"), + "UPDATE sqb_principals SET disabled_at=$1 WHERE principal_id=$2", + vec![ + Parameter::Time(chrono::Utc::now()), + Parameter::Id(principal), + ], + ) + .await + .expect("disable administrator"); + let (transaction, _) = query( + transaction, + "DELETE FROM sqb_role_assignments WHERE principal_id=$1", + vec![Parameter::Id(principal)], + ) + .await + .expect("lose administrator assignment"); + commit(transaction).await.expect("commit simulated loss"); + let revision = security_event_count(&fixture.store) + .await + .expect("revision"); + for (table, operation) in [ + ("sqb_security_audit", "INSERT"), + ("sqb_role_assignments", "INSERT"), + ("sqb_sessions", "UPDATE"), + ] { + let fault = TransactionFault { + description: "grant-admin publication fails", + table, + operation, + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'recovery unavailable')", + postgres_body: "RAISE EXCEPTION 'recovery unavailable'", + }; + install_fault(&fixture.store, &fault).await; + assert!( + grant_admin(&fixture.store, &context, principal) + .await + .is_err() + ); + remove_fault(&fixture.store, &fault).await; + assert_eq!( + security_event_count(&fixture.store) + .await + .expect("revision after rollback"), + revision + ); + let (transaction, rows) = query(begin(&fixture.store, TransactionMode::Read).await.expect("inspect rollback"), + "SELECT (SELECT COUNT(*) FROM sqb_principals WHERE disabled_at IS NOT NULL) AS disabled, (SELECT COUNT(*) FROM sqb_role_assignments) AS roles, (SELECT COUNT(*) FROM sqb_sessions WHERE revoked_at IS NULL) AS sessions, (SELECT COUNT(*) FROM sqb_security_audit WHERE action='admin.granted') AS recovered", vec![]).await.expect("rollback counts"); + assert_eq!( + serde_json::to_value(&rows[0]).expect("row"), + json!({"disabled":1,"roles":0,"sessions":2,"recovered":0}) + ); + commit(transaction).await.expect("finish rollback read"); + } + assert!( + grant_admin(&fixture.store, &context, Uuid::nil()) + .await + .is_err(), + "grant-admin requires an existing person" + ); + grant_admin(&fixture.store, &context, principal) + .await + .expect("restore administrator while the store stays open"); + assert!( + authenticate_session(&fixture.store, &session.bearer, SessionAccess::Read) + .await + .expect("old session") + .is_none() + ); + assert!( + authenticate_session( + &fixture.store, + &external_session.bearer, + SessionAccess::Read + ) + .await + .expect("old SSO session") + .is_none() + ); + assert!(matches!( + login_password(&fixture.store, &context, (name, password)) + .await + .expect("fresh password login"), + PasswordLogin::Authenticated(_) + )); + assert!( + bootstrap_user(&fixture.store, &context, "replacement-admin") + .await + .is_err(), + "grant-admin never reopens bootstrap" + ); + let (transaction, rows) = query(begin(&fixture.store, TransactionMode::Read).await.expect("inspect recovery"), + "SELECT actor, granting_role, after_json FROM sqb_security_audit WHERE action='admin.granted'", vec![]).await.expect("durable recovery audit"); + assert_eq!( + rows.len(), + test_case.expected_recoveries, + "{}", + test_case.description + ); + assert_eq!(rows[0]["actor"], "local-control-store-owner"); + assert_eq!(rows[0]["granting_role"], "control-store-owner"); + let after = &rows[0]["after_json"]; + assert_eq!(after["sessions_revoked"], 2); + commit(transaction).await.expect("finish audit read"); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_types.rs b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs new file mode 100644 index 0000000000..d4dc5ef2e3 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs @@ -0,0 +1,57 @@ +use std::sync::{Arc, Mutex, atomic::AtomicU16}; + +use serde_json::Value; + +pub(crate) use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct CredentialTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_hash_length: usize, +} + +pub(crate) struct AdministratorTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_administrators: usize, +} + +pub(crate) struct AdministratorRecoveryTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_recoveries: usize, +} + +pub(crate) struct AuditTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_page_length: usize, +} + +#[derive(Clone, Debug)] +pub(crate) struct AuditWebhookReceiver { + pub(crate) requests: Arc>>, + pub(crate) status: Arc, +} + +pub(crate) struct AuditRetentionTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_retained: i64, +} + +pub(crate) struct CustomRoleTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_after_removal: serde_json::Value, +} + +/// The first administrator, and a member holding the published support role. +pub(crate) struct RoleFixture { + pub(crate) context: crate::credentials::models::AuditContext, + pub(crate) location_id: uuid::Uuid, + pub(crate) administrator: crate::credentials::models::IssuedSession, + pub(crate) member: crate::credentials::models::IssuedSession, + pub(crate) member_id: uuid::Uuid, +} diff --git a/crates/sqlbuild-orchestration/tests/platform/helpers.rs b/crates/sqlbuild-orchestration/tests/platform/helpers.rs new file mode 100644 index 0000000000..46d8ad3fa3 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/helpers.rs @@ -0,0 +1,223 @@ +use crate::scheduler::tests::contracts::ticks::helpers::calendar_text; +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +use axum::Json; +use axum::extract::State; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::IntoResponse; + +use crate::platform::models::{PlatformEvent, PlatformEventEnvelope}; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::publish_run_recall::publish_run_recall; +use crate::runs::models::{RecallBatch, RecallRecord}; +use crate::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; +use crate::store::tests::helpers::read_recall_events; +use chrono::{TimeDelta, Utc}; +use serde_json::json; +use sqlbuild_recall::records::models::RecallValue; +use sqlbuild_recall::schema::models::Table; +use tokio::task::JoinHandle; +use uuid::Uuid; + +use crate::platform::tests::test_types::{Observation, WebhookReceiver}; +use crate::runs::tests::helpers::{queue, request}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; + +pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { + let future = Utc::now() + TimeDelta::days(10); + let mut triggers = BTreeMap::new(); + for index in 0..101 { + triggers.insert( + format!("daily_orders_{index}"), + json!({ + "description": "Refresh orders", "job_key": "orders", + "starts_at": calendar_text(0), + "schedule": {"kind": "calendar", "times": [calendar_text(0), future], + "refresh_at": calendar_text(0)} + }), + ); + } + for (name, status) in [("expired_orders", "running"), ("paused_orders", "paused")] { + triggers.insert(name.into(), json!({ + "description": "Refresh orders", "job_key": "orders", "default_status": status, + "starts_at": calendar_text(0), + "schedule": {"kind": "calendar", "times": [calendar_text(0)], "refresh_at": calendar_text(0)} + })); + } + triggers.insert( + "future_orders".into(), + json!({ + "description": "Future orders refresh", "job_key": "orders", + "schedule": {"kind": "calendar", "times": [future], "refresh_at": future} + }), + ); + SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: None, + environment_hash: "orders-watchdog-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::from([( + "orders".into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"materialization": "table"}), + raw_sql: Some("select 1 as order_id".into()), + compiled_sql: Some("select 1 as order_id".into()), + python_source: None, + }, + )]), + documents: BTreeMap::from([ + ( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), + ), + ("triggers".into(), json!(triggers)), + ]), + } +} + +pub(crate) async fn wait_for_scans( + store: &ControlStore, + replicas: &BTreeSet, + tasks: &[JoinHandle>], +) -> Vec { + tokio::time::timeout(Duration::from_secs(10), async { + let mut cursor = 0; + let mut observations = Vec::new(); + let mut scanned = BTreeSet::new(); + for _ in 0..1000 { + let Some(_) = replicas.difference(&scanned).next() else { + return observations; + }; + assert!( + tasks.iter().all(|task| !task.is_finished()), + "watchdog failed before scanning" + ); + let batch: Vec = read_recall_events(store, cursor, 1000) + .await + .expect("read health facts") + .into_iter() + .map(|row| serde_json::from_value(json!(row)).expect("typed platform event")) + .collect(); + cursor = batch.last().map_or(cursor, |row| row.storage_id); + scanned.extend(batch.iter().filter_map(|row| { + let PlatformEvent::Heartbeat { replica_id } = row.payload.platform else { + return None; + }; + Some(replica_id) + })); + observations.extend(batch); + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("watchdog exhausted scan completion attempts"); + }) + .await + .expect("both replicas finish a paged scan without a scheduler") +} + +pub(crate) async fn receive_webhook( + State(receiver): State, + headers: HeaderMap, + Json(event): Json, +) -> impl IntoResponse { + assert_eq!( + headers.get("Idempotency-Key").expect("deduplication key"), + event.event_id.to_string().as_str() + ); + let mut state = receiver.state.lock().expect("receiver state"); + state.requests.push(event); + receiver.delivered.send_replace(state.requests.len()); + ( + StatusCode::from_u16(state.status).expect("configured response status"), + [(axum::http::header::LOCATION, "/events")], + ) +} + +pub(crate) async fn wait_for_webhook( + receiver: &WebhookReceiver, + dispatcher: &JoinHandle>, + count: usize, +) { + let mut delivered = receiver.delivered.subscribe(); + tokio::time::timeout( + Duration::from_secs(10), + delivered.wait_for(|received| *received >= count), + ) + .await + .expect("dispatcher drains more than one bounded page") + .expect("webhook receiver remains available"); + assert!( + !dispatcher.is_finished(), + "delivery loop stopped before draining" + ); +} + +pub(crate) async fn publish_platform_impostor(store: &ControlStore, identity: (Uuid, Uuid)) { + let mut requested = request(identity, 42, 0, &["orders"]); + requested.image_id = None; + queue(store, &requested).await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim") + .expect("work"); + let batch = RecallBatch { + invocation_id: "platform/impostor".into(), + records: vec![RecallRecord { + table: Table::Events, + values: BTreeMap::from([ + ( + "event_id".into(), + RecallValue::Text(Uuid::now_v7().to_string()), + ), + ( + "event_type".into(), + RecallValue::Text("platform_heartbeat".into()), + ), + ("event_schema_version".into(), RecallValue::Integer(1)), + ("event_kind".into(), RecallValue::Text("runtime".into())), + ("severity".into(), RecallValue::Text("info".into())), + ( + "occurred_at".into(), + RecallValue::Text("2026-01-01T00:00:00Z".into()), + ), + ("producer".into(), RecallValue::Text("platform".into())), + ("invocation_sequence".into(), RecallValue::Integer(0)), + ( + "payload".into(), + RecallValue::Text( + json!({"platform": {"kind": "heartbeat", "replica_id": Uuid::now_v7()}}) + .to_string(), + ), + ), + ]), + }], + }; + let result = publish_run_recall(store, (claim.request.run_id, claim.claim_id), &batch) + .await + .expect("publish untrusted producer"); + assert_eq!(result.delivered, [(Table::Events, 1)]); + assert!(result.conflicts.is_empty()); +} + +/// Every platform fact still stored after retention, in storage order. +pub(crate) async fn retained_platform_facts(store: &ControlStore) -> Vec { + read_recall_events(store, 0, 1000) + .await + .expect("read retained facts") + .into_iter() + .filter(|row| row["producer"] == "platform") + .map(|row| serde_json::from_value(json!(row)).expect("typed platform event")) + .collect() +} diff --git a/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs b/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs new file mode 100644 index 0000000000..0f5c53a679 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs @@ -0,0 +1,10 @@ +//! Platform contracts compiled within the platform owner. + +#[path = "helpers.rs"] +mod helpers; +#[path = "test_types.rs"] +mod test_types; +#[path = "test_watchdog.rs"] +mod test_watchdog; +#[path = "test_webhook.rs"] +mod test_webhook; diff --git a/crates/sqlbuild-orchestration/tests/platform/test_types.rs b/crates/sqlbuild-orchestration/tests/platform/test_types.rs new file mode 100644 index 0000000000..52ecb92e0c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/test_types.rs @@ -0,0 +1,53 @@ +use std::sync::{Arc, Mutex}; + +use crate::platform::models::{PlatformEvent, PlatformEventEnvelope}; +use chrono::{DateTime, Utc}; +use serde::Deserialize; + +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct WatchdogTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) leaders: usize, + pub(crate) expected_no_leader: bool, + pub(crate) expected_overdue: usize, + pub(crate) expected_low_horizons: usize, +} + +pub(crate) struct LocationWatchdogTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_broken: usize, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct Observation { + pub(crate) storage_id: i64, + pub(crate) occurred_at: DateTime, + pub(crate) payload: PlatformPayload, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct PlatformPayload { + pub(crate) platform: PlatformEvent, +} + +pub(crate) struct PlatformSinkTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) initial_status: u16, + pub(crate) expected_events: usize, +} + +#[derive(Clone, Debug)] +pub(crate) struct WebhookReceiver { + pub(crate) state: Arc>, + pub(crate) delivered: tokio::sync::watch::Sender, +} + +#[derive(Debug)] +pub(crate) struct WebhookState { + pub(crate) status: u16, + pub(crate) requests: Vec, +} diff --git a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs new file mode 100644 index 0000000000..2c86bf4d88 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs @@ -0,0 +1,433 @@ +use std::collections::BTreeSet; +use std::time::Duration; + +use crate::platform::_helpers::retention::retain; +use crate::platform::_helpers::watchdog::scan_health; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::main::run_maintenance::run_maintenance; +use crate::platform::models::{LocationHealth, LocationHealthReport, PlatformEvent}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::query::query; +use crate::store::models::{Parameter, TransactionMode}; +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::platform::tests::helpers::{retained_platform_facts, snapshot, wait_for_scans}; +use crate::platform::tests::test_types::{LocationWatchdogTestCase, Observation, WatchdogTestCase}; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_durable_deduplicated_and_independent_of_leadership() + { + let test_cases = [ + WatchdogTestCase { + description: "SQLite without a leader", + backend: Backend::Sqlite, + leaders: 0, + expected_no_leader: true, + expected_overdue: 102, + expected_low_horizons: 1, + }, + WatchdogTestCase { + description: "PostgreSQL without a leader", + backend: Backend::Postgres, + leaders: 0, + expected_no_leader: true, + expected_overdue: 102, + expected_low_horizons: 1, + }, + WatchdogTestCase { + description: "SQLite with a frozen leader", + backend: Backend::Sqlite, + leaders: 1, + expected_no_leader: false, + expected_overdue: 102, + expected_low_horizons: 1, + }, + WatchdogTestCase { + description: "PostgreSQL with a frozen leader", + backend: Backend::Postgres, + leaders: 1, + expected_no_leader: false, + expected_overdue: 102, + expected_low_horizons: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate health store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input + .documents + .get_mut("triggers") + .expect("trigger document")["removed_orders"] = + input.documents["triggers"]["expired_orders"].clone(); + publish_snapshot(store, &input) + .await + .expect("old definition"); + input.generation = 2; + input + .documents + .get_mut("triggers") + .expect("trigger document") + .as_object_mut() + .expect("trigger map") + .remove("removed_orders"); + publish_snapshot(store, &input) + .await + .expect("current definition"); + for _ in 0..test_case.leaders { + acquire_lease(store, Uuid::now_v7()) + .await + .expect("acquire lease") + .expect("leader"); + } + let second = second_store(store).await; + let replicas = BTreeSet::from([Uuid::now_v7(), Uuid::now_v7()]); + let (stop, shutdown) = tokio::sync::watch::channel(false); + let tasks: Vec<_> = [store.clone(), second.clone()] + .into_iter() + .zip(replicas.iter()) + .map(|(store, id)| { + let id = *id; + let shutdown = shutdown.clone(); + tokio::spawn(async move { + run_maintenance((&store, id), (Default::default(), &[]), (|_| {}, shutdown)) + .await + }) + }) + .collect(); + let observations = wait_for_scans(store, &replicas, &tasks).await; + let overdue: BTreeSet<_> = observations + .iter() + .filter_map(|row| { + let PlatformEvent::TickOverdue { trigger_id, .. } = row.payload.platform else { + return None; + }; + Some(trigger_id) + }) + .collect(); + assert_eq!( + overdue.len(), + test_case.expected_overdue, + "{}", + test_case.description + ); + let horizons: BTreeSet<_> = observations + .iter() + .filter_map(|row| { + let PlatformEvent::CalendarHorizonLow { trigger_id, .. } = row.payload.platform + else { + return None; + }; + Some(trigger_id) + }) + .collect(); + assert_eq!( + horizons.len(), + test_case.expected_low_horizons, + "paused and removed calendars are excluded; refresh cadence alone is not a low horizon" + ); + assert_eq!( + observations + .iter() + .any(|row| matches!(row.payload.platform, PlatformEvent::SchedulerNoLeader)), + test_case.expected_no_leader + ); + let logical_keys: BTreeSet<_> = observations + .iter() + .map(|row| { + ( + row.occurred_at, + serde_json::to_string(&row.payload.platform).expect("event identity"), + ) + }) + .collect(); + assert_eq!( + logical_keys.len(), + observations.len(), + "replicas do not duplicate an observation within its window" + ); + assert!( + observations + .iter() + .all(|row| row.occurred_at.timestamp().rem_euclid(30) == 0) + ); + stop.send(true).expect("shutdown both replicas"); + for task in tasks { + tokio::time::timeout(Duration::from_secs(2), task) + .await + .expect("prompt shutdown") + .expect("join watchdog") + .expect("watchdog stopped"); + } + let transaction = begin(store, TransactionMode::Write) + .await + .expect("age health facts"); + let aged = transaction.now - TimeDelta::days(3); + let (transaction, _) = query( + transaction, + "UPDATE recall_events SET occurred_at=$1 WHERE producer='platform'", + vec![Parameter::Time(aged)], + ) + .await + .expect("aged health facts"); + commit(transaction).await.expect("aged facts committed"); + retain(store, Default::default(), &[]) + .await + .expect("retention pass"); + let alerts = observations + .iter() + .filter(|row| !matches!(row.payload.platform, PlatformEvent::Heartbeat { .. })) + .count(); + let remaining = retained_platform_facts(store).await; + assert_eq!( + remaining + .iter() + .filter(|row| matches!(row.payload.platform, PlatformEvent::Heartbeat { .. })) + .count(), + 0, + "{}: expired replica heartbeats are pruned", + test_case.description + ); + assert!( + remaining.len() >= alerts, + "{}: alert facts outlive the heartbeat horizon for webhooks and sensors", + test_case.description + ); + let (_running, idle) = tokio::sync::watch::channel(false); + scan_health(store, Uuid::now_v7(), &idle) + .await + .expect("later window scan"); + let rescanned = retained_platform_facts(store).await; + let alert_count = |facts: &[Observation]| { + facts + .iter() + .filter(|row| !matches!(row.payload.platform, PlatformEvent::Heartbeat { .. })) + .count() + }; + assert_eq!( + (alert_count(&rescanned), rescanned.len()), + (alert_count(&remaining), remaining.len() + 1), + "{}: a persisting condition is not recorded again; only the heartbeat is new", + test_case.description + ); + close_store(second).await; + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_stale_location_sessions_when_watchdogs_scan_then_latest_target_liveness_controls_paged_outage_events() + { + let test_cases = [ + LocationWatchdogTestCase { + description: "SQLite latest target receipts", + backend: Backend::Sqlite, + expected_broken: 104, + }, + LocationWatchdogTestCase { + description: "PostgreSQL latest target receipts", + backend: Backend::Postgres, + expected_broken: 104, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let archived = create_location(store, "inventory", "test-admin") + .await + .expect("archived location"); + create_location(store, "products", "test-admin") + .await + .expect("never observed location"); + for (location_id, count) in [(location, 105), (archived, 1)] { + for index in 0..count { + for _ in 0..2 { + report_location_health( + store, + location_id, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: Uuid::now_v7(), + target: format!("orders_{index:03}"), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("register old session"); + } + } + } + let transaction = begin(store, TransactionMode::Write) + .await + .expect("age sessions"); + let stale_at = transaction.now - TimeDelta::seconds(105); + let (transaction, _) = query( + transaction, + "UPDATE sqb_location_instances SET last_seen_at=$1", + vec![Parameter::Time(stale_at)], + ) + .await + .expect("age all old sessions"); + let now = transaction.now; + let (transaction, _) = query( + transaction, + "UPDATE sqb_locations SET archived_at=$1 WHERE location_id=$2", + vec![Parameter::Time(now), Parameter::Id(archived)], + ) + .await + .expect("archive location"); + commit(transaction).await.expect("commit fixture"); + // A fresh successor protects only its target, not another target of the same location. + let healthy_session = Uuid::now_v7(); + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: healthy_session, + target: "orders_104".into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("healthy successor"); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("age successor"); + let recent_at = transaction.now - TimeDelta::seconds(75); + let (transaction, _) = query( + transaction, + "UPDATE sqb_location_instances SET last_seen_at=$1 WHERE instance_id=$2", + vec![Parameter::Time(recent_at), Parameter::Id(healthy_session)], + ) + .await + .expect("two missed heartbeats"); + commit(transaction).await.expect("commit successor time"); + let second = second_store(store).await; + let replicas = BTreeSet::from([Uuid::now_v7(), Uuid::now_v7()]); + let (stop, shutdown) = tokio::sync::watch::channel(false); + let tasks: Vec<_> = [store.clone(), second.clone()] + .into_iter() + .zip(replicas.iter()) + .map(|(store, id)| { + let id = *id; + let shutdown = shutdown.clone(); + tokio::spawn(async move { + run_maintenance((&store, id), (Default::default(), &[]), (|_| {}, shutdown)) + .await + }) + }) + .collect(); + let observations = wait_for_scans(store, &replicas, &tasks).await; + let broken: BTreeSet<_> = observations + .iter() + .filter_map(|row| { + let PlatformEvent::LocationBroken { + location_id, + ref target, + last_seen_at, + } = row.payload.platform + else { + return None; + }; + assert_eq!( + location_id, location, + "archived and never-observed locations are excluded" + ); + assert_eq!( + last_seen_at, stale_at, + "payload retains the latest server receipt time" + ); + Some(target.clone()) + }) + .collect(); + assert_eq!( + broken, + (0..test_case.expected_broken) + .map(|index| format!("orders_{index:03}")) + .collect(), + "{}: both pages are read, old replicas deduplicate, and the healthy target is excluded", + test_case.description + ); + let keys: BTreeSet<_> = observations + .iter() + .map(|row| { + ( + row.occurred_at, + serde_json::to_string(&row.payload.platform).expect("identity"), + ) + }) + .collect(); + assert_eq!( + keys.len(), + observations.len(), + "replicas share observation keys" + ); + stop.send(true).expect("stop replicas"); + for task in tasks { + tokio::time::timeout(Duration::from_secs(2), task) + .await + .expect("prompt shutdown") + .expect("join") + .expect("watchdog stopped"); + } + let transaction = begin(store, TransactionMode::Write) + .await + .expect("age instances"); + let aged = transaction.now - TimeDelta::days(8); + let (transaction, _) = query( + transaction, + "UPDATE sqb_location_instances SET last_seen_at=$1", + vec![Parameter::Time(aged)], + ) + .await + .expect("aged instances"); + commit(transaction).await.expect("aged instances committed"); + retain(store, Default::default(), &[]) + .await + .expect("retention pass"); + let transaction = begin(store, TransactionMode::Read) + .await + .expect("read instances"); + let (transaction, instances) = query( + transaction, + "SELECT COUNT(*) AS count FROM sqb_location_instances", + vec![], + ) + .await + .expect("count instances"); + commit(transaction).await.expect("read instances committed"); + assert_eq!( + instances[0]["count"], 0, + "{}: instances unseen past the location horizon are pruned", + test_case.description + ); + close_store(second).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/platform/test_webhook.rs b/crates/sqlbuild-orchestration/tests/platform/test_webhook.rs new file mode 100644 index 0000000000..6cb93e77ae --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/test_webhook.rs @@ -0,0 +1,199 @@ +use std::collections::BTreeSet; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use axum::Router; +use axum::routing::post; +use uuid::Uuid; + +use crate::platform::main::run_maintenance::run_maintenance; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::webhooks::_helpers::delivery::deliver_webhook; +use crate::webhooks::main::{connect_webhook::connect_webhook, run_webhook::run_webhook}; +use crate::webhooks::models::WebhookSource; + +use crate::credentials::tests::location_helpers::create_location; +use crate::platform::tests::helpers::{ + publish_platform_impostor, receive_webhook, snapshot, wait_for_scans, wait_for_webhook, +}; +use crate::platform::tests::test_types::{PlatformSinkTestCase, WebhookReceiver, WebhookState}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_location_outage_when_webhook_delivery_retries_then_all_health_facts_survive_restart_and_lost_acknowledgements() + { + let test_cases = [ + PlatformSinkTestCase { + description: "SQLite durable platform delivery", + backend: Backend::Sqlite, + initial_status: 503, + expected_events: 105, + }, + PlatformSinkTestCase { + description: "PostgreSQL durable platform delivery", + backend: Backend::Postgres, + initial_status: 307, + expected_events: 105, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate delivery store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("publish schedules"); + // No location process and no scheduler are running. + let replica_id = Uuid::now_v7(); + let (stop, shutdown) = tokio::sync::watch::channel(false); + let watchdog_store = store.clone(); + let tasks = [tokio::spawn(async move { + run_maintenance( + (&watchdog_store, replica_id), + (Default::default(), &[]), + (|_| {}, shutdown), + ) + .await + })]; + let observations = wait_for_scans(store, &BTreeSet::from([replica_id]), &tasks).await; + stop.send(true).expect("stop watchdog"); + for task in tasks { + task.await + .expect("watchdog task") + .expect("watchdog stopped"); + } + assert_eq!( + observations.len(), + test_case.expected_events, + "{}", + test_case.description + ); + publish_platform_impostor(store, (location, published.snapshot_id)).await; + + let state = Arc::new(Mutex::new(WebhookState { + status: test_case.initial_status, + requests: Vec::new(), + })); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("webhook listener"); + let endpoint = format!( + "http://delivery:sample-password@{}/events?token=sample-token", + listener.local_addr().expect("address") + ); + let receiver = WebhookReceiver { + state: state.clone(), + delivered: tokio::sync::watch::channel(0).0, + }; + let app = Router::new() + .route("/events", post(receive_webhook)) + .with_state(receiver.clone()); + let (http_stop, http_shutdown) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + axum::serve(listener, app) + .with_graceful_shutdown(async { + drop(http_shutdown.await); + }) + .await + }); + let sink = + connect_webhook(&endpoint, WebhookSource::Platform).expect("configure webhook"); + let diagnostic = format!("{sink:?}"); + assert!(!diagnostic.contains("sample-password")); + assert!(!diagnostic.contains("sample-token")); + assert!(deliver_webhook(store, &sink).await.is_err()); + assert_eq!( + state.lock().expect("receiver").requests.len(), + 1, + "failure stops the page" + ); + state.lock().expect("receiver").status = 204; + let fault = TransactionFault { + description: "lose a successful webhook acknowledgement", + table: "sqb_platform_sink_deliveries", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'test receipt failure')", + postgres_body: "RAISE EXCEPTION 'test receipt failure'", + }; + install_fault(store, &fault).await; + assert!(deliver_webhook(store, &sink).await.is_err()); + remove_fault(store, &fault).await; + let second = second_store(store).await; + let restarted = connect_webhook(&endpoint, WebhookSource::Platform) + .expect("restart webhook client"); + let (sink_stop, sink_shutdown) = tokio::sync::watch::channel(false); + let sink_store = second.clone(); + let dispatcher = + tokio::spawn( + async move { run_webhook(&sink_store, restarted, sink_shutdown).await }, + ); + wait_for_webhook(&receiver, &dispatcher, test_case.expected_events + 2).await; + sink_stop.send(true).expect("stop dispatcher"); + tokio::time::timeout(Duration::from_secs(2), dispatcher) + .await + .expect("bounded dispatcher shutdown") + .expect("dispatcher task") + .expect("dispatcher stopped"); + // Shutdown may cancel the last receipt after the remote endpoint accepted it. + assert!( + deliver_webhook(store, &sink) + .await + .expect("reconcile in-flight delivery") + <= 1 + ); + assert_eq!( + deliver_webhook(store, &sink).await.expect("durable cursor"), + 0 + ); + { + let receiver = state.lock().expect("receiver"); + let requests = &receiver.requests; + assert!( + (test_case.expected_events + 2..=test_case.expected_events + 3) + .contains(&requests.len()) + ); + assert_eq!(requests[0], requests[1], "HTTP retry retains full identity"); + assert_eq!( + requests[1], requests[2], + "lost receipt retries the same fact" + ); + assert_eq!( + requests[2..] + .iter() + .map(|event| event.event_id) + .collect::>() + .len(), + test_case.expected_events + ); + let mut delivered = requests[2..].to_vec(); + delivered.dedup(); + assert_eq!(delivered.len(), observations.len()); + for (event, observation) in delivered.iter().zip(&observations) { + assert_eq!(event.storage_id, observation.storage_id); + assert_eq!(event.occurred_at, observation.occurred_at); + assert_eq!(event.payload.platform, observation.payload.platform); + } + } + http_stop.send(()).expect("stop receiver"); + tokio::time::timeout(Duration::from_secs(2), server) + .await + .expect("bounded shutdown") + .expect("server task") + .expect("server stopped"); + close_store(second).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs new file mode 100644 index 0000000000..6b2c0e2b1d --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs @@ -0,0 +1,72 @@ +//! Real slot accounting and successful completion through the public worker lifecycle. + +use std::collections::BTreeMap; + +use crate::runs::main::change_run::change_run; +use crate::runs::models::{ + NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome, +}; +use crate::store::models::ControlStore; +use uuid::Uuid; + +pub(crate) async fn slot_counts(store: &ControlStore) -> BTreeMap { + let sql = "SELECT group_key, count(*) FROM sqb_concurrency_slots GROUP BY group_key"; + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + return rusqlite::Connection::open(path) + .expect("inspect SQLite") + .prepare(sql) + .expect("slot query") + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .expect("SQLite slot counts") + .collect::>() + .expect("decode slot counts"); + }; + sqlx_core::query_as::query_as::<_, (String, i64)>(sql) + .fetch_all(pool) + .await + .expect("PostgreSQL slot counts") + .into_iter() + .collect() +} + +pub(crate) async fn succeed(store: &ControlStore, claim: &RunClaim) { + let mut changes = vec![RunChange::Started]; + changes.extend( + claim + .request + .selection + .iter() + .map(|node| RunChange::NodeFinished { + node_key: node.clone(), + outcome: NodeOutcome::Success, + rows_affected: Some(1), + error_code: None, + skipped_because_of: None, + }), + ); + changes.push(RunChange::Finished { + outcome: RunOutcome::Success, + reason: "synthetic child completed".into(), + exit_code: Some(0), + }); + for change in changes { + change_run( + store, + RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await + .expect("worker fact"); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs new file mode 100644 index 0000000000..c24ed3b721 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -0,0 +1,104 @@ +//! Run fixtures exercise native queues and Recall against isolated real databases. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::runs::_helpers::requests::publication::publish_request; +use crate::runs::models::{RunKind, RunOrigin, RunRequest}; +use crate::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; +use serde_json::json; +use uuid::Uuid; + +pub(crate) async fn request_run( + store: &ControlStore, + request: &RunRequest, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, applied) = publish_request(transaction, request).await?; + commit(transaction).await?; + Ok(applied) +} + +pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { + SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: Some("sha256:synthetic-orders-image".into()), + environment_hash: "orders-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: ["customers", "inventory", "orders", "products"] + .into_iter() + .map(|name| { + ( + name.into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"materialization": "table"}), + raw_sql: Some("select 1 as id".into()), + compiled_sql: Some("select 1 as id".into()), + python_source: None, + }, + ) + }) + .collect(), + documents: BTreeMap::new(), + } +} + +pub(crate) fn request( + (location_id, snapshot_id): (Uuid, Uuid), + id: u128, + priority: i64, + nodes: &[&str], +) -> RunRequest { + let selection: BTreeSet = nodes.iter().map(|name| (*name).into()).collect(); + let run_id = Uuid::from_u128(id); + RunRequest { + run_id, + location_id, + target: "development".into(), + snapshot_id, + image_id: Some("sha256:synthetic-orders-image".into()), + kind: RunKind::Adhoc, + origin: RunOrigin::Manual, + job_key: None, + parent_run_id: None, + root_run_id: run_id, + trigger_id: None, + tick_id: None, + rerun_of_run_id: None, + requested_by: "test-operator".into(), + priority, + write_set: selection.clone(), + selection, + settings: BTreeMap::new(), + window: None, + concurrency_groups: BTreeSet::new(), + } +} + +pub(crate) async fn queue(store: &ControlStore, request: &RunRequest) { + assert!( + request_run(store, request) + .await + .expect("queue pinned request") + ); + assert!( + !request_run(store, request) + .await + .expect("retry pinned request") + ); + let mut changed = request.clone(); + changed.priority += 1; + assert!( + request_run(store, &changed).await.is_err(), + "a reused run ID cannot change its request" + ); +} diff --git a/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs new file mode 100644 index 0000000000..4f99fce097 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs @@ -0,0 +1,68 @@ +//! Advance persisted liveness, not the scheduler clock, to exercise real monitoring transactions. + +use std::collections::BTreeMap; + +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunClaim, RunRequest}; +use crate::scheduler::models::SchedulerLease; +use crate::store::models::ControlStore; +use chrono::{TimeDelta, Utc}; +use uuid::Uuid; + +use crate::runs::tests::helpers::queue; + +pub(crate) async fn start_run( + store: &ControlStore, + lease: &SchedulerLease, + input: &RunRequest, +) -> RunClaim { + queue(store, input).await; + let claim = claim_next_run(store, lease, &BTreeMap::new(), None) + .await + .expect("claim queued run") + .expect("run available"); + assert_eq!(claim.request.run_id, input.run_id); + change_run( + store, + RunAuthority::Worker { + run_id: input.run_id, + claim_id: claim.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: input.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Started, + }, + ) + .await + .expect("start claimed worker"); + claim +} + +pub(crate) async fn age_heartbeat(store: &ControlStore, run_id: Uuid, seconds: i64) { + let timestamp = Utc::now() - TimeDelta::seconds(seconds); + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + rusqlite::Connection::open(path) + .expect("open liveness fixture") + .execute( + "UPDATE sqb_runs SET last_heartbeat_at=?1 WHERE run_id=?2", + rusqlite::params![ + timestamp.to_rfc3339_opts(chrono::SecondsFormat::Micros, true), + run_id.to_string() + ], + ) + .expect("age SQLite heartbeat"); + return; + }; + sqlx_core::query::query("UPDATE sqb_runs SET last_heartbeat_at=$1 WHERE run_id=$2") + .bind(timestamp) + .bind(run_id) + .execute(pool) + .await + .expect("age PostgreSQL heartbeat"); +} diff --git a/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs new file mode 100644 index 0000000000..5b31b83c2f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs @@ -0,0 +1,30 @@ +//! Terminal facts exercise claim release through the public run lifecycle. + +use crate::runs::main::change_run::change_run; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome}; +use crate::scheduler::models::SchedulerLease; +use crate::store::models::ControlStore; +use uuid::Uuid; + +pub(crate) async fn finish(store: &ControlStore, lease: &SchedulerLease, claim: &RunClaim) { + let request = RunChangeRequest { + event_id: Uuid::new_v5(&claim.claim_id, b"test-finished"), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Finished { + outcome: RunOutcome::Failed, + reason: "synthetic worker exit".into(), + exit_code: Some(1), + }, + }; + assert!( + change_run(store, RunAuthority::Scheduler(lease), &request) + .await + .expect("terminate claimed work") + ); + assert!( + !change_run(store, RunAuthority::Scheduler(lease), &request) + .await + .expect("retry terminal fact") + ); +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_checks.rs b/crates/sqlbuild-orchestration/tests/runs/test_checks.rs new file mode 100644 index 0000000000..0fe7151327 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_checks.rs @@ -0,0 +1,259 @@ +//! Check observations are claim-bound facts, never implicit runnable selection members. + +use std::collections::BTreeMap; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::main::{ + change_run::change_run, claim_next_run::claim_next_run, list_run_nodes::list_run_nodes, + list_runs::list_runs, +}; +use crate::runs::models::{ + NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::test_types::RunLifecycleTestCase; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::SnapshotNode; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_attached_checks_when_a_claim_reports_observations_then_frozen_membership_and_execution_history_are_preserved() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite check observations", + backend: Backend::Sqlite, + expected_status: RunStatus::Failed, + }, + RunLifecycleTestCase { + description: "PostgreSQL check observations", + backend: Backend::Postgres, + expected_status: RunStatus::Failed, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + for name in [ + "failed", + "warned", + "skipped", + "cancelled", + "passed", + "blocked", + ] { + input.nodes.insert( + format!("check:{name}"), + SnapshotNode { + kind: "check".into(), + definition: json!({"name": name}), + raw_sql: None, + compiled_sql: None, + python_source: Some("def orders_valid(ctx): return ctx.pass_()".into()), + }, + ); + } + input + .nodes + .insert("check:products".into(), input.nodes["products"].clone()); + let published = publish_snapshot(store, &input).await.expect("snapshot"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + queue( + store, + &request((location, published.snapshot_id), 1, 1, &["orders"]), + ) + .await; + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim") + .expect("available"); + let worker = RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }; + let report = |change| RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }; + change_run(store, worker, &report(RunChange::Started)) + .await + .expect("started"); + let before = read_recall_events(store, 0, 100) + .await + .expect("before rejected reports"); + for change in [ + RunChange::CheckFinished { + node_key: "check:passed".into(), + outcome: NodeOutcome::Success, + }, + RunChange::CheckStarted { + node_key: "check:unknown".into(), + }, + RunChange::CheckStarted { + node_key: "check:products".into(), + }, + RunChange::CheckStarted { + node_key: "orders".into(), + }, + ] { + assert!(change_run(store, worker, &report(change)).await.is_err()); + } + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("rejected facts rolled back"), + before + ); + for (name, outcome) in [ + ("failed", NodeOutcome::Failed), + ("warned", NodeOutcome::Warn), + ("skipped", NodeOutcome::Skipped), + ("cancelled", NodeOutcome::Cancelled), + ("passed", NodeOutcome::Success), + ] { + let started = report(RunChange::CheckStarted { + node_key: format!("check:{name}"), + }); + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &started) + .await + .is_err() + ); + assert!( + change_run( + store, + worker, + &RunChangeRequest { + claim_id: Some(Uuid::now_v7()), + ..started.clone() + } + ) + .await + .is_err() + ); + assert!( + change_run(store, worker, &started) + .await + .expect("actual check start") + ); + assert!( + !change_run(store, worker, &started) + .await + .expect("idempotent start") + ); + let finished = report(RunChange::CheckFinished { + node_key: format!("check:{name}"), + outcome, + }); + assert!( + change_run(store, worker, &finished) + .await + .expect("actual check result") + ); + assert!( + !change_run(store, worker, &finished) + .await + .expect("idempotent completion") + ); + assert!( + change_run(store, worker, &report(finished.change.clone())) + .await + .is_err() + ); + assert!( + change_run(store, worker, &report(started.change.clone())) + .await + .is_err() + ); + } + change_run( + store, + worker, + &report(RunChange::CheckFinished { + node_key: "check:blocked".into(), + outcome: NodeOutcome::Skipped, + }), + ) + .await + .expect("blocked check was not executed"); + let nodes = list_run_nodes(store, claim.request.run_id) + .await + .expect("frozen nodes"); + assert_eq!(nodes.len(), 1); + assert_eq!(nodes[0].node_key, "orders"); + assert_eq!(nodes[0].status, "pending"); + let facts = read_recall_events(store, 0, 100) + .await + .expect("check facts"); + assert_eq!( + facts + .iter() + .filter(|row| row["event_type"] == "run_check_started") + .count(), + 5 + ); + assert_eq!( + facts + .iter() + .filter(|row| row["event_type"] == "run_check_finished") + .count(), + 6 + ); + for change in [ + RunChange::NodeStarted { + node_key: "orders".into(), + }, + RunChange::NodeFinished { + node_key: "orders".into(), + outcome: NodeOutcome::Success, + rows_affected: Some(1), + error_code: None, + skipped_because_of: None, + }, + RunChange::Finished { + outcome: RunOutcome::Failed, + reason: "Attached check failed".into(), + exit_code: Some(1), + }, + ] { + change_run(store, worker, &report(change)) + .await + .expect("finish execution without new membership"); + } + let runs = list_runs(store, (location, "development"), 10) + .await + .expect("runs"); + assert_eq!( + runs[0].status, test_case.expected_status, + "{}", + test_case.description + ); + assert_eq!( + runs[0].node_counts_json, + BTreeMap::from([("success".into(), 1)]) + ); + assert_eq!(runs[0].selection_json, claim.request.selection); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs new file mode 100644 index 0000000000..d8c4151143 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs @@ -0,0 +1,532 @@ +use std::collections::BTreeMap; +use std::num::NonZeroU32; + +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::list_run_claims::list_run_claims; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::RunStatus; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::claim_helpers::succeed; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::terminal_helpers::finish; +use crate::runs::tests::test_types::{ + BacklogClaimTestCase, GroupLimitTestCase, PriorityClaimsTestCase, ScopedClaimTestCase, +}; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_higher_priority_work_for_other_scopes_when_a_local_launcher_claims_then_only_its_location_and_target_are_reserved() + { + let test_cases = [ + ScopedClaimTestCase { + description: "SQLite local dispatch cannot reserve another location or target", + backend: Backend::Sqlite, + expected_selected_run: 1, + expected_unscoped_run: 3, + }, + ScopedClaimTestCase { + description: "PostgreSQL local dispatch cannot reserve another location or target", + backend: Backend::Postgres, + expected_selected_run: 1, + expected_unscoped_run: 3, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let other_published = publish_snapshot(store, &snapshot(other)) + .await + .expect("other location snapshot"); + let mut secondary = snapshot(location); + secondary.target = "secondary".into(); + let secondary = publish_snapshot(store, &secondary) + .await + .expect("other target snapshot"); + let wanted = request((location, published.snapshot_id), 1, 0, &["orders"]); + let mut other_target = request((location, secondary.snapshot_id), 2, 10, &["orders"]); + other_target.target = "secondary".into(); + let other_location = request((other, other_published.snapshot_id), 3, 20, &["orders"]); + for input in [&wanted, &other_target, &other_location] { + queue(store, input).await; + } + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let claim = claim_next_run( + store, + &lease, + &BTreeMap::new(), + Some((location, "development")), + ) + .await + .expect("scoped claim") + .expect("matching work"); + assert_eq!( + claim.request.run_id, + Uuid::from_u128(test_case.expected_selected_run), + "{}", + test_case.description + ); + assert_eq!( + list_run_claims(store, (location, "development"), None) + .await + .expect("recover queued claim"), + vec![claim.clone()], + "recovery includes a committed claim before launch" + ); + assert!( + list_run_claims(store, (location, "development"), Some(claim.request.run_id)) + .await + .expect("page after claimed run") + .is_empty() + ); + assert!( + list_run_claims(store, (other, "development"), None) + .await + .expect("unclaimed neighboring location") + .is_empty() + ); + assert!( + list_run_claims(store, (location, "secondary"), None) + .await + .expect("unclaimed neighboring target") + .is_empty() + ); + assert!( + claim_next_run( + store, + &lease, + &BTreeMap::new(), + Some((location, "development")) + ) + .await + .expect("scope exhausted") + .is_none() + ); + finish(store, &lease, &claim).await; + assert!( + list_run_claims(store, (location, "development"), None) + .await + .expect("terminal claims are not recoverable") + .is_empty() + ); + let next = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("unrestricted claim") + .expect("other work was not reserved"); + assert_eq!( + next.request.run_id, + Uuid::from_u128(test_case.expected_unscoped_run), + "{}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rollback_and_terminal_facts_release_claims() + { + let test_cases = [ + PriorityClaimsTestCase { + description: "SQLite claims prioritize independent write sets and release atomically", + backend: Backend::Sqlite, + expected_initial_claims: [2, 3], + expected_claim_events: 4, + expected_terminal_events: 2, + }, + PriorityClaimsTestCase { + description: "PostgreSQL claims prioritize independent write sets and release atomically", + backend: Backend::Postgres, + expected_initial_claims: [2, 3], + expected_claim_events: 4, + expected_terminal_events: 2, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate run store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("create location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("publish snapshot"); + let scope = (location, published.snapshot_id); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + for input in [ + request(scope, 1, 1, &["orders"]), + request(scope, 2, 3, &["orders"]), + request(scope, 3, 2, &["products"]), + ] { + queue(store, &input).await; + } + let second = second_store(store).await; + let limits = BTreeMap::new(); + let (first, other) = tokio::join!( + claim_next_run(store, &lease, &limits, None), + claim_next_run(&second, &lease, &limits, None) + ); + let first = first.expect("first claimant").expect("claim available"); + let other = other + .expect("second claimant") + .expect("independent claim available"); + let claimed: std::collections::BTreeSet<_> = + [first.request.run_id, other.request.run_id] + .into_iter() + .collect(); + assert_eq!( + claimed, + test_case + .expected_initial_claims + .into_iter() + .map(Uuid::from_u128) + .collect(), + "{}", + test_case.description + ); + assert!( + claim_next_run(store, &lease, &limits, None) + .await + .expect("blocked run stays queued") + .is_none() + ); + queue(store, &request(scope, 4, 9, &["customers", "orders"])).await; + queue(store, &request(scope, 5, 0, &["customers"])).await; + let independent = claim_next_run(store, &lease, &limits, None) + .await + .expect("try partially conflicting candidate") + .expect("independent work proceeds"); + assert_eq!( + independent.request.run_id, + Uuid::from_u128(5), + "customers was not leaked by the failed customers/orders claim" + ); + let orders = [&first, &other] + .into_iter() + .find(|claim| claim.request.run_id == Uuid::from_u128(2)) + .expect("orders owner"); + finish(store, &lease, orders).await; + finish(store, &lease, &independent).await; + let next = claim_next_run(store, &lease, &limits, None) + .await + .expect("terminal cleanup committed") + .expect("next priority run claims both nodes"); + assert_eq!(next.request.run_id, Uuid::from_u128(4)); + let events = read_recall_events(store, 0, 100) + .await + .expect("read immutable run stream"); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_claimed") + .count(), + test_case.expected_claim_events, + "{}", + test_case.description + ); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_terminated") + .count(), + test_case.expected_terminal_events, + "{}", + test_case.description + ); + let runs = list_runs(store, (location, "development"), 100) + .await + .expect("read atomic projections"); + assert_eq!( + runs.iter() + .filter(|run| run.status == RunStatus::Failed) + .count(), + 2 + ); + close_store(second).await; + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_named_group_limits_when_capacity_is_lowered_then_active_high_slots_still_count_and_terminal_cleanup_unblocks_work() + { + let test_cases = [ + GroupLimitTestCase { + description: "SQLite occupied high slots still count after lowering a group limit", + backend: Backend::Sqlite, + expected_unblocked_run: 3, + }, + GroupLimitTestCase { + description: "PostgreSQL occupied high slots still count after lowering a group limit", + backend: Backend::Postgres, + expected_unblocked_run: 3, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate run store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("create location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("publish snapshot"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let mut limits = BTreeMap::from([( + "warehouse".into(), + NonZeroU32::new(2).expect("nonzero limit"), + )]); + for id in 1..=3 { + let mut input = request((location, published.snapshot_id), id, 0, &["orders"]); + input.write_set.clear(); + input.concurrency_groups.insert("warehouse".into()); + queue(store, &input).await; + } + assert!( + claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .is_err(), + "unknown groups cannot silently become unlimited" + ); + let first = claim_next_run(store, &lease, &limits, None) + .await + .expect("first slot") + .expect("available"); + let second = claim_next_run(store, &lease, &limits, None) + .await + .expect("second slot") + .expect("available"); + assert_eq!(first.request.run_id, Uuid::from_u128(1)); + assert_eq!(second.request.run_id, Uuid::from_u128(2)); + assert!( + claim_next_run(store, &lease, &limits, None) + .await + .expect("group full") + .is_none() + ); + finish(store, &lease, &first).await; + limits.insert( + "warehouse".into(), + NonZeroU32::new(1).expect("nonzero lowered limit"), + ); + assert!( + claim_next_run(store, &lease, &limits, None) + .await + .expect("high slot counts after limit reduction") + .is_none() + ); + finish(store, &lease, &second).await; + let third = claim_next_run(store, &lease, &limits, None) + .await + .expect("released group") + .expect("third queued run"); + assert_eq!( + third.request.run_id, + Uuid::from_u128(test_case.expected_unblocked_run), + "{}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_more_than_one_page_of_blocked_runs_when_claiming_then_lower_priority_independent_work_is_not_starved() + { + let test_cases = [ + BacklogClaimTestCase { + description: "SQLite scans past more than a full page of blocked work", + backend: Backend::Sqlite, + expected_independent_run: 105, + }, + BacklogClaimTestCase { + description: "PostgreSQL scans past more than a full page of blocked work", + backend: Backend::Postgres, + expected_independent_run: 105, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let scope = (location, published.snapshot_id); + let mut lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::new(); + queue(store, &request(scope, 1, 0, &["orders"])).await; + claim_next_run(store, &lease, &limits, None) + .await + .expect("claim blocker") + .expect("available"); + for batch in (2..=104).collect::>().chunks(20) { + for id in batch { + queue(store, &request(scope, *id, 0, &["orders"])).await; + } + lease = crate::scheduler::main::renew_lease::renew_lease(store, &lease) + .await + .expect("keep leader live while preparing backlog"); + } + queue(store, &request(scope, 105, -1, &["customers"])).await; + let started = std::time::Instant::now(); + let independent = claim_next_run(store, &lease, &limits, None) + .await + .expect("scan past a full blocked page") + .expect("independent work"); + assert_eq!( + independent.request.run_id, + Uuid::from_u128(test_case.expected_independent_run), + "{}", + test_case.description + ); + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "claim scan exceeded five seconds: {}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_blocked_multi_limit_waiter_when_younger_work_arrives_then_shared_limits_drain_without_blocking_unrelated_work() + { + let test_cases = [ + GroupLimitTestCase { + description: "SQLite protects priority across atomic limit sets", + backend: Backend::Sqlite, + expected_unblocked_run: 4, + }, + GroupLimitTestCase { + description: "PostgreSQL protects priority across atomic limit sets", + backend: Backend::Postgres, + expected_unblocked_run: 4, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let scope = (location, published.snapshot_id); + for (id, priority, nodes, groups) in [ + (1, 100, &["inventory"][..], &["inventory_limit"][..]), + ( + 2, + 10, + &["orders"][..], + &["orders_limit", "inventory_limit"][..], + ), + (3, 1, &["customers"][..], &["orders_limit"][..]), + (4, 0, &["products"][..], &[][..]), + ] { + let mut input = request(scope, id, priority, nodes); + input.concurrency_groups = groups.iter().map(|key| (*key).into()).collect(); + queue(store, &input).await; + } + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([ + ("orders_limit".into(), NonZeroU32::new(1).expect("one")), + ("inventory_limit".into(), NonZeroU32::new(1).expect("one")), + ]); + let blocker = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("highest priority"); + assert_eq!(blocker.request.run_id, Uuid::from_u128(1)); + let unrelated = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("unrelated"); + assert_eq!( + unrelated.request.run_id, + Uuid::from_u128(test_case.expected_unblocked_run), + "{}", + test_case.description + ); + assert!( + claim_next_run(store, &lease, &limits, None) + .await + .expect("wait") + .is_none() + ); + succeed(store, &blocker).await; + let protected = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("older waiter"); + assert_eq!(protected.request.run_id, Uuid::from_u128(2)); + succeed(store, &protected).await; + let younger = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("younger retry"); + assert_eq!(younger.request.run_id, Uuid::from_u128(3)); + succeed(store, &younger).await; + succeed(store, &unrelated).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs new file mode 100644 index 0000000000..ba16619fc2 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs @@ -0,0 +1,114 @@ +use std::collections::BTreeMap; + +use crate::declarations::main::list_triggers::list_triggers; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::models::TickDecision; +use crate::scheduler::tests::contracts::helpers::{acquire_lease, evaluate_trigger}; +use crate::scheduler::tests::contracts::ticks::helpers::calendar_text; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::test_types::CoalescedRunTestCase; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_coalescing_triggers_when_a_previous_run_has_been_claimed_then_new_ticks_wait_in_a_new_run_instead_of_covering_executing_work() + { + let test_cases = [ + CoalescedRunTestCase { + description: "SQLite coalesces only work that has not been claimed", + backend: Backend::Sqlite, + expected_runs: 2, + }, + CoalescedRunTestCase { + description: "PostgreSQL coalesces only work that has not been claimed", + backend: Backend::Postgres, + expected_runs: 2, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), + ); + input.documents.insert("triggers".into(), json!({"daily_orders": {"description": "Refresh orders", "job_key": "orders", "starts_at": calendar_text(0), + "schedule": {"kind": "calendar", "times": [calendar_text(0)], "refresh_at": "2030-01-01T00:00:00Z"}}})); + let published = publish_snapshot(store, &input) + .await + .expect("first calendar publication"); + let trigger = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + evaluate_trigger(store, &lease, trigger) + .await + .expect("first tick"); + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim first run") + .expect("scheduled run"); + assert_eq!(claim.request.snapshot_id, published.snapshot_id); + input.generation = 2; + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = + json!([calendar_text(0), calendar_text(1)]); + publish_snapshot(store, &input) + .await + .expect("refresh calendar"); + let plan = evaluate_trigger(store, &lease, trigger) + .await + .expect("second tick") + .expect("active trigger"); + assert_eq!(plan.actions.len(), 1); + let TickDecision::Requested { run_id: waiting } = plan.actions[0].decision else { + panic!("future work must not be covered by an already claimed run"); + }; + assert_ne!(waiting, claim.request.run_id); + input.generation = 3; + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = + json!([calendar_text(0), calendar_text(1), calendar_text(2)]); + publish_snapshot(store, &input) + .await + .expect("next calendar refresh"); + let plan = evaluate_trigger(store, &lease, trigger) + .await + .expect("third tick") + .expect("active trigger"); + assert_eq!( + plan.actions[0].decision, + TickDecision::Covered { + covered_by_run_id: waiting + } + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("one executing and one waiting") + .len(), + test_case.expected_runs, + "{}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs new file mode 100644 index 0000000000..ddc82ea434 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs @@ -0,0 +1,32 @@ +#[path = "claims/helpers.rs"] +mod claim_helpers; +#[path = "helpers.rs"] +pub(crate) mod helpers; +#[path = "monitor/helpers.rs"] +mod monitor_helpers; +#[path = "termination/helpers.rs"] +pub(crate) mod terminal_helpers; +#[path = "test_checks.rs"] +mod test_checks; +#[path = "test_claims.rs"] +mod test_claims; +#[path = "test_coalesce.rs"] +mod test_coalesce; +#[path = "test_faults.rs"] +mod test_faults; +#[path = "test_group_admission.rs"] +mod test_group_admission; +#[path = "test_groups.rs"] +mod test_groups; +#[path = "test_lifecycle.rs"] +mod test_lifecycle; +#[path = "test_monitor.rs"] +mod test_monitor; +#[path = "test_permissions.rs"] +mod test_permissions; +#[path = "test_requests.rs"] +mod test_requests; +#[path = "test_reruns.rs"] +mod test_reruns; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs new file mode 100644 index 0000000000..8b4107f241 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs @@ -0,0 +1,176 @@ +use std::collections::BTreeMap; +use std::num::NonZeroU32; + +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::heartbeat_run::heartbeat_run; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::test_types::{ClaimFailureTestCase, TransactionFault}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_failures_during_claim_publication_when_retried_then_claims_slots_liveness_facts_and_projection_commit_together() + { + let test_cases = [ + ClaimFailureTestCase { + description: "SQLite rolls back every claim-publication failure window", + backend: Backend::Sqlite, + expected_run: 1, + }, + ClaimFailureTestCase { + description: "PostgreSQL rolls back every claim-publication failure window", + backend: Backend::Postgres, + expected_run: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let mut input = request( + (location, published.snapshot_id), + 1, + 1, + &["customers", "orders"], + ); + input.concurrency_groups.insert("warehouse".into()); + queue(store, &input).await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([( + "warehouse".into(), + NonZeroU32::new(1).expect("positive limit"), + )]); + let facts = read_recall_events(store, 0, 100) + .await + .expect("requested and queued facts"); + let projected = list_runs(store, (location, "development"), 100) + .await + .expect("queued projection"); + for (table, operation) in [ + ("sqb_write_claims", "INSERT"), + ("sqb_concurrency_slots", "INSERT"), + ("recall_events", "INSERT"), + ("sqb_runs", "UPDATE"), + ] { + let point = TransactionFault { + description: "claim publication failure", + table, + operation, + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected claim failure')", + postgres_body: "RAISE EXCEPTION 'injected claim failure'", + }; + install_fault(store, &point).await; + assert!( + claim_next_run(store, &lease, &limits, None).await.is_err(), + "{} at {table}", + point.description + ); + remove_fault(store, &point).await; + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("rolled back claim fact"), + facts + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("rolled back claim projection"), + projected + ); + } + let expiry = TransactionFault { + description: "claim fence expires before commit", + table: "sqb_runs", + operation: "UPDATE", + condition: "WHEN NEW.worker_instance IS NOT NULL AND OLD.worker_instance IS NULL", + sqlite_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + postgres_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + }; + install_fault(store, &expiry).await; + assert!( + matches!( + claim_next_run(store, &lease, &limits, None).await, + Err(ControlError::Fenced) + ), + "{}", + expiry.description + ); + remove_fault(store, &expiry).await; + let claim = claim_next_run(store, &lease, &limits, None) + .await + .expect("retry complete claim") + .expect("no leaked claims or slots"); + assert_eq!(claim.request, input); + assert_eq!( + claim.request.run_id, + Uuid::from_u128(test_case.expected_run), + "{}", + test_case.description + ); + assert!( + claim_next_run(store, &lease, &limits, None) + .await + .expect("already claimed run is not relaunched") + .is_none() + ); + expire_lease(store).await; + let next = acquire_lease(store, Uuid::now_v7()) + .await + .expect("take over") + .expect("new leader"); + let launched = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: input.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Launched { + container_id: Some("synthetic-orders-worker".into()), + }, + }; + assert!( + matches!( + change_run(store, RunAuthority::Scheduler(&lease), &launched).await, + Err(ControlError::Fenced) + ), + "a former leader cannot acknowledge a launch after takeover" + ); + assert!( + change_run(store, RunAuthority::Scheduler(&next), &launched) + .await + .expect("successor reconciles the existing claim") + ); + assert!( + !heartbeat_run(store, (input.run_id, claim.claim_id)) + .await + .expect("workers remain alive through scheduler failover") + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs new file mode 100644 index 0000000000..b900611fa2 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs @@ -0,0 +1,872 @@ +//! Real database admission preserves completion orders, capacities and fair retries across replicas. + +use std::collections::{BTreeMap, BTreeSet}; +use std::num::NonZeroU32; + +use crate::credentials::models::AuditContext; +use crate::runs::main::{ + advance_groups::advance_groups, claim_next_run::claim_next_run, list_runs::list_runs, + request_job_run::request_job_run, +}; +use crate::runs::models::{JobRunRequest, RunDispatch, RunStatus}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::claim_helpers::{slot_counts, succeed}; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::test_types::{ + AdmissionParent, GroupAdmissionTestCase, GroupChildLimitTestCase, GroupQueueTestCase, + MixedAdmissionTestCase, RunLifecycleTestCase, +}; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_mixed_physical_and_parent_waiters_when_admitting_then_priority_and_queue_order_win() { + let test_cases = [ + MixedAdmissionTestCase { + description: "higher-priority physical request precedes parent admission", + physical_priority: 10, + parent_priority: 1, + expected_selections: ["inventory", "orders"], + }, + MixedAdmissionTestCase { + description: "earlier physical request wins at equal priority", + physical_priority: 1, + parent_priority: 1, + expected_selections: ["inventory", "orders"], + }, + MixedAdmissionTestCase { + description: "higher-priority parent precedes physical request", + physical_priority: 1, + parent_priority: 10, + expected_selections: ["orders", "inventory"], + }, + ]; + for test_case in test_cases { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + json!({"stage": {"description": "Build orders", "selection": ["orders"]}}), + ); + input.documents.insert("groups".into(), json!({"daily": {"description": "Group orders", "selection": ["orders"], "concurrency_groups": ["warehouse"], "jobs": {"stage": []}}})); + let published = publish_snapshot(store, &input).await.expect("publish"); + let mut physical = request( + (location, published.snapshot_id), + 900, + test_case.physical_priority, + &["inventory"], + ); + physical.concurrency_groups.insert("warehouse".into()); + queue(store, &physical).await; + request_job_run( + store, + &actor, + &JobRunRequest { + run_id: Uuid::from_u128(100), + location_id: location, + target: "development".into(), + job_key: "daily".into(), + reason: "Exercise mixed queue ordering".into(), + priority: test_case.parent_priority, + window: None, + }, + ) + .await + .expect("queue group"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = + BTreeMap::from([("warehouse".into(), NonZeroU32::new(1).expect("one"))]); + let scope = (location, "development"); + for index in 0..test_case.expected_selections.len() { + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("advance"); + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim") + .expect(test_case.description); + assert_eq!( + claim.request.selection, + BTreeSet::from([test_case.expected_selections[index].into()]), + "{} {backend:?}", + test_case.description + ); + assert_eq!(slot_counts(store).await.get("warehouse"), Some(&1)); + succeed(store, &claim).await; + } + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish parent"); + assert!( + list_runs(store, scope, 10) + .await + .expect("runs") + .iter() + .all(|run| run.status == RunStatus::Success) + ); + assert!(slot_counts(store).await.is_empty()); + close_test_store(fixture).await; + }); + } + } +} + +#[test] +fn given_group_children_sharing_a_limit_of_one_when_claiming_then_they_run_one_at_a_time() { + let test_cases = [ + GroupChildLimitTestCase { + description: "SQLite children share the group's warehouse slot in turn", + backend: Backend::Sqlite, + expected_claimed_together: 1, + expected_jobs: BTreeSet::from(["load_customers", "load_inventory"]), + }, + GroupChildLimitTestCase { + description: "PostgreSQL children share the group's warehouse slot in turn", + backend: Backend::Postgres, + expected_claimed_together: 1, + expected_jobs: BTreeSet::from(["load_customers", "load_inventory"]), + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/group_shared_limit.json"); + let case: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("DuckShop group")) + .expect("group case"); + input.documents.insert("jobs".into(), case["jobs"].clone()); + input + .documents + .insert("groups".into(), case["groups"].clone()); + publish_snapshot(store, &input).await.expect("publish"); + request_job_run( + store, + &actor, + &JobRunRequest { + run_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + job_key: "daily".into(), + reason: "Load the warehouse once".into(), + priority: 1, + window: None, + }, + ) + .await + .expect("queue group"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([("warehouse".to_owned(), NonZeroU32::MIN)]); + let dispatch = RunDispatch::from(Some((location, "development"))); + advance_groups(store, &lease, (&limits, dispatch)) + .await + .expect("admit group"); + let mut first = Vec::new(); + for _ in 0..3 { + first.extend( + claim_next_run(store, &lease, &limits, dispatch) + .await + .expect("claim"), + ); + } + assert_eq!( + first.len(), + test_case.expected_claimed_together, + "{}: one child holds the warehouse slot", + test_case.description + ); + assert_eq!( + slot_counts(store).await, + BTreeMap::from([("warehouse".to_owned(), 1)]) + ); + succeed(store, &first[0]).await; + first.extend( + claim_next_run(store, &lease, &limits, dispatch) + .await + .expect("claim after finish"), + ); + assert_eq!( + first + .iter() + .map(|claim| claim.request.job_key.as_deref().unwrap_or_default()) + .collect::>(), + test_case.expected_jobs, + "{}: the other child runs once the slot is free", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_an_unserved_queued_group_when_dispatching_an_eligible_scope_then_unknown_child_limits_do_not_block_it() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite dispatch excludes unserved prospective demand", + backend: Backend::Sqlite, + expected_status: RunStatus::Success, + }, + RunLifecycleTestCase { + description: "PostgreSQL dispatch excludes unserved prospective demand", + backend: Backend::Postgres, + expected_status: RunStatus::Success, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let unserved = create_location(store, "inventory", "test-admin") + .await + .expect("unserved location"); + let served = create_location(store, "orders", "test-admin") + .await + .expect("served location"); + for location in [unserved, served] { + let mut input = snapshot(location); + input.documents.insert("jobs".into(), json!({"stage": {"description": "Build orders", "selection": ["orders"], "concurrency_groups": if location == unserved {vec!["inventory_limit"]} else {vec![]}}})); + input.documents.insert("groups".into(), json!({"daily": {"description": "Group orders", "selection": ["orders"], "jobs": {"stage": []}}})); + publish_snapshot(store, &input).await.expect("publish"); + request_job_run( + store, + &actor, + &JobRunRequest { + run_id: location, + location_id: location, + target: "development".into(), + job_key: "daily".into(), + reason: "Exercise launcher eligibility".into(), + priority: 1, + window: None, + }, + ) + .await + .expect("queue group"); + } + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::new(); + let eligible = BTreeSet::from([(served, "development".into())]); + let dispatch = RunDispatch { + scope: Some((served, "development")), + eligible: Some(&eligible), + }; + advance_groups(store, &lease, (&limits, dispatch)) + .await + .expect("eligible parent advances"); + let claim = claim_next_run(store, &lease, &limits, dispatch) + .await + .expect("eligible dispatch") + .expect("eligible child"); + assert_eq!(claim.request.location_id, served); + succeed(store, &claim).await; + advance_groups(store, &lease, (&limits, dispatch)) + .await + .expect("finish eligible parent"); + assert!( + list_runs(store, (served, "development"), 10) + .await + .expect("served runs") + .iter() + .all(|run| run.status == test_case.expected_status), + "{}", + test_case.description + ); + assert!( + list_runs(store, (unserved, "development"), 10) + .await + .expect("unserved runs") + .iter() + .all(|run| run.status == RunStatus::Queued) + ); + assert!(slot_counts(store).await.is_empty()); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_capacity_and_multistage_cycles_when_replicas_admit_and_resume_groups_then_every_group_finishes_within_limits() + { + let test_cases = [ + GroupAdmissionTestCase { + description: "capacity two admits both groups with every child limit", + limits: &[("orders_limit", 2), ("inventory_limit", 2)], + parents: &[ + AdmissionParent { + holds: &["orders_limit"], + children: &[&["inventory_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&["orders_limit"]], + }, + ], + expected_running: 2, + }, + GroupAdmissionTestCase { + description: "a group waits while another holds the limit its child needs", + limits: &[("orders_limit", 2), ("inventory_limit", 1)], + parents: &[ + AdmissionParent { + holds: &["orders_limit"], + children: &[&["inventory_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&["orders_limit"]], + }, + ], + expected_running: 1, + }, + GroupAdmissionTestCase { + description: "a three-way cycle admits one group at a time", + limits: &[ + ("orders_limit", 1), + ("inventory_limit", 1), + ("products_limit", 1), + ], + parents: &[ + AdmissionParent { + holds: &["orders_limit"], + children: &[&["inventory_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&["products_limit"]], + }, + AdmissionParent { + holds: &["products_limit"], + children: &[&["orders_limit"]], + }, + ], + expected_running: 1, + }, + GroupAdmissionTestCase { + description: "a later stage's limit is taken before the first stage runs", + limits: &[("orders_limit", 1), ("inventory_limit", 1)], + parents: &[ + AdmissionParent { + holds: &["orders_limit"], + children: &[&[], &["inventory_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&[], &["orders_limit"]], + }, + ], + expected_running: 1, + }, + GroupAdmissionTestCase { + description: "a group holding every child limit excludes its peer until it finishes", + limits: &[ + ("orders_limit", 1), + ("inventory_limit", 1), + ("products_limit", 1), + ], + parents: &[ + AdmissionParent { + holds: &["orders_limit"], + children: &[&["inventory_limit", "products_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&["products_limit"]], + }, + ], + expected_running: 1, + }, + GroupAdmissionTestCase { + description: "a group's child limits are taken with its own limits", + limits: &[ + ("orders_limit", 1), + ("inventory_limit", 1), + ("products_limit", 1), + ("support_limit", 1), + ], + parents: &[ + AdmissionParent { + holds: &["orders_limit", "products_limit"], + children: &[&["inventory_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&["orders_limit", "products_limit"]], + }, + AdmissionParent { + holds: &["support_limit"], + children: &[&["orders_limit"]], + }, + ], + expected_running: 1, + }, + ]; + for test_case in test_cases { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + let mut jobs = serde_json::Map::new(); + let mut groups = serde_json::Map::new(); + let mut child_count = 0; + for (index, parent) in test_case.parents.iter().enumerate() { + let mut members = serde_json::Map::new(); + let mut selection = BTreeSet::new(); + for (stage, needs) in parent.children.iter().enumerate() { + let name = format!("orders_job_{index}_{stage}"); + let node = + ["orders", "customers", "inventory", "products"][child_count % 4]; + child_count += 1; + selection.insert(node); + jobs.insert(name.clone(), json!({"description": "Build a synthetic stage", "selection": [node], "concurrency_groups": needs})); + let after: Vec = (0..stage) + .map(|earlier| format!("orders_job_{index}_{earlier}")) + .collect(); + members.insert(name, json!(after)); + } + groups.insert(format!("orders_group_{index}"), json!({"description": "Complete a synthetic group", "selection": selection, "concurrency_groups": parent.holds, "jobs": members})); + } + input.documents.insert("jobs".into(), json!(jobs)); + input.documents.insert("groups".into(), json!(groups)); + publish_snapshot(store, &input) + .await + .expect("publish groups"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits: BTreeMap = test_case + .limits + .iter() + .map(|(key, count)| { + ( + (*key).into(), + NonZeroU32::new(*count).expect("positive capacity"), + ) + }) + .collect(); + let scope = (location, "development"); + for index in 0..test_case.parents.len() { + request_job_run( + store, + &actor, + &JobRunRequest { + // Higher-priority arrivals contend with groups that already hold their limits. + run_id: Uuid::from_u128(1000 - index as u128), + location_id: location, + target: "development".into(), + job_key: format!("orders_group_{index}"), + reason: "Exercise capacity-aware admission".into(), + priority: index as i64 + 1, + window: None, + }, + ) + .await + .expect("request group"); + let (first, second) = tokio::join!( + advance_groups(store, &lease, (&limits, scope)), + advance_groups(&replica, &lease, (&limits, scope)) + ); + first.expect("first scheduler admits higher-priority arrival"); + second.expect("replica admits only within limits"); + } + for _ in 0..test_case.parents.len() { + let (first, second) = tokio::join!( + advance_groups(store, &lease, (&limits, scope)), + advance_groups(&replica, &lease, (&limits, scope)) + ); + first.expect("first scheduler advance"); + second.expect("second scheduler advance"); + } + let initial = list_runs(store, scope, 100).await.expect("initial runs"); + assert_eq!( + initial + .iter() + .filter(|run| run.status == RunStatus::Running) + .count(), + test_case.expected_running, + "{} {backend:?}", + test_case.description + ); + for iteration in 0..child_count { + let owner = [store, &replica][iteration % 2]; + for _ in 0..test_case.parents.len() { + advance_groups(owner, &lease, (&limits, scope)) + .await + .expect("resume group from durable state"); + } + let claim = claim_next_run(owner, &lease, &limits, Some(scope)) + .await + .expect("claim") + .expect(test_case.description); + // Re-evaluate while a physical child still holds its temporary claims. + advance_groups(&replica, &lease, (&limits, scope)) + .await + .expect("advance during running claim"); + let occupied = slot_counts(store).await; + assert!( + occupied + .iter() + .all(|(key, count)| *count <= i64::from(limits[key].get())), + "{} {occupied:?}", + test_case.description + ); + succeed(owner, &claim).await; + } + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("aggregate final children"); + let finished = list_runs(&replica, scope, 100) + .await + .expect("finished groups"); + assert_eq!(finished.len(), test_case.parents.len() + child_count); + assert!( + finished.iter().all(|run| run.status == RunStatus::Success), + "{} {finished:?}", + test_case.description + ); + assert!( + slot_counts(store).await.is_empty(), + "terminal parents release their own slots" + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } + } +} + +#[test] +fn given_an_older_blocked_allow_group_when_an_independent_generation_is_queued_then_overlap_allow_still_runs() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite preserves overlap allow across generations", + backend: Backend::Sqlite, + expected_status: RunStatus::Running, + }, + RunLifecycleTestCase { + description: "PostgreSQL preserves overlap allow across generations", + backend: Backend::Postgres, + expected_status: RunStatus::Running, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + json!({"stage": {"description": "Build a pinned stage", "selection": ["orders"]}}), + ); + input.documents.insert("groups".into(), json!({"daily": {"description": "Allow independent generations", "selection": ["orders"], "concurrency_groups": ["orders_limit"], "jobs": {"stage": []}}})); + let first = publish_snapshot(store, &input) + .await + .expect("first snapshot"); + let older = JobRunRequest { + run_id: Uuid::from_u128(2000), + location_id: location, + target: "development".into(), + job_key: "daily".into(), + reason: "Older group".into(), + priority: 1, + window: None, + }; + request_job_run(store, &actor, &older) + .await + .expect("older request"); + input.generation = 2; + input.documents.get_mut("jobs").expect("jobs")["stage"]["selection"] = + json!(["inventory"]); + input.documents.get_mut("groups").expect("groups")["daily"]["selection"] = + json!(["inventory"]); + input.documents.get_mut("groups").expect("groups")["daily"]["concurrency_groups"] = + json!(["inventory_limit"]); + publish_snapshot(store, &input) + .await + .expect("second snapshot"); + let newer = JobRunRequest { + run_id: Uuid::from_u128(1000), + reason: "Independent replacement generation".into(), + ..older.clone() + }; + request_job_run(store, &actor, &newer) + .await + .expect("newer request"); + let mut physical = request((location, first.snapshot_id), 3000, 10, &["products"]); + physical.concurrency_groups.insert("orders_limit".into()); + queue(store, &physical).await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([ + ("orders_limit".into(), NonZeroU32::new(1).expect("one")), + ("inventory_limit".into(), NonZeroU32::new(1).expect("one")), + ]); + let scope = (location, "development"); + let blocker = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("physical claim") + .expect("higher priority blocker"); + assert_eq!(blocker.request.run_id, physical.run_id); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("independent group admission"); + let runs = list_runs(store, scope, 100).await.expect("runs"); + assert_eq!( + runs.iter() + .find(|run| run.run_id == older.run_id) + .expect("older") + .status, + RunStatus::Queued + ); + assert_eq!( + runs.iter() + .find(|run| run.run_id == newer.run_id) + .expect("newer") + .status, + test_case.expected_status, + "{}", + test_case.description + ); + let child = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("new child") + .expect("allow generation can progress"); + assert_eq!(child.request.parent_run_id, Some(newer.run_id)); + assert_eq!(child.request.selection, ["inventory".to_owned()].into()); + succeed(store, &child).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish replacement parent"); + succeed(store, &blocker).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("old parent retries after blocker drains"); + let child = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("old child") + .expect("older group makes progress"); + assert_eq!(child.request.parent_run_id, Some(older.run_id)); + succeed(store, &child).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("complete all groups"); + assert!( + list_runs(store, scope, 100) + .await + .expect("finished") + .iter() + .all(|run| run.status == RunStatus::Success) + ); + assert!(slot_counts(store).await.is_empty()); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_large_healthy_project_with_blocked_groups_when_claiming_then_unrelated_work_has_a_bounded_write_lock() + { + let test_cases = [ + GroupQueueTestCase { + description: "SQLite scans a full blocked group page without expanding unrelated bodies", + backend: Backend::Sqlite, + group_count: 105, + node_count: 3000, + expected_max_claim_millis: 5000, + }, + GroupQueueTestCase { + description: "PostgreSQL scans a full blocked group page without expanding unrelated bodies", + backend: Backend::Postgres, + group_count: 105, + node_count: 3000, + expected_max_claim_millis: 5000, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + let template = input.nodes["orders"].clone(); + for index in 0..test_case.node_count { + let mut node = template.clone(); + node.definition["description"] = + json!(format!("Orders {index} {}", "details ".repeat(128))); + input.nodes.insert(format!("orders_{index}"), node); + } + let jobs = (0..test_case.group_count).map(|index| (format!("job_{index}"), json!({"description": "Build orders", "selection": ["orders"], "concurrency_groups": ["warehouse"]}))).collect::>(); + let groups = (0..test_case.group_count).map(|index| (format!("group_{index}"), json!({"description": "Independent order group", "selection": ["orders"], "jobs": {format!("job_{index}"): []}}))).collect::>(); + input.documents.insert("jobs".into(), json!(jobs)); + input.documents.insert("groups".into(), json!(groups)); + input.documents.insert( + "columns".into(), + json!({"orders": "unused display metadata ".repeat(100000)}), + ); + let published = publish_snapshot(store, &input) + .await + .expect("publish large project"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([("warehouse".into(), NonZeroU32::new(1).expect("one"))]); + let scope = (location, "development"); + let mut physical = request((location, published.snapshot_id), 1, 10, &["products"]); + physical.concurrency_groups.insert("warehouse".into()); + queue(store, &physical).await; + let blocker = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim") + .expect("blocker"); + for index in 0..test_case.group_count { + request_job_run( + store, + &actor, + &JobRunRequest { + run_id: Uuid::from_u128(1000 + index as u128), + location_id: location, + target: "development".into(), + job_key: format!("group_{index}"), + reason: "Exercise blocked group queue".into(), + priority: 1, + window: None, + }, + ) + .await + .expect("request group"); + } + let started = std::time::Instant::now(); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("admit independent parents"); + eprintln!( + "{:?}: {} parent admissions in {:?}", + test_case.backend, + test_case.group_count, + started.elapsed() + ); + let unrelated = request((location, published.snapshot_id), 2, -1, &["customers"]); + queue(store, &unrelated).await; + let started = std::time::Instant::now(); + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("scan blocked children") + .expect("unrelated work"); + let elapsed = started.elapsed(); + eprintln!( + "{:?}: {} groups / {} nodes, claim transaction {:?}", + test_case.backend, test_case.group_count, test_case.node_count, elapsed + ); + assert_eq!( + claim.request.run_id, unrelated.run_id, + "{}", + test_case.description + ); + assert!( + elapsed.as_millis() < test_case.expected_max_claim_millis, + "{}: {elapsed:?}", + test_case.description + ); + succeed(store, &claim).await; + succeed(store, &blocker).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs new file mode 100644 index 0000000000..10f1b53bf8 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs @@ -0,0 +1,424 @@ +//! Pinned groups coordinate real transactions, inherited slots, cancellation and replay. + +use std::collections::BTreeMap; +use std::num::NonZeroU32; + +use crate::credentials::models::AuditContext; +use crate::runs::main::{ + advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, + list_runs::list_runs, request_job_run::request_job_run, run_execution::run_execution, +}; +use crate::runs::models::{ + JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::claim_helpers::{slot_counts, succeed}; +use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_pinned_group_when_redeployed_and_cancelled_then_children_keep_scope_and_parent_waits_for_terminal_evidence() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite group cancellation and replay", + backend: Backend::Sqlite, + expected_status: RunStatus::Cancelled, + }, + RunLifecycleTestCase { + description: "PostgreSQL group cancellation and replay", + backend: Backend::Postgres, + expected_status: RunStatus::Cancelled, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input.nodes.get_mut("orders").expect("orders").definition["deps"] = + json!(["customers"]); + input.documents.insert("jobs".into(), json!({ + "staging": {"description": "Stage customers", "selection": ["customers"], "concurrency_groups": ["warehouse"]}, + "marts": {"description": "Build orders", "selection": ["orders"], "concurrency_groups": ["warehouse"]} + })); + input.documents.insert("groups".into(), json!({"daily_orders": { + "description": "Stage then build orders", "selection": ["customers", "orders"], + "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["staging"]} + }})); + let published = publish_snapshot(store, &input) + .await + .expect("publish group"); + let request = JobRunRequest { + run_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + job_key: "daily_orders".into(), + reason: "Build the group".into(), + priority: 1, + window: None, + }; + assert!( + request_job_run(store, &context, &request) + .await + .expect("request group") + ); + assert!( + !request_job_run(store, &context, &request) + .await + .expect("retry group") + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = + BTreeMap::from([("warehouse".into(), NonZeroU32::new(1).expect("one slot"))]); + let scope = (location, "development"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("start virtual parent"); + let first = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim staging") + .expect("staging ready"); + assert_eq!(first.request.job_key.as_deref(), Some("staging")); + assert_eq!(first.request.parent_run_id, Some(request.run_id)); + assert_eq!(first.request.write_set, ["customers".to_owned()].into()); + assert!( + claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("dependency waits") + .is_none() + ); + let authority = RunAuthority::Worker { + run_id: first.request.run_id, + claim_id: first.claim_id, + }; + for change in [ + RunChange::Started, + RunChange::NodeFinished { + node_key: "customers".into(), + outcome: NodeOutcome::Success, + rows_affected: Some(11), + error_code: None, + skipped_because_of: None, + }, + RunChange::Finished { + outcome: RunOutcome::Success, + reason: "staging completed".into(), + exit_code: Some(0), + }, + ] { + change_run( + store, + authority, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: first.request.run_id, + claim_id: Some(first.claim_id), + change, + }, + ) + .await + .expect("staging fact"); + } + input.generation = 2; + input.documents.get_mut("jobs").expect("jobs")["marts"]["selection"] = + json!(["inventory"]); + input.documents.get_mut("groups").expect("groups")["daily_orders"]["selection"] = + json!(["customers", "inventory"]); + publish_snapshot(store, &input) + .await + .expect("redeploy different membership"); + let second = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim marts") + .expect("marts ready"); + assert_eq!(second.request.snapshot_id, published.snapshot_id); + assert_eq!(second.request.selection, ["orders".to_owned()].into()); + assert_eq!(second.request.write_set, second.request.selection); + assert!( + run_execution(store, (second.request.run_id, second.claim_id)) + .await + .expect("execution") + .expect("pinned") + .skipped_nodes + .is_empty() + ); + let authority = RunAuthority::Worker { + run_id: second.request.run_id, + claim_id: second.claim_id, + }; + change_run( + store, + authority, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: second.request.run_id, + claim_id: Some(second.claim_id), + change: RunChange::Started, + }, + ) + .await + .expect("start marts"); + change_run( + store, + RunAuthority::Operator(&context), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: request.run_id, + claim_id: None, + change: RunChange::CancelRequested { + actor: context.actor.clone(), + reason: "Cancel the group".into(), + }, + }, + ) + .await + .expect("cancel parent"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("propagate cancellation"); + let runs = list_runs(store, scope, 100).await.expect("runs"); + assert_eq!( + runs.iter() + .find(|run| run.run_id == request.run_id) + .expect("parent") + .status, + RunStatus::Running, + "{}", + test_case.description + ); + assert_eq!( + runs.iter() + .find(|run| run.run_id == second.request.run_id) + .expect("active child") + .status, + RunStatus::Running + ); + let standalone = JobRunRequest { + run_id: Uuid::now_v7(), + job_key: "staging".into(), + ..request.clone() + }; + request_job_run(store, &context, &standalone) + .await + .expect("standalone request"); + assert!( + claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("parent retains its slot") + .is_none() + ); + change_run( + store, + authority, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: second.request.run_id, + claim_id: Some(second.claim_id), + change: RunChange::Finished { + outcome: RunOutcome::Cancelled, + reason: "worker confirmed its executor stopped".into(), + exit_code: Some(143), + }, + }, + ) + .await + .expect("physical child finished"); + let fault = TransactionFault { + description: "group terminal publication rolls back", + table: "recall_events", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected group publication failure')", + postgres_body: "RAISE EXCEPTION 'injected group publication failure'", + }; + install_fault(store, &fault).await; + assert!( + advance_groups(store, &lease, (&limits, scope)) + .await + .is_err() + ); + remove_fault(store, &fault).await; + for _ in 0..2 { + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("retry group completion"); + } + let runs = list_runs(store, scope, 100).await.expect("completed runs"); + assert_eq!( + runs.iter() + .find(|run| run.run_id == request.run_id) + .expect("parent") + .status, + test_case.expected_status + ); + let standalone_claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("slot released") + .expect("standalone staging ready"); + assert_eq!(standalone_claim.request.run_id, standalone.run_id); + assert_eq!(standalone_claim.request.parent_run_id, None); + let events = read_recall_events(store, 0, 1000).await.expect("facts"); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_terminated" + && event["invocation_id"] == format!("scheduler/{}", request.run_id)) + .count(), + 1 + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_opposing_parent_and_child_limits_when_advancing_groups_then_both_groups_finish_without_exceeding_limits() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite opposing group reservations", + backend: Backend::Sqlite, + expected_status: RunStatus::Running, + }, + RunLifecycleTestCase { + description: "PostgreSQL opposing group reservations", + backend: Backend::Postgres, + expected_status: RunStatus::Running, + }, + ]; + let mut completed = Vec::new(); + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input.documents.insert("jobs".into(), json!({ + "orders_job": {"description": "Build orders", "selection": ["orders"], "concurrency_groups": ["inventory_limit"]}, + "inventory_job": {"description": "Build inventory", "selection": ["customers"], "concurrency_groups": ["orders_limit"]} + })); + input.documents.insert("groups".into(), json!({ + "orders_group": {"description": "Orders group", "selection": ["orders"], "concurrency_groups": ["orders_limit"], "jobs": {"orders_job": []}}, + "inventory_group": {"description": "Inventory group", "selection": ["customers"], "concurrency_groups": ["inventory_limit"], "jobs": {"inventory_job": []}} + })); + publish_snapshot(store, &input) + .await + .expect("publish groups"); + for job_key in ["orders_group", "inventory_group"] { + request_job_run( + store, + &context, + &JobRunRequest { + run_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + job_key: job_key.into(), + reason: "Build independent groups".into(), + priority: 1, + window: None, + }, + ) + .await + .expect("request group"); + } + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([ + ("orders_limit".into(), NonZeroU32::new(1).expect("one slot")), + ( + "inventory_limit".into(), + NonZeroU32::new(1).expect("one slot"), + ), + ]); + let scope = (location, "development"); + for _ in 0..2 { + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("advance groups"); + } + let runs = list_runs(store, scope, 100).await.expect("runs"); + assert_eq!(runs.len(), 4, "{}", test_case.description); + assert_eq!( + runs.iter() + .filter(|run| run.status == test_case.expected_status) + .count(), + 1, + "{}", + test_case.description + ); + for _ in 0..2 { + let child = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim child") + .expect("a group must make progress"); + let slots = slot_counts(store).await; + assert_eq!( + slots.len(), + 2, + "child-only slot is acquired only for its worker" + ); + assert!(slots.values().all(|occupied| *occupied == 1), "{slots:?}"); + assert!( + claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("held child") + .is_none() + ); + succeed(store, &child).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("complete parent and retry waiting group"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("retry independent of run-id ordering"); + } + let runs = list_runs(store, scope, 100).await.expect("completed runs"); + assert!( + runs.iter().all(|run| run.status == RunStatus::Success), + "{runs:?}" + ); + completed.push(test_case.description); + close_test_store(fixture).await; + }); + } + assert_eq!(completed.len(), 2, "both backends must run"); +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs new file mode 100644 index 0000000000..f032ef356a --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -0,0 +1,724 @@ +use std::collections::BTreeMap; + +use crate::credentials::models::AuditContext; +use crate::platform::models::PlatformEvent; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::heartbeat_run::heartbeat_run; +use crate::runs::main::list_run_nodes::list_run_nodes; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{ + NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_facts_outcomes_and_claims_are_atomic() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite preserves facts, outcomes and claims through worker failures", + backend: Backend::Sqlite, + expected_status: RunStatus::Partial, + }, + RunLifecycleTestCase { + description: "PostgreSQL preserves facts, outcomes and claims through worker failures", + backend: Backend::Postgres, + expected_status: RunStatus::Partial, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let scope = (location, published.snapshot_id); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + queue(store, &request(scope, 1, 2, &["customers", "orders"])).await; + queue(store, &request(scope, 2, 1, &["customers", "orders"])).await; + let limits = BTreeMap::new(); + let claim = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("available"); + let worker = RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }; + let started = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Started, + }; + let other_worker = RunAuthority::Worker { + run_id: Uuid::from_u128(2), + claim_id: Uuid::now_v7(), + }; + let operator = AuditContext { + actor: "test-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + assert!(change_run(store, other_worker, &started).await.is_err()); + assert!( + change_run(store, RunAuthority::Operator(&operator), &started) + .await + .is_err() + ); + assert!( + change_run(store, worker, &started) + .await + .expect("worker can start before launcher acknowledgement") + ); + assert!( + !change_run(store, worker, &started) + .await + .expect("identical retry") + ); + let launched = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::Launched { + container_id: Some("synthetic-orders-worker".into()), + }, + ..started.clone() + }; + change_run(store, RunAuthority::Scheduler(&lease), &launched) + .await + .expect("late launcher acknowledgement"); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("runs") + .iter() + .find(|run| run.run_id == claim.request.run_id) + .expect("started run") + .status, + RunStatus::Running + ); + let launch_failed = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::LaunchFailed { + code: "docker_launch_failed".into(), + }, + ..started.clone() + }; + assert!(change_run(store, worker, &launch_failed).await.is_err()); + assert!( + change_run(store, RunAuthority::Operator(&operator), &launch_failed) + .await + .is_err() + ); + let wrong_claim = RunChangeRequest { + claim_id: Some(Uuid::now_v7()), + ..launch_failed.clone() + }; + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &wrong_claim) + .await + .is_err() + ); + let before_launch_failure = read_recall_events(store, 0, 100) + .await + .expect("before launch failure"); + let fault = TransactionFault { + description: "platform launch observation must commit with its run fact", + table: "recall_events", + operation: "INSERT", + condition: "WHEN NEW.event_type='platform_launch_failed'", + sqlite_body: "SELECT RAISE(ABORT,'injected launch observation failure')", + postgres_body: "RAISE EXCEPTION 'injected launch observation failure'", + }; + install_fault(store, &fault).await; + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &launch_failed) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("rolled back launch fact"), + before_launch_failure + ); + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &launch_failed) + .await + .expect("launch observation") + ); + assert!( + !change_run(store, RunAuthority::Scheduler(&lease), &launch_failed) + .await + .expect("idempotent retry") + ); + let observations: Vec<_> = read_recall_events(store, 0, 100) + .await + .expect("health facts") + .into_iter() + .filter(|row| row["event_type"] == "platform_launch_failed") + .collect(); + assert_eq!(observations.len(), 1); + let platform: PlatformEvent = + serde_json::from_value(observations[0]["payload"]["platform"].clone()) + .expect("typed launch failure"); + assert_eq!( + platform, + PlatformEvent::LaunchFailed { + run_id: started.run_id, + claim_id: claim.claim_id, + code: "docker_launch_failed".into() + } + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("live run") + .into_iter() + .find(|run| run.run_id == started.run_id) + .expect("original run") + .status, + RunStatus::Running + ); + assert!( + claim_next_run(store, &lease, &limits, None) + .await + .expect("physical ownership retained") + .is_none() + ); + let success = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::Finished { + outcome: RunOutcome::Success, + reason: "worker complete".into(), + exit_code: Some(0), + }, + ..started.clone() + }; + assert!( + change_run(store, worker, &success).await.is_err(), + "unfinished nodes cannot be marked successful" + ); + for (node, outcome) in [ + ("customers", NodeOutcome::Success), + ("orders", NodeOutcome::Failed), + ] { + let node_start = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::NodeStarted { + node_key: node.into(), + }, + ..started.clone() + }; + change_run(store, worker, &node_start) + .await + .expect("node started"); + let completed = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::NodeFinished { + node_key: node.into(), + outcome, + rows_affected: Some(17), + error_code: (outcome == NodeOutcome::Failed).then(|| "E100".into()), + skipped_because_of: None, + }, + ..started.clone() + }; + let before_nodes = list_run_nodes(store, started.run_id) + .await + .expect("node before fault"); + let before_events = read_recall_events(store, 0, 100) + .await + .expect("events before node fault"); + for table in ["sqb_run_nodes", "sqb_runs"] { + let point = TransactionFault { + description: "node projection and counter failure", + table, + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected node failure')", + postgres_body: "RAISE EXCEPTION 'injected node failure'", + }; + install_fault(store, &point).await; + assert!( + change_run(store, worker, &completed).await.is_err(), + "{} at {table}", + point.description + ); + remove_fault(store, &point).await; + assert_eq!( + list_run_nodes(store, started.run_id) + .await + .expect("node rollback"), + before_nodes + ); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("node fact rollback"), + before_events + ); + } + assert!( + change_run(store, worker, &completed) + .await + .expect("node completed") + ); + assert!( + !change_run(store, worker, &completed) + .await + .expect("node report retry") + ); + } + assert!( + change_run(store, worker, &success).await.is_err(), + "mixed node outcomes are partial, not success" + ); + let partial = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::Finished { + outcome: RunOutcome::Partial, + reason: "independent customer work completed".into(), + exit_code: Some(1), + }, + ..started.clone() + }; + let before = read_recall_events(store, 0, 100) + .await + .expect("before terminal"); + for (table, operation) in [ + ("recall_events", "INSERT"), + ("sqb_runs", "UPDATE"), + ("sqb_write_claims", "DELETE"), + ] { + let point = TransactionFault { + description: "terminal transaction failure", + table, + operation, + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected terminal failure')", + postgres_body: "RAISE EXCEPTION 'injected terminal failure'", + }; + install_fault(store, &point).await; + assert!( + change_run(store, worker, &partial).await.is_err(), + "{} at {table} must roll back the complete transition", + point.description + ); + remove_fault(store, &point).await; + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("rolled back event"), + before + ); + assert!( + claim_next_run(store, &lease, &limits, None) + .await + .expect("claims retained") + .is_none() + ); + } + let unresponsive = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::Unresponsive, + ..started.clone() + }; + change_run(store, RunAuthority::Scheduler(&lease), &unresponsive) + .await + .expect("monitor records loss of heartbeat"); + assert!( + !heartbeat_run(store, (started.run_id, claim.claim_id)) + .await + .expect("heartbeat recovers the worker") + ); + assert!( + heartbeat_run(store, (started.run_id, Uuid::now_v7())) + .await + .is_err(), + "another claim cannot report this run alive" + ); + let cancel = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::CancelRequested { + actor: "test-operator".into(), + reason: "Stop the orders build".into(), + }, + ..started.clone() + }; + assert!( + change_run(store, worker, &cancel).await.is_err(), + "workers cannot impersonate operators" + ); + change_run(store, RunAuthority::Operator(&operator), &cancel) + .await + .expect("record cancellation"); + assert!( + heartbeat_run(store, (started.run_id, claim.claim_id)) + .await + .expect("heartbeat reports pending cancellation") + ); + assert!( + claim_next_run(store, &lease, &limits, None) + .await + .expect("cancel request does not release an executing run's claims") + .is_none() + ); + assert!( + change_run(store, worker, &partial) + .await + .expect("terminal report atomically releases claims") + ); + assert!( + !change_run(store, worker, &partial) + .await + .expect("terminal retry after liveness cleanup") + ); + let mut changed = partial.clone(); + changed.change = RunChange::Finished { + outcome: RunOutcome::Success, + reason: "changed retry".into(), + exit_code: Some(0), + }; + assert!( + change_run(store, worker, &changed).await.is_err(), + "same event ID cannot change its content" + ); + assert!( + heartbeat_run(store, (started.run_id, claim.claim_id)) + .await + .is_err(), + "terminal runs cannot be revived by a late heartbeat" + ); + let projected = list_runs(store, (location, "development"), 100) + .await + .expect("terminal projection") + .into_iter() + .find(|run| run.run_id == started.run_id) + .expect("original run"); + assert_eq!( + projected.status, test_case.expected_status, + "{}", + test_case.description + ); + assert_eq!( + projected.node_counts_json, + BTreeMap::from([("success".into(), 1), ("failed".into(), 1)]) + ); + let nodes = list_run_nodes(store, started.run_id) + .await + .expect("node evidence"); + assert_eq!(nodes[0].node_key, "customers"); + assert_eq!(nodes[0].status, "success"); + assert_eq!(nodes[0].rows_affected, Some(17)); + assert_eq!(nodes[1].error_code.as_deref(), Some("E100")); + let late_failure = RunChangeRequest { + event_id: Uuid::now_v7(), + ..launch_failed.clone() + }; + let before_late = read_recall_events(store, 0, 100) + .await + .expect("terminal facts"); + let expiry = TransactionFault { + description: "late launch observation loses leadership before commit", + sqlite_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + postgres_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ..fault + }; + for (point, fenced) in [(&fault, false), (&expiry, true)] { + install_fault(store, point).await; + let failed = + change_run(store, RunAuthority::Scheduler(&lease), &late_failure).await; + let error = failed.expect_err(point.description); + assert_eq!( + matches!(error, ControlError::Fenced), + fenced, + "{}", + point.description + ); + remove_fault(store, point).await; + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("terminal rollback"), + before_late + ); + } + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &late_failure) + .await + .expect("late observation") + ); + assert!( + !change_run(store, RunAuthority::Scheduler(&lease), &late_failure) + .await + .expect("late retry") + ); + assert!(change_run(store, worker, &late_failure).await.is_err()); + assert!( + change_run(store, RunAuthority::Operator(&operator), &late_failure) + .await + .is_err() + ); + for invalid in [ + RunChangeRequest { + event_id: Uuid::now_v7(), + claim_id: Some(Uuid::now_v7()), + ..late_failure.clone() + }, + RunChangeRequest { + event_id: Uuid::now_v7(), + claim_id: None, + ..late_failure.clone() + }, + RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: Uuid::from_u128(2), + ..late_failure.clone() + }, + RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: Uuid::from_u128(2), + claim_id: Some(Uuid::new_v5(&Uuid::from_u128(2), b"run_claimed")), + ..late_failure.clone() + }, + RunChangeRequest { + event_id: Uuid::now_v7(), + ..started.clone() + }, + RunChangeRequest { + event_id: Uuid::now_v7(), + ..launched.clone() + }, + ] { + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &invalid) + .await + .is_err(), + "terminal authority remains narrow: {invalid:?}" + ); + } + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("unchanged terminal projection") + .into_iter() + .find(|run| run.run_id == started.run_id) + .expect("original run"), + projected + ); + let recorded = read_recall_events(store, 0, 100) + .await + .expect("late fact pair"); + assert_eq!( + recorded + .iter() + .filter(|row| row["event_type"] == "platform_launch_failed") + .count(), + 2 + ); + expire_lease(store).await; + assert!( + matches!( + change_run(store, RunAuthority::Scheduler(&lease), &late_failure).await, + Err(ControlError::Fenced) + ), + "even duplicate observations require a live leader" + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect successor") + .expect("successor"); + let following = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim following run") + .expect("released capacity"); + assert_eq!(following.request.run_id, Uuid::from_u128(2)); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_unclaimed_work_when_an_operator_cancels_then_a_terminal_fact_prevents_launch_and_preserves_the_actor() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite cancellation before launch preserves the operator and terminal fact", + backend: Backend::Sqlite, + expected_status: RunStatus::Cancelled, + }, + RunLifecycleTestCase { + description: "PostgreSQL cancellation before launch preserves the operator and terminal fact", + backend: Backend::Postgres, + expected_status: RunStatus::Cancelled, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + queue( + store, + &request((location, published.snapshot_id), 1, 0, &["orders"]), + ) + .await; + let request = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: Uuid::from_u128(1), + claim_id: None, + change: RunChange::CancelRequested { + actor: "test-operator".into(), + reason: "Cancel queued orders".into(), + }, + }; + let operator = AuditContext { + actor: "test-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let other = AuditContext { + actor: "another-operator".into(), + ..operator.clone() + }; + assert!( + change_run(store, RunAuthority::Operator(&other), &request) + .await + .is_err() + ); + let before_events = read_recall_events(store, 0, 100) + .await + .expect("facts before audit failure"); + let point = TransactionFault { + description: "cancellation audit failure rolls back terminal facts and projections", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected cancellation audit failure')", + postgres_body: "RAISE EXCEPTION 'injected cancellation audit failure'", + }; + install_fault(store, &point).await; + assert!( + change_run(store, RunAuthority::Operator(&operator), &request) + .await + .is_err(), + "{}", + point.description + ); + remove_fault(store, &point).await; + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("cancel rollback") + .remove(0) + .status, + RunStatus::Queued + ); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("fact rollback"), + before_events + ); + assert!( + change_run(store, RunAuthority::Operator(&operator), &request) + .await + .expect("cancel before launch") + ); + assert!( + !change_run(store, RunAuthority::Operator(&operator), &request) + .await + .expect("retry cancellation") + ); + let mut changed = request.clone(); + changed.change = RunChange::CancelRequested { + actor: operator.actor.clone(), + reason: "Different cancellation reason".into(), + }; + assert!( + change_run(store, RunAuthority::Operator(&operator), &changed) + .await + .is_err(), + "retry identity includes the cancellation reason" + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + assert!( + claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("cancelled work cannot launch") + .is_none() + ); + let projected = list_runs(store, (location, "development"), 100) + .await + .expect("cancelled projection") + .remove(0); + assert_eq!( + projected.status, test_case.expected_status, + "{}", + test_case.description + ); + assert_eq!( + projected.node_counts_json, + BTreeMap::from([("cancelled".into(), 1)]) + ); + let events = read_recall_events(store, 0, 100) + .await + .expect("immutable cancellation facts"); + assert_eq!( + events + .iter() + .map(|row| row["event_type"].as_str().expect("type")) + .collect::>(), + [ + "run_requested", + "run_queued", + "run_cancel_requested", + "run_terminated" + ] + ); + assert_eq!( + events[2]["payload"]["change"]["change"]["actor"], + "test-operator" + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs new file mode 100644 index 0000000000..62fe549bc8 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs @@ -0,0 +1,342 @@ +//! Real scheduler cycles detect silent workers without surrendering their physical write claims. + +use std::collections::BTreeMap; +use std::time::Duration; + +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::heartbeat_run::heartbeat_run; +use crate::runs::main::list_runs::list_runs; +use crate::runs::main::monitor_runs::monitor_runs; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus}; +use crate::scheduler::main::renew_lease::renew_lease; +use crate::scheduler::models::SchedulerState; +use crate::scheduler::tests::contracts::helpers::{acquire_lease, scheduler_cycle}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::monitor_helpers::{age_heartbeat, start_run}; +use crate::runs::tests::test_types::{ + MonitorBacklogTestCase, RunMonitorTestCase, TransactionFault, +}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_silent_and_live_workers_when_the_scheduler_monitors_then_fenced_atomic_facts_retain_claims_and_allow_recovery() + { + let test_cases = [ + RunMonitorTestCase { + description: "SQLite observes only running workers with expired heartbeats", + backend: Backend::Sqlite, + expected_statuses: [ + RunStatus::Unresponsive, + RunStatus::Running, + RunStatus::Launched, + RunStatus::Failed, + RunStatus::Queued, + ], + }, + RunMonitorTestCase { + description: "PostgreSQL observes only running workers with expired heartbeats", + backend: Backend::Postgres, + expected_statuses: [ + RunStatus::Unresponsive, + RunStatus::Running, + RunStatus::Launched, + RunStatus::Failed, + RunStatus::Queued, + ], + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let scope = (location, published.snapshot_id); + let holder = Uuid::now_v7(); + let lease = acquire_lease(store, holder) + .await + .expect("elect") + .expect("leader"); + let silent = start_run(store, &lease, &request(scope, 1, 0, &["orders"])).await; + start_run(store, &lease, &request(scope, 2, 0, &["customers"])).await; + age_heartbeat(store, Uuid::from_u128(2), 40).await; + queue(store, &request(scope, 3, 0, &["inventory"])).await; + let launching = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim") + .expect("launching"); + change_run( + store, + RunAuthority::Scheduler(&lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: launching.request.run_id, + claim_id: Some(launching.claim_id), + change: RunChange::Launched { + container_id: Some("orders-worker".into()), + }, + }, + ) + .await + .expect("launcher acknowledges"); + let failed = start_run(store, &lease, &request(scope, 4, 0, &["products"])).await; + change_run( + store, + RunAuthority::Worker { + run_id: failed.request.run_id, + claim_id: failed.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: failed.request.run_id, + claim_id: Some(failed.claim_id), + change: RunChange::Finished { + outcome: RunOutcome::Failed, + reason: "worker failed".into(), + exit_code: Some(1), + }, + }, + ) + .await + .expect("terminal run"); + queue(store, &request(scope, 5, 0, &["orders"])).await; + age_heartbeat(store, silent.request.run_id, 50).await; + age_heartbeat(store, launching.request.run_id, 50).await; + let before = read_recall_events(store, 0, 100) + .await + .expect("before monitoring"); + let projection = list_runs(store, (location, "development"), 100) + .await + .expect("before monitoring"); + let fault = TransactionFault { + description: "monitor projection failure rolls back the observation", + table: "sqb_runs", + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected monitor failure')", + postgres_body: "RAISE EXCEPTION 'injected monitor failure'", + }; + install_fault(store, &fault).await; + assert!( + monitor_runs(store, &lease).await.is_err(), + "{}", + test_case.description + ); + remove_fault(store, &fault).await; + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("fact rollback"), + before + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("projection rollback"), + projection + ); + expire_lease(store).await; + let (state, observed) = scheduler_cycle(store, SchedulerState::new(holder)) + .await + .expect("monitoring cycle"); + assert_eq!(observed.unresponsive, 1, "{}", test_case.description); + let statuses: BTreeMap<_, _> = list_runs(store, (location, "development"), 100) + .await + .expect("monitored statuses") + .into_iter() + .map(|record| (record.run_id, record.status)) + .collect(); + assert_eq!( + std::array::from_fn(|index| statuses[&Uuid::from_u128(index as u128 + 1)]), + test_case.expected_statuses, + "{}", + test_case.description + ); + let recorded = read_recall_events(store, 0, 100) + .await + .expect("observation"); + assert_eq!(recorded.len(), before.len() + 1); + assert_eq!( + recorded.last().expect("fact")["event_type"], + "run_unresponsive" + ); + let (state, repeated) = scheduler_cycle(store, state) + .await + .expect("repeated monitor"); + assert_eq!(repeated.unresponsive, 0); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("no repeated facts"), + recorded + ); + let (_, follower) = scheduler_cycle(store, SchedulerState::new(Uuid::now_v7())) + .await + .expect("follower"); + assert!(!follower.leader); + assert_eq!(follower.unresponsive, 0); + assert!( + !heartbeat_run(store, (silent.request.run_id, silent.claim_id)) + .await + .expect("worker recovers") + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("recovered projection") + .into_iter() + .find(|record| record.run_id == silent.request.run_id) + .expect("recovered run") + .status, + RunStatus::Running + ); + age_heartbeat(store, silent.request.run_id, 50).await; + let (_, second_loss) = scheduler_cycle(store, state) + .await + .expect("new heartbeat loss"); + assert_eq!(second_loss.unresponsive, 1); + let events = read_recall_events(store, 0, 100) + .await + .expect("successive losses"); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_unresponsive") + .count(), + 2 + ); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_recovered") + .count(), + 1 + ); + expire_lease(store).await; + assert!(matches!( + monitor_runs(store, &lease).await, + Err(ControlError::Fenced) + )); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("fenced monitor"), + events + ); + let successor = acquire_lease(store, Uuid::now_v7()) + .await + .expect("takeover") + .expect("successor"); + assert!( + claim_next_run(store, &successor, &BTreeMap::new(), None) + .await + .expect("write claims remain held after heartbeat loss and takeover") + .is_none() + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_silent_worker_backlog_when_monitoring_then_bounded_transactions_continue_immediately_until_drained() + { + let test_cases = [ + MonitorBacklogTestCase { + description: "SQLite drains more than one monitor page", + backend: Backend::Sqlite, + run_count: 105, + expected_batches: [100, 5, 0], + }, + MonitorBacklogTestCase { + description: "PostgreSQL drains more than one monitor page", + backend: Backend::Postgres, + run_count: 105, + expected_batches: [100, 5, 0], + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let holder = Uuid::now_v7(); + let mut lease = acquire_lease(store, holder) + .await + .expect("elect") + .expect("leader"); + for index in 0..test_case.run_count { + let mut input = request( + (location, published.snapshot_id), + index as u128 + 1, + 0, + &["orders"], + ); + input.write_set.clear(); + start_run(store, &lease, &input).await; + age_heartbeat(store, input.run_id, 50).await; + lease = renew_lease(store, &lease) + .await + .expect("renew during setup"); + } + expire_lease(store).await; + let mut state = SchedulerState::new(holder); + let mut batches = Vec::new(); + for expected in test_case.expected_batches { + let cycle; + (state, cycle) = scheduler_cycle(store, state).await.expect("monitor batch"); + batches.push(cycle.unresponsive); + assert_eq!( + cycle.wait_for, + if expected == 100 { + Duration::ZERO + } else { + Duration::from_secs(5) + } + ); + } + assert_eq!( + batches, test_case.expected_batches, + "{}", + test_case.description + ); + let events = read_recall_events(store, 0, 1000) + .await + .expect("monitor facts"); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_unresponsive") + .count(), + test_case.run_count + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs b/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs new file mode 100644 index 0000000000..131c80a45a --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs @@ -0,0 +1,405 @@ +//! Real session, role, immutable-job and database paths enforce sensitive-run permissions. + +use serde_json::json; +use uuid::Uuid; + +use crate::authorization::main::{ + assign_builtin_role::assign_builtin_role, bootstrap_user::bootstrap_user, + change_administrator::change_administrator, +}; +use crate::credentials::main::{ + identities::create_principal::create_principal, sessions::login_password::login_password, + sessions::set_password::set_password, +}; +use crate::credentials::models::{ + AdministratorChange, AuditContext, BuiltinRole, DefaultRole, PasswordAuthority, PasswordLogin, + RoleAssignmentRequest, +}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::main::{ + advance_groups::advance_groups, cancel_person_run::cancel_person_run, change_run::change_run, + list_person_runs::list_person_runs, list_runs::list_runs, request_job_run::request_job_run, + request_person_job_run::request_person_job_run, request_rerun::request_rerun, +}; +use crate::runs::models::{ + CancelRunRequest, JobRunRequest, RerunRequest, RunAuthority, RunChange, RunChangeRequest, +}; +use crate::runs::tests::helpers::{request_run, snapshot}; +use crate::runs::tests::test_types::PersonRunPermissionTestCase; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_administrators_can_rewrite_history() + { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json"); + let inputs: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("DuckShop person lifecycle")) + .expect("lifecycle settings"); + let settings_count = inputs["job_settings"].as_array().expect("settings").len(); + let test_cases = [ + PersonRunPermissionTestCase { + description: "viewer cannot run ordinary jobs", + role: BuiltinRole::Viewer, + settings_range: 0..1, + expected_allowed: false, + }, + PersonRunPermissionTestCase { + description: "operator can run ordinary jobs", + role: BuiltinRole::Operator, + settings_range: 0..1, + expected_allowed: true, + }, + PersonRunPermissionTestCase { + description: "developer can run ordinary jobs", + role: BuiltinRole::Developer, + settings_range: 0..1, + expected_allowed: true, + }, + PersonRunPermissionTestCase { + description: "operator cannot rewrite history", + role: BuiltinRole::Operator, + settings_range: 1..settings_count, + expected_allowed: false, + }, + PersonRunPermissionTestCase { + description: "developer cannot rewrite history", + role: BuiltinRole::Developer, + settings_range: 1..settings_count, + expected_allowed: false, + }, + PersonRunPermissionTestCase { + description: "installation administrator can rewrite history", + role: BuiltinRole::Admin, + settings_range: 1..settings_count, + expected_allowed: true, + }, + ]; + for backend in [Backend::Sqlite, Backend::Postgres] { + for test_case in &test_cases { + for settings_index in test_case.settings_range.clone() { + run(async { + let fixture = open_test_store(backend).await.expect("real database"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("trusted peer")), + }; + let admin = bootstrap_user(store, &context, "orders-admin") + .await + .expect("bootstrap"); + let person = create_principal(store, &context, "orders-person") + .await + .expect("role-free person"); + let mut sessions = Vec::new(); + for (id, name) in [(admin, "orders-admin"), (person, "orders-person")] { + set_password( + store, + &context, + PasswordAuthority::Operator(id), + "orders-password-2026", + ) + .await + .expect("provision password"); + let PasswordLogin::Authenticated(session) = + login_password(store, &context, (name, "orders-password-2026")) + .await + .expect("login") + else { + panic!("valid session") + }; + sessions.push(session); + } + let credentials = sessions[1].identity.principal_id; + let location_id = create_location(store, "orders", "test-owner") + .await + .expect("location"); + let mut input = snapshot(location_id); + input.documents.insert("jobs".into(), json!({ + "daily_orders": {"description":"Build orders", "selection":["orders"], "settings":inputs["job_settings"][settings_index]}, + "customers": {"description":"Build customers", "selection":["customers"], "settings":inputs["job_settings"][0]} + })); + input.documents.insert("groups".into(), json!({"orders_group": {"description":"Build both jobs", "selection":["customers","orders"], "jobs":{"daily_orders":[],"customers":[]}}})); + let published = publish_snapshot(store, &input) + .await + .expect("publish immutable job"); + let request = JobRunRequest { + run_id: Uuid::now_v7(), + location_id, + target: "development".into(), + job_key: "daily_orders".into(), + reason: "Refresh order totals".into(), + priority: 0, + window: None, + }; + assert!( + matches!( + request_person_job_run(store, (&context, credentials), &request).await, + Err(ControlError::Denied { .. }) + ), + "no role is the default" + ); + let assignment = RoleAssignmentRequest { + principal_id: person, + role: test_case.role.into(), + location_id: None, + target: None, + revoke: false, + }; + assert!( + assign_builtin_role( + store, + (&context, sessions[0].identity.principal_id), + &assignment + ) + .await + .expect("authenticated assignment") + ); + assert!( + list_person_runs( + store, + (&context, credentials), + (location_id, "development", 10), + DefaultRole::None, + ) + .await + .expect("built-in metadata access") + .is_empty() + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let mut reruns = Vec::new(); + for job_key in ["daily_orders", "orders_group"] { + let result = request_person_job_run( + store, + (&context, credentials), + &JobRunRequest { + run_id: Uuid::now_v7(), + job_key: job_key.into(), + ..request.clone() + }, + ) + .await; + assert_eq!( + matches!(&result, Err(ControlError::Denied { .. })), + !test_case.expected_allowed, + "{}: {result:?}", + test_case.description + ); + assert_eq!( + result.ok(), + test_case.expected_allowed.then_some(true), + "{}", + test_case.description + ); + let source_id = Uuid::now_v7(); + request_job_run( + store, + &context, + &JobRunRequest { + run_id: source_id, + job_key: job_key.into(), + ..request.clone() + }, + ) + .await + .expect("owner creates source with frozen settings"); + let cancellation = CancelRunRequest { + run_id: source_id, + event_id: Uuid::now_v7(), + reason: "Stop before worker launch".into(), + }; + let can_cancel = test_case.role != BuiltinRole::Viewer; + for expected in [true, false] { + assert_eq!( + cancel_person_run(store, (&context, credentials), &cancellation) + .await + .ok(), + can_cancel.then_some(expected), + "cancellation permits stopping even sensitive work" + ); + } + let source_id = Uuid::now_v7(); + request_job_run( + store, + &context, + &JobRunRequest { + run_id: source_id, + job_key: job_key.into(), + ..request.clone() + }, + ) + .await + .expect("owner creates independent rerun source"); + change_run( + store, + RunAuthority::Operator(&context), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: source_id, + claim_id: None, + change: RunChange::CancelRequested { + actor: context.actor.clone(), + reason: cancellation.reason.clone(), + }, + }, + ) + .await + .expect("owner stops independent rerun source"); + for _ in 0..2 { + advance_groups( + store, + &lease, + (&Default::default(), (location_id, "development")), + ) + .await + .expect("scheduler confirms stopped children and parent"); + } + let rerun = RerunRequest { + run_id: Uuid::now_v7(), + source_run_id: source_id, + from_failure: false, + reason: "Rerun the original frozen execution".into(), + priority: 0, + }; + for expected in [true, false] { + let result = + request_rerun(store, (&context, Some(credentials)), &rerun).await; + assert_eq!( + result.as_ref().ok().copied(), + test_case.expected_allowed.then_some(expected), + "{}: frozen job/group rerun and idempotent retry: {result:?}", + test_case.description, + ); + assert_eq!( + matches!(result, Err(ControlError::Denied { .. })), + !test_case.expected_allowed, + "denial must be authorization, not invalid input" + ); + } + reruns.push(rerun); + } + let mut adhoc = crate::runs::tests::helpers::request( + (location_id, published.snapshot_id), + Uuid::now_v7().as_u128(), + 0, + &["orders"], + ); + adhoc.settings = + serde_json::from_value(inputs["job_settings"][settings_index].clone()) + .expect("ad-hoc frozen settings"); + request_run(store, &adhoc) + .await + .expect("owner requests ad-hoc execution"); + cancel_person_run( + store, + (&context, sessions[0].identity.principal_id), + &CancelRunRequest { + run_id: adhoc.run_id, + event_id: Uuid::now_v7(), + reason: "Stop ad-hoc source".into(), + }, + ) + .await + .expect("administrator stops ad-hoc execution"); + let adhoc_rerun = RerunRequest { + run_id: Uuid::now_v7(), + source_run_id: adhoc.run_id, + from_failure: false, + reason: "Repeat ad-hoc execution".into(), + priority: 0, + }; + let can_rerun_adhoc = test_case.role == BuiltinRole::Admin + || (test_case.role == BuiltinRole::Developer && settings_index == 0); + let result = + request_rerun(store, (&context, Some(credentials)), &adhoc_rerun).await; + assert_eq!( + result.as_ref().ok().copied(), + can_rerun_adhoc.then_some(true), + "ad-hoc: {result:?}" + ); + assert_eq!( + matches!(result, Err(ControlError::Denied { .. })), + !can_rerun_adhoc + ); + reruns.push(adhoc_rerun); + assert_eq!( + list_runs(store, (location_id, "development"), 100) + .await + .expect("queue projection") + .len(), + 9 + 8 * usize::from(test_case.expected_allowed) + + usize::from(can_rerun_adhoc), + "owner sources plus allowed manual/rerun jobs and expanded groups only" + ); + let administrator_credentials = sessions[0].identity.principal_id; + assert!( + change_administrator( + store, + (&context, administrator_credentials), + (person, AdministratorChange::Disable), + ) + .await + .expect("disable person") + ); + assert!( + assign_builtin_role( + store, + (&context, administrator_credentials), + &RoleAssignmentRequest { + revoke: true, + ..assignment + } + ) + .await + .expect("revoke a disabled person's grant before reenabling") + ); + assert!( + change_administrator( + store, + (&context, administrator_credentials), + (person, AdministratorChange::Enable), + ) + .await + .expect("enable person") + ); + assert!( + matches!( + request_person_job_run( + store, + (&context, credentials), + &JobRunRequest { + run_id: Uuid::now_v7(), + ..request + } + ) + .await, + Err(ControlError::Denied { .. }) + ), + "reenabling cannot restore a revoked grant" + ); + for rerun in reruns { + assert!( + matches!( + request_rerun(store, (&context, Some(credentials)), &rerun).await, + Err(ControlError::Denied { .. }) + ), + "retry must reauthorize after revocation" + ); + } + close_test_store(fixture).await; + }); + } + } + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs new file mode 100644 index 0000000000..a66096e888 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs @@ -0,0 +1,249 @@ +//! Published-job requests freeze execution inputs and audit atomically on both databases. + +use std::collections::BTreeMap; + +use crate::credentials::models::AuditContext; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::list_runs::list_runs; +use crate::runs::main::request_job_run::request_job_run; +use crate::runs::models::{JobRunRequest, RunOrigin, RunStatus, RunWindow}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_and_frozen_execution_commit_once() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite queues and audits the pinned job exactly once", + backend: Backend::Sqlite, + expected_status: RunStatus::Queued, + }, + RunLifecycleTestCase { + description: "PostgreSQL queues and audits the pinned job exactly once", + backend: Backend::Postgres, + expected_status: RunStatus::Queued, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location_id); + let mut source = input.nodes["orders"].clone(); + source.kind = "source".into(); + input.nodes.insert("warehouse_orders".into(), source); + input.documents.insert("jobs".into(), json!({"daily_orders": { + "description": "Build orders from the warehouse source", "selection": ["orders", "warehouse_orders"], + "settings": {"full_refresh": true}, "concurrency_groups": ["orders-load"] + }})); + let published = publish_snapshot(store, &input).await.expect("publish job"); + let request = JobRunRequest { + run_id: Uuid::now_v7(), + location_id, + target: "development".into(), + job_key: "daily_orders".into(), + reason: "Refresh orders".into(), + priority: 7, + window: Some(RunWindow::Timestamp { + start: "2026-07-03T01:17:23+05:30".parse().expect("start"), + end: "2026-07-04T01:17:23+05:30".parse().expect("end"), + }), + }; + for invalid in [ + JobRunRequest { + target: "production".into(), + ..request.clone() + }, + JobRunRequest { + location_id: Uuid::now_v7(), + ..request.clone() + }, + JobRunRequest { + job_key: "unknown_job".into(), + ..request.clone() + }, + JobRunRequest { + reason: " ".into(), + ..request.clone() + }, + JobRunRequest { + window: Some(RunWindow::Integer { start: 11, end: 3 }), + ..request.clone() + }, + JobRunRequest { + window: Some(RunWindow::Timestamp { + start: "2026-07-04T01:17:23Z".parse().expect("start"), + end: "2026-07-03T01:17:23Z".parse().expect("end"), + }), + ..request.clone() + }, + ] { + assert!(request_job_run(store, &context, &invalid).await.is_err()); + } + let fault = TransactionFault { + description: "manual run audit failure", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected audit failure')", + postgres_body: "RAISE EXCEPTION 'injected audit failure'", + }; + install_fault(store, &fault).await; + assert!(request_job_run(store, &context, &request).await.is_err()); + remove_fault(store, &fault).await; + assert!( + list_runs(store, (location_id, "development"), 10) + .await + .expect("rolled back projection") + .is_empty() + ); + assert!( + read_recall_events(store, 0, 100) + .await + .expect("rolled back facts") + .is_empty() + ); + let (first, retry) = tokio::join!( + request_job_run(store, &context, &request), + request_job_run(store, &context, &request) + ); + assert_ne!( + first.expect("first request"), + retry.expect("concurrent retry") + ); + let facts = read_recall_events(store, 0, 100) + .await + .expect("request facts"); + assert_eq!(facts.len(), 2); + input.generation = 2; + input.image_id = Some("sha256:synthetic-orders-image-2".into()); + input.documents.get_mut("jobs").expect("jobs")["daily_orders"]["selection"] = + json!(["inventory"]); + input.documents.get_mut("jobs").expect("jobs")["daily_orders"]["settings"] = + json!({"full_refresh": false}); + publish_snapshot(store, &input).await.expect("redeploy"); + assert!( + !request_job_run(store, &context, &request) + .await + .expect("retry keeps original snapshot") + ); + let mut changed = request.clone(); + changed.reason = "Different request".into(); + assert!(request_job_run(store, &context, &changed).await.is_err()); + changed = request.clone(); + changed.window = Some(RunWindow::Integer { start: 3, end: 11 }); + assert!(request_job_run(store, &context, &changed).await.is_err()); + let other_actor = AuditContext { + actor: "other-operator".into(), + ..context.clone() + }; + assert!( + request_job_run(store, &other_actor, &request) + .await + .is_err() + ); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("no duplicate facts"), + facts + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([( + "orders-load".into(), + std::num::NonZeroU32::new(1).expect("positive group limit"), + )]); + let claim = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("queued request"); + assert_eq!(claim.request.run_id, request.run_id); + assert_eq!(claim.request.snapshot_id, published.snapshot_id); + assert_eq!(claim.request.origin, RunOrigin::Backfill); + assert_eq!( + claim.request.window, + Some(RunWindow::Timestamp { + start: "2026-07-02T19:47:23Z".parse().expect("normalized start"), + end: "2026-07-03T19:47:23Z".parse().expect("normalized end"), + }) + ); + assert_eq!( + claim.request.image_id.as_deref(), + Some("sha256:synthetic-orders-image") + ); + assert_eq!( + claim.request.selection, + ["orders".into(), "warehouse_orders".into()].into() + ); + assert_eq!(claim.request.write_set, ["orders".into()].into()); + assert_eq!( + claim.request.settings, + BTreeMap::from([("full_refresh".into(), json!(true))]) + ); + assert_eq!( + claim.request.concurrency_groups, + ["orders-load".into()].into() + ); + assert_eq!(claim.request.priority, 7); + assert_eq!(claim.request.requested_by, "test-operator"); + let fresh = JobRunRequest { + run_id: Uuid::now_v7(), + window: Some(RunWindow::Integer { start: 7, end: 7 }), + ..request.clone() + }; + assert!( + request_job_run(store, &context, &fresh) + .await + .expect("new request uses redeploy") + ); + let runs = list_runs(store, (location_id, "development"), 10) + .await + .expect("queued jobs"); + assert_eq!(runs.len(), 2); + assert_eq!( + runs.iter() + .find(|run| run.run_id == fresh.run_id) + .expect("new run") + .status, + test_case.expected_status, + "{}", + test_case.description + ); + assert_eq!( + runs.iter() + .find(|run| run.run_id == fresh.run_id) + .expect("new run") + .selection_json, + ["inventory".into()].into() + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs new file mode 100644 index 0000000000..cc08da0422 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -0,0 +1,344 @@ +//! Terminal retries retain their snapshot and independent limits without repeating successful work. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::credentials::models::AuditContext; +use crate::runs::main::{ + advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, + list_runs::list_runs, request_job_run::request_job_run, request_rerun::request_rerun, +}; +use crate::runs::models::{ + JobRunRequest, NodeOutcome, RerunRequest, RunAuthority, RunChange, RunChangeRequest, RunOrigin, + RunOutcome, RunStatus, RunWindow, +}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::claim_helpers::{slot_counts, succeed}; +use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::test_types::{ + RerunExecutionTestCase, RunLifecycleTestCase, TransactionFault, +}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_nodes_retry_atomically() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite pinned group rerun", + backend: Backend::Sqlite, + expected_status: RunStatus::Success, + }, + RunLifecycleTestCase { + description: "PostgreSQL pinned group rerun", + backend: Backend::Postgres, + expected_status: RunStatus::Success, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input.documents.insert("jobs".into(), json!({ + "staging": {"description": "Stage orders", "selection": ["orders", "customers"], "concurrency_groups": ["warehouse"], "settings": {"vars": {"batch": 23}}}, + "marts": {"description": "Build inventory", "selection": ["inventory"], "concurrency_groups": ["inventory_limit"]}, + "completed": {"description": "Build products", "selection": ["products"]} + })); + input.documents.insert("groups".into(), json!({"daily_orders": { + "description": "Build daily orders", "selection": ["customers", "orders", "inventory", "products"], + "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["completed"], "completed": ["staging"]} + }})); + let published = publish_snapshot(store, &input).await.expect("publish"); + let source_id = Uuid::now_v7(); + request_job_run( + store, + &context, + &JobRunRequest { + run_id: source_id, + location_id: location, + target: "development".into(), + job_key: "daily_orders".into(), + reason: "Build orders".into(), + priority: 1, + window: Some(RunWindow::Integer { start: -3, end: 11 }), + }, + ) + .await + .expect("request group"); + let retry = RerunRequest { + run_id: Uuid::from_u128(1), + source_run_id: source_id, + from_failure: true, + reason: "Retry failed orders".into(), + priority: 7, + }; + assert!( + request_rerun(store, (&context, None), &retry) + .await + .is_err(), + "a live source cannot be rerun" + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([ + ( + "warehouse".into(), + std::num::NonZeroU32::new(1).expect("positive capacity"), + ), + ( + "inventory_limit".into(), + std::num::NonZeroU32::new(1).expect("positive capacity"), + ), + ]); + let scope = (location, "development"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("admit source"); + let node_outcomes = BTreeMap::from([ + ("products", NodeOutcome::Success), + ("orders", NodeOutcome::Failed), + ("customers", NodeOutcome::Cancelled), + ("inventory", NodeOutcome::Skipped), + ]); + let job_outcomes = BTreeMap::from([ + ("staging", RunOutcome::Failed), + ("completed", RunOutcome::Success), + ("marts", RunOutcome::Failed), + ]); + for _ in 0..3 { + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim source") + .expect("ready child"); + let mut changes = vec![RunChange::Started]; + for node in &claim.request.selection { + let outcome = node_outcomes[node.as_str()]; + changes.push(RunChange::NodeFinished { + node_key: node.clone(), + outcome, + rows_affected: None, + error_code: None, + skipped_because_of: (outcome == NodeOutcome::Skipped) + .then(|| "orders".into()), + }); + } + changes.push(RunChange::Finished { + outcome: job_outcomes[claim.request.job_key.as_deref().expect("source job")], + reason: "synthetic source outcome".into(), + exit_code: Some(1), + }); + for change in changes { + change_run( + store, + RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await + .expect("source worker fact"); + } + } + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish source"); + assert_eq!( + list_runs(store, scope, 20) + .await + .expect("list completed source") + .iter() + .find(|run| run.run_id == source_id) + .expect("source exists") + .status, + RunStatus::Partial + ); + input.generation = 2; + input.image_id = Some("sha256:synthetic-orders-image-2".into()); + input.documents.get_mut("jobs").expect("job declarations")["staging"]["settings"] = + json!({"vars": {"batch": 91}}); + publish_snapshot(store, &input).await.expect("redeploy"); + let fault = TransactionFault { + description: "rerun audit failure", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected audit failure')", + postgres_body: "RAISE EXCEPTION 'injected audit failure'", + }; + install_fault(store, &fault).await; + assert!( + request_rerun(store, (&context, None), &retry) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert_eq!( + list_runs(store, scope, 20) + .await + .expect("list rolled-back runs") + .len(), + 4, + "audit failure rolls back parent and all children" + ); + let (first, again) = tokio::join!( + request_rerun(store, (&context, None), &retry), + request_rerun(store, (&context, None), &retry) + ); + assert_ne!(first.expect("rerun"), again.expect("replica retry")); + let changed = RerunRequest { + from_failure: false, + ..retry.clone() + }; + assert!( + request_rerun(store, (&context, None), &changed) + .await + .is_err() + ); + let other = AuditContext { + actor: "other-operator".into(), + ..context.clone() + }; + assert!(request_rerun(store, (&other, None), &retry).await.is_err()); + let settings = BTreeMap::from([ + ( + "staging", + BTreeMap::from([("vars".into(), json!({"batch": 23}))]), + ), + ("completed", BTreeMap::new()), + ("marts", BTreeMap::new()), + ]); + for execution in [ + RerunExecutionTestCase { + description: "failure retry omits successful and cancelled nodes", + request: retry.clone(), + expected_applied: false, + expected_selection: ["orders", "inventory"].map(str::to_owned).into(), + expected_jobs: &["staging", "marts"], + }, + RerunExecutionTestCase { + description: "full retry retains all original nodes", + request: RerunRequest { + run_id: Uuid::now_v7(), + from_failure: false, + ..retry.clone() + }, + expected_applied: true, + expected_selection: ["orders", "inventory", "customers", "products"] + .map(str::to_owned) + .into(), + expected_jobs: &["staging", "completed", "marts"], + }, + ] { + let request = execution.request; + assert_eq!( + request_rerun(store, (&context, None), &request) + .await + .expect("publish rerun"), + execution.expected_applied, + "{}", + execution.description + ); + let runs = list_runs(store, scope, 20).await.expect("list reruns"); + assert_eq!( + runs.iter() + .find(|run| run.run_id == request.run_id) + .expect("rerun exists") + .selection_json, + execution.expected_selection + ); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("admit rerun"); + let mut selected: BTreeSet = BTreeSet::new(); + for expected_job in execution.expected_jobs { + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim rerun") + .expect("ready retry child"); + assert_eq!( + claim.request.job_key.as_deref(), + Some(*expected_job), + "a completed intermediate job must not erase its ancestor's wait boundary" + ); + assert_eq!(claim.request.snapshot_id, published.snapshot_id); + assert_eq!( + claim.request.image_id, + Some("sha256:synthetic-orders-image".into()) + ); + assert_eq!(claim.request.origin, RunOrigin::Rerun); + assert_eq!(claim.request.priority, 7); + assert_eq!(claim.request.settings, settings[expected_job]); + assert_eq!( + claim.request.window, + Some(RunWindow::Integer { start: -3, end: 11 }) + ); + selected.extend(claim.request.selection.iter().cloned()); + succeed(store, &claim).await; + } + assert_eq!(selected, execution.expected_selection); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish rerun"); + assert_eq!( + list_runs(store, scope, 20) + .await + .expect("list finished reruns") + .iter() + .find(|run| run.run_id == request.run_id) + .expect("finished rerun exists") + .status, + test_case.expected_status, + "{}", + test_case.description + ); + assert!(slot_counts(store).await.is_empty()); + assert!( + !request_rerun(store, (&context, None), &request) + .await + .expect("completed transport retry") + ); + let successful = RerunRequest { + run_id: Uuid::now_v7(), + source_run_id: request.run_id, + from_failure: true, + ..request.clone() + }; + assert!( + request_rerun(store, (&context, None), &successful) + .await + .is_err(), + "successful work has nothing to retry from failure" + ); + } + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs new file mode 100644 index 0000000000..0214432a6d --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -0,0 +1,116 @@ +//! Expected queue outcomes for each real control-store backend. + +use std::collections::BTreeSet; + +use crate::credentials::models::BuiltinRole; +use crate::runs::models::{RerunRequest, RunStatus}; + +pub(crate) use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct PersonRunPermissionTestCase { + pub(crate) description: &'static str, + pub(crate) role: BuiltinRole, + pub(crate) settings_range: std::ops::Range, + pub(crate) expected_allowed: bool, +} + +pub(crate) struct ScopedClaimTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_selected_run: u128, + pub(crate) expected_unscoped_run: u128, +} + +pub(crate) struct PriorityClaimsTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_initial_claims: [u128; 2], + pub(crate) expected_claim_events: usize, + pub(crate) expected_terminal_events: usize, +} + +pub(crate) struct GroupLimitTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_unblocked_run: u128, +} + +pub(crate) struct BacklogClaimTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_independent_run: u128, +} + +pub(crate) struct CoalescedRunTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_runs: usize, +} + +pub(crate) struct ClaimFailureTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_run: u128, +} + +pub(crate) struct RunLifecycleTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_status: RunStatus, +} + +pub(crate) struct RerunExecutionTestCase { + pub(crate) description: &'static str, + pub(crate) request: RerunRequest, + pub(crate) expected_applied: bool, + pub(crate) expected_selection: BTreeSet, + pub(crate) expected_jobs: &'static [&'static str], +} + +pub(crate) struct RunMonitorTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_statuses: [RunStatus; 5], +} + +pub(crate) struct MonitorBacklogTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) run_count: usize, + pub(crate) expected_batches: [usize; 3], +} + +pub(crate) struct AdmissionParent { + pub(crate) holds: &'static [&'static str], + pub(crate) children: &'static [&'static [&'static str]], +} + +pub(crate) struct GroupAdmissionTestCase { + pub(crate) description: &'static str, + pub(crate) limits: &'static [(&'static str, u32)], + pub(crate) parents: &'static [AdmissionParent], + pub(crate) expected_running: usize, +} + +pub(crate) struct GroupChildLimitTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_claimed_together: usize, + pub(crate) expected_jobs: BTreeSet<&'static str>, +} + +pub(crate) struct GroupQueueTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) group_count: usize, + pub(crate) node_count: usize, + pub(crate) expected_max_claim_millis: u128, +} + +pub(crate) struct MixedAdmissionTestCase { + pub(crate) description: &'static str, + pub(crate) physical_priority: i64, + pub(crate) parent_priority: i64, + pub(crate) expected_selections: [&'static str; 2], +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs new file mode 100644 index 0000000000..a1e8ee33dc --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs @@ -0,0 +1,72 @@ +//! Contention and expiry exercise scheduler ownership against real databases, not fake clocks. + +pub(crate) use crate::scheduler::main::_acquire_lease::acquire_lease; +pub(crate) use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; +pub(crate) use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; +use crate::scheduler::main::renew_lease::renew_lease; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store}; +use crate::store::tests::store_test_types::Backend; + +pub(crate) async fn leadership_contract(backend: Backend) -> Option { + let test_store = open_test_store(backend).await?; + let store = &test_store.store; + migrate_store(store) + .await + .expect("migrate leadership store"); + let second = second_store(store).await; + let (first, other) = tokio::join!( + acquire_lease(store, Uuid::now_v7()), + acquire_lease(&second, Uuid::now_v7()) + ); + let first = first.expect("first contender completes"); + let other = other.expect("second contender completes"); + assert_ne!( + first.is_some(), + other.is_some(), + "exactly one concurrent contender is elected" + ); + let lease = first.or(other).expect("one elected holder"); + let renewed = renew_lease(&second, &lease) + .await + .expect("live epoch renews"); + assert_eq!(renewed.epoch, lease.epoch); + assert!(renewed.expires_at >= lease.expires_at); + assert!( + acquire_lease(store, lease.holder) + .await + .expect("duplicate acquisition completes") + .is_none() + ); + let mut forged = lease.clone(); + forged.holder = Uuid::now_v7(); + assert!(matches!( + renew_lease(store, &forged).await, + Err(ControlError::Fenced) + )); + expire_lease(store).await; + assert!( + matches!(renew_lease(store, &lease).await, Err(ControlError::Fenced)), + "even an uncontested expired holder must reacquire a new epoch" + ); + let successor = acquire_lease(&second, Uuid::now_v7()) + .await + .expect("acquire expired lease") + .expect("successor elected"); + assert!(matches!( + renew_lease(store, &lease).await, + Err(ControlError::Fenced) + )); + renew_lease(store, &successor) + .await + .expect("successor owns the current fence"); + let epoch = successor.epoch; + close_store(second).await; + close_test_store(test_store).await; + Some(epoch) +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs new file mode 100644 index 0000000000..13bdfd4250 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs @@ -0,0 +1,28 @@ +use crate::scheduler::tests::contracts::helpers::leadership_contract; +use crate::scheduler::tests::contracts::test_types::LeadershipTestCase; +use crate::store::tests::store_helpers::run; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_two_schedulers_when_leadership_expires_then_only_the_new_epoch_can_renew() { + let test_cases = [ + LeadershipTestCase { + description: "SQLite contenders and takeover", + backend: Backend::Sqlite, + expected_epoch: 2, + }, + LeadershipTestCase { + description: "PostgreSQL contenders and takeover", + backend: Backend::Postgres, + expected_epoch: 2, + }, + ]; + for test_case in test_cases { + let observed = run(leadership_contract(test_case.backend)); + assert!( + observed.is_none_or(|epoch| epoch == test_case.expected_epoch), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs new file mode 100644 index 0000000000..c70bd73153 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs @@ -0,0 +1,10 @@ +//! Real-store scheduler contracts compiled within the scheduler owner. + +#[path = "helpers.rs"] +pub(crate) mod helpers; +#[path = "test_contract.rs"] +mod test_contract; +#[path = "test_types.rs"] +mod test_types; +#[path = "ticks/test_ticks.rs"] +pub(crate) mod ticks; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_sensor_fairness.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_sensor_fairness.rs new file mode 100644 index 0000000000..5259d9ee0b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_sensor_fairness.rs @@ -0,0 +1,128 @@ +//! Unavailable callback locations cannot monopolize the shared due-sensor page. + +use chrono::{TimeDelta, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; +use crate::scheduler::models::SchedulerState; +use crate::scheduler::tests::test_types::SensorFairnessTestCase; +use crate::sensors::models::SensorEvaluationOutcome; +use crate::sensors::tests::helpers::{evaluations, snapshot}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::query::query; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_pending_custom_sensors_when_scheduling_then_rechecks_wait_and_other_locations_progress() { + let test_cases = [ + SensorFairnessTestCase { + description: "SQLite pending callback waits", + backend: Backend::Sqlite, + pending: 1, + builtin_status: "paused", + expected_completed: 0, + }, + SensorFairnessTestCase { + description: "PostgreSQL pending callback waits", + backend: Backend::Postgres, + pending: 1, + builtin_status: "paused", + expected_completed: 0, + }, + SensorFairnessTestCase { + description: "SQLite unavailable callback page permits builtin progress", + backend: Backend::Sqlite, + pending: 100, + builtin_status: "running", + expected_completed: 1, + }, + SensorFairnessTestCase { + description: "PostgreSQL unavailable callback page permits builtin progress", + backend: Backend::Postgres, + pending: 100, + builtin_status: "running", + expected_completed: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate fairness store"); + let custom = create_location(store, "fulfillment", "test-admin") + .await + .expect("callback location"); + let builtin = create_location(store, "reporting", "test-admin") + .await + .expect("builtin location"); + let mut input = snapshot(custom); + input.documents.insert("triggers".into(), json!({})); + for index in 0..test_case.pending { + input.documents.get_mut("triggers").expect("triggers")[format!("orders_{index}")] = json!({ + "description": "Poll fulfillment", "job_key": "orders", "when": {"kind": "custom", "function": "orders_ready", "code_hash": "orders_v1"}, + "minimum_interval_seconds": 1 + }); + } + publish_snapshot(store, &input) + .await + .expect("publish unavailable callbacks"); + publish_snapshot(store, &snapshot(builtin)) + .await + .expect("publish independent builtin"); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("set due order"); + let (transaction, _) = query(transaction, + "UPDATE sqb_trigger_state SET next_due_at=$1 WHERE trigger_id IN (SELECT trigger_id FROM sqb_triggers WHERE location_id=$2)", + vec![Parameter::Time(Utc::now() - TimeDelta::hours(2)), Parameter::Id(custom)]).await.expect("callbacks first"); + let (transaction, _) = query(transaction, + "UPDATE sqb_trigger_state SET next_due_at=$1,status=$2 WHERE trigger_id IN (SELECT trigger_id FROM sqb_triggers WHERE location_id=$3)", + vec![Parameter::Time(Utc::now() - TimeDelta::hours(1)), Parameter::Text(test_case.builtin_status.into()), Parameter::Id(builtin)]).await.expect("builtin last"); + commit(transaction).await.expect("commit due order"); + let (state, first) = scheduler_cycle(store, SchedulerState::new(Uuid::now_v7())) + .await + .expect("pending page"); + assert_eq!( + first.evaluated, test_case.pending, + "{}", + test_case.description + ); + assert_eq!( + first.wait_for.is_zero(), + test_case.expected_completed == 1, + "{}: only genuinely overdue unprocessed work should require an immediate retry", + test_case.description + ); + scheduler_cycle(store, state).await.expect("next page"); + let facts = evaluations(store).await; + assert_eq!( + facts + .iter() + .filter(|fact| fact.outcome == SensorEvaluationOutcome::Requested) + .count(), + test_case.pending + test_case.expected_completed, + "{}", + test_case.description + ); + assert_eq!( + facts + .iter() + .filter(|fact| fact.outcome != SensorEvaluationOutcome::Requested) + .count(), + test_case.expected_completed, + "{}: builtin at another location completes without waiting for unavailable callbacks", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs new file mode 100644 index 0000000000..fd5543086b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs @@ -0,0 +1,10 @@ +//! Leadership scenarios run independently on both supported engines. + +use crate::store::tests::store_test_types::Backend; + +#[derive(Debug)] +pub(crate) struct LeadershipTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_epoch: i64, +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs new file mode 100644 index 0000000000..855dde7ab0 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -0,0 +1,731 @@ +//! Trigger and failure fixtures exercise the scheduler's transactional tick boundary. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::LazyLock; + +use chrono::{DateTime, DurationRound, SecondsFormat, TimeDelta, Timelike, Utc}; + +use crate::declarations::main::list_triggers::list_triggers; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::main::_acquire_lease::acquire_lease; +use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; +use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; +use crate::scheduler::main::list_ticks::list_ticks; +use crate::scheduler::main::run_scheduler::run_scheduler; +use crate::scheduler::models::{SchedulerLease, SchedulerState, TickOutcome, TickRecord}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::scheduler::tests::contracts::ticks::test_types::{ + FaultAnomalies, FaultStep, IdleLauncher, TransactionFault, +}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::helpers::read_recall_events; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store}; +use crate::store::tests::store_test_types::Backend; + +/// More than one due-trigger page of requesting triggers keeps every crash point reachable. +const FAULT_TRIGGERS: usize = 400; +const FAULT_BACKLOG_MINUTES: i64 = 1; + +/// Calendar fixtures stay inside the tick-retention horizon: day 0 is midnight three days ago. +pub(crate) fn calendar_time(day: i64) -> DateTime { + static BASE: LazyLock> = LazyLock::new(|| { + (Utc::now() - TimeDelta::days(3)) + .duration_trunc(TimeDelta::days(1)) + .expect("calendar base") + }); + *BASE + TimeDelta::days(day) +} + +pub(crate) fn calendar_text(day: i64) -> String { + calendar_time(day).to_rfc3339_opts(SecondsFormat::Secs, true) +} + +pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { + SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: Some("sha256:synthetic-orders-image".into()), + environment_hash: "orders-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::from([( + "orders".into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"materialization": "table"}), + raw_sql: Some("select 1 as order_id".into()), + compiled_sql: Some("select 1 as order_id".into()), + python_source: None, + }, + )]), + documents: BTreeMap::from([ + ( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), + ), + ( + "triggers".into(), + json!({"daily_orders": {"description": "Refresh orders", "job_key": "orders", "starts_at": calendar_text(0), + "schedule": {"kind": "calendar", "times": [calendar_text(0), calendar_text(1), calendar_text(2), "2030-01-01T00:00:00Z"], "refresh_at": "2029-01-01T00:00:00Z"}}}), + ), + ]), + } +} + +async fn trigger_audit_rows(store: &ControlStore) -> Vec<(String, String, String, String, String)> { + let sql = "SELECT CAST(event_id AS TEXT), actor, granting_role, CAST(before_json AS TEXT), CAST(after_json AS TEXT) FROM sqb_security_audit WHERE action='trigger.changed'"; + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + return rusqlite::Connection::open(path) + .expect("inspect trigger audit") + .prepare(sql) + .expect("audit query") + .query_map([], |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }) + .expect("read trigger audits") + .collect::>() + .expect("audit rows"); + }; + sqlx_core::query_as::query_as(sql) + .fetch_all(pool) + .await + .expect("trigger audits") +} + +pub(crate) async fn trigger_audits(store: &ControlStore) -> Vec { + trigger_audit_rows(store) + .await + .into_iter() + .map(|(event_id, actor, granting_role, before, after)| { + json!({ + "event_id": event_id, + "actor": actor, + "granting_role": granting_role, + "before_json": serde_json::from_str::(&before).expect("before state"), + "after_json": serde_json::from_str::(&after).expect("after state"), + }) + }) + .collect() +} + +/// Crash points inside the tick transaction, from the tick row to the cursor and the fence. +pub(crate) fn tick_faults() -> [TransactionFault; 7] { + [ + TransactionFault { + description: "tick insertion", + table: "sqb_ticks", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected tick failure')", + postgres_body: "RAISE EXCEPTION 'injected tick failure'", + }, + TransactionFault { + description: "requested event", + table: "recall_events", + operation: "INSERT", + condition: "WHEN NEW.event_type='run_requested'", + sqlite_body: "SELECT RAISE(ABORT,'injected requested failure')", + postgres_body: "RAISE EXCEPTION 'injected requested failure'", + }, + TransactionFault { + description: "queued event", + table: "recall_events", + operation: "INSERT", + condition: "WHEN NEW.event_type='run_queued'", + sqlite_body: "SELECT RAISE(ABORT,'injected queued failure')", + postgres_body: "RAISE EXCEPTION 'injected queued failure'", + }, + TransactionFault { + description: "run projection", + table: "sqb_runs", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected run failure')", + postgres_body: "RAISE EXCEPTION 'injected run failure'", + }, + TransactionFault { + description: "node projection", + table: "sqb_run_nodes", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected node failure')", + postgres_body: "RAISE EXCEPTION 'injected node failure'", + }, + TransactionFault { + description: "cursor commit", + table: "sqb_trigger_state", + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected cursor failure')", + postgres_body: "RAISE EXCEPTION 'injected cursor failure'", + }, + TransactionFault { + description: "lease expires during append", + table: "sqb_runs", + operation: "INSERT", + condition: "", + sqlite_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + postgres_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + }, + ] +} + +pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usize)> { + let fixture = open_test_store(backend).await?; + let store = &fixture.store; + migrate_store(store).await.expect("migrate tick store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("create location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publish job and trigger"); + let trigger_id = list_triggers(store, location, "development") + .await + .expect("published triggers")[0] + .trigger_id; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("acquire lease") + .expect("elected"); + let second = second_store(store).await; + for point in tick_faults() { + install_fault(store, &point).await; + let failed = evaluate_trigger(store, &lease, trigger_id).await; + assert!(failed.is_err(), "{} must abort", point.description); + remove_fault(store, &point).await; + assert!( + list_ticks(&second, trigger_id, 100) + .await + .expect("rolled-back ticks") + .is_empty(), + "{}", + point.description + ); + assert!( + list_runs(&second, (location, "development"), 100) + .await + .expect("rolled-back runs") + .is_empty(), + "{}", + point.description + ); + assert!( + read_recall_events(&second, 0, 100) + .await + .expect("rolled-back events") + .is_empty(), + "{}", + point.description + ); + assert!( + list_triggers(&second, location, "development") + .await + .expect("rolled-back cursor")[0] + .last_tick_for + .is_none(), + "{}", + point.description + ); + } + let (left, right) = tokio::join!( + evaluate_trigger(store, &lease, trigger_id), + evaluate_trigger(&second, &lease, trigger_id) + ); + let first_count = left + .expect("first scheduler") + .expect("active trigger") + .actions + .len(); + let second_count = right + .expect("second scheduler") + .expect("active trigger") + .actions + .len(); + assert_eq!( + first_count + second_count, + 1, + "restarts never repeat committed ticks" + ); + let ticks = list_ticks(store, trigger_id, 100) + .await + .expect("committed ticks"); + let runs = list_runs(store, (location, "development"), 100) + .await + .expect("committed run"); + let events = read_recall_events(store, 0, 100) + .await + .expect("committed events"); + assert_eq!( + events + .iter() + .map(|row| row["event_type"].as_str().expect("event type")) + .collect::>(), + ["run_requested", "run_queued"] + ); + assert_eq!(ticks[0].outcome, TickOutcome::Requested); + assert!( + ticks[1..] + .iter() + .all(|tick| tick.outcome == TickOutcome::Covered + && tick.covered_by_run_id == Some(runs[0].run_id)) + ); + assert_eq!( + runs[0].last_event_storage_id, + events[1]["storage_id"].as_i64().expect("event cursor") + ); + expire_lease(store).await; + let successor = acquire_lease(&second, Uuid::now_v7()) + .await + .expect("takeover") + .expect("new leader"); + assert!(matches!( + evaluate_trigger(store, &lease, trigger_id).await, + Err(ControlError::Fenced) + )); + assert!( + evaluate_trigger(&second, &successor, trigger_id) + .await + .expect("failover catch-up") + .expect("active trigger") + .actions + .is_empty() + ); + let counts = (ticks.len(), runs.len()); + close_store(second).await; + close_test_store(fixture).await; + Some(counts) +} + +/// Test-only clock skew: move a trigger's past calendar, cursor and ticks back by `days`. +pub(crate) async fn shift_schedule(store: &ControlStore, trigger_id: Uuid, days: i64) { + let shift = |value: &mut Value| { + let time: DateTime = serde_json::from_value(value.clone()).expect("calendar time"); + *value = json!(time - TimeDelta::days(days)); + }; + let past = |value: &&mut Value| { + serde_json::from_value::>((*value).clone()).expect("calendar time") + < Utc::now() + }; + let transaction = begin(store, TransactionMode::Write).await.expect("shift"); + let (transaction, rows) = query( + transaction, + "SELECT definition_json FROM sqb_triggers WHERE trigger_id=$1", + vec![Parameter::Id(trigger_id)], + ) + .await + .expect("trigger definition"); + let mut definition = rows[0]["definition_json"].clone(); + shift(&mut definition["starts_at"]); + for time in definition["schedule"]["times"] + .as_array_mut() + .expect("calendar times") + .iter_mut() + .filter(past) + { + shift(time); + } + let (transaction, _) = query( + transaction, + "UPDATE sqb_triggers SET definition_json=$1 WHERE trigger_id=$2", + vec![Parameter::Json(definition), Parameter::Id(trigger_id)], + ) + .await + .expect("shift calendar"); + let (transaction, ticks) = query( + transaction, + "SELECT tick_id, scheduled_for FROM sqb_ticks WHERE trigger_id=$1", + vec![Parameter::Id(trigger_id)], + ) + .await + .expect("ticks"); + let mut transaction = transaction; + for tick in ticks { + let mut time = tick["scheduled_for"].clone(); + shift(&mut time); + (transaction, _) = query( + transaction, + "UPDATE sqb_ticks SET scheduled_for=$1 WHERE tick_id=$2", + vec![ + Parameter::Time(serde_json::from_value(time).expect("tick time")), + Parameter::Id(serde_json::from_value(tick["tick_id"].clone()).expect("tick")), + ], + ) + .await + .expect("shift tick"); + } + let (transaction, state) = query( + transaction, + "SELECT started_at, last_tick_for FROM sqb_trigger_state WHERE trigger_id=$1", + vec![Parameter::Id(trigger_id)], + ) + .await + .expect("trigger state"); + let started: DateTime = + serde_json::from_value(state[0]["started_at"].clone()).expect("start"); + let last: Option> = + serde_json::from_value(state[0]["last_tick_for"].clone()).expect("cursor"); + let (transaction, _) = query( + transaction, + "UPDATE sqb_trigger_state SET started_at=$1, last_tick_for=$2 WHERE trigger_id=$3", + vec![ + Parameter::Time(started - TimeDelta::days(days)), + Parameter::OptionalTime(last.map(|time| time - TimeDelta::days(days))), + Parameter::Id(trigger_id), + ], + ) + .await + .expect("shift state"); + commit(transaction).await.expect("commit shift"); +} + +/// A later covered tick, as if another due time arrived while the requested run still waited. +pub(crate) async fn append_covered_tick( + store: &ControlStore, + trigger_id: Uuid, + scheduled_for: DateTime, +) { + let transaction = begin(store, TransactionMode::Write) + .await + .expect("covered tick"); + let (transaction, _) = query( + transaction, + "INSERT INTO sqb_ticks (tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, \ + run_id, covered_by_run_id) SELECT $1, trigger_id, $2, evaluated_at, epoch, \ + 'covered', NULL, run_id FROM sqb_ticks WHERE trigger_id=$3 AND outcome='requested'", + vec![ + Parameter::Id(Uuid::now_v7()), + Parameter::Time(scheduled_for), + Parameter::Id(trigger_id), + ], + ) + .await + .expect("append covered tick"); + commit(transaction).await.expect("commit covered tick"); +} + +/// Rewrite one trigger-state column the way an earlier build or a past start would have left it. +pub(crate) async fn set_trigger_state( + store: &ControlStore, + trigger_id: Uuid, + sql: &str, + time: DateTime, +) { + let transaction = begin(store, TransactionMode::Write).await.expect("state"); + let (transaction, _) = query( + transaction, + sql, + vec![Parameter::Time(time), Parameter::Id(trigger_id)], + ) + .await + .expect("rewrite trigger state"); + commit(transaction).await.expect("commit state"); +} + +pub(crate) async fn tick_times(store: &ControlStore, trigger_id: Uuid) -> Vec> { + list_ticks(store, trigger_id, 1000) + .await + .expect("ticks") + .into_iter() + .map(|tick| tick.scheduled_for) + .collect() +} + +/// Deterministic splitmix64 choices, so a failing seed replays exactly. +pub(crate) fn draw(state: &mut u64, bound: u64) -> usize { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut value = *state; + value = (value ^ (value >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + value = (value ^ (value >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + usize::try_from((value ^ (value >> 31)) % bound).expect("choice fits usize") +} + +pub(crate) fn draw_step(state: &mut u64) -> FaultStep { + FaultStep { + restart: [None, None, None, None, Some(0), Some(1)][draw(state, 6)], + expire: draw(state, 4) == 0, + fault: draw(state, 14).min(7), + cycles: [[true, false], [false, true], [true, true]][draw(state, 3)], + } +} + +/// One process cycle; an error is that process crashing and restarting with a new holder. +async fn cycle_or_restart(store: &ControlStore, state: SchedulerState) -> (SchedulerState, usize) { + let result = scheduler_cycle(store, state).await; + let crashed = usize::from(result.is_err()); + ( + result.map_or_else(|_| SchedulerState::new(Uuid::now_v7()), |(state, _)| state), + crashed, + ) +} + +/// Seeded faults across two schedulers, then recovery; armed steps advance the crash point once fired. +pub(crate) async fn fault_run( + backend: Backend, + seed: u64, + (steps, mut point): (usize, usize), +) -> Option<(FaultAnomalies, BTreeMap<&'static str, usize>, usize)> { + let fixture = open_test_store(backend).await?; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + (0..FAULT_TRIGGERS) + .map(|index| { + ( + format!("orders_{index}"), + json!({"description": "Build orders", "selection": ["orders"]}), + ) + }) + .collect(), + ); + input.documents.insert( + "triggers".into(), + (0..FAULT_TRIGGERS) + .map(|index| { + ( + format!("minutely_{index}"), + json!({"description": "Minutely orders", "job_key": format!("orders_{index}"), + "starts_at": "2026-01-01T00:00:00Z", + "schedule": {"kind": "cron", "expression": "* * * * *", "timezone": "UTC"}}), + ) + }) + .collect(), + ); + publish_snapshot(store, &input).await.expect("publication"); + let trigger_ids: Vec = list_triggers(store, location, "development") + .await + .expect("triggers") + .iter() + .map(|trigger| trigger.trigger_id) + .collect(); + let trigger_id = trigger_ids[0]; + let started = Utc::now() + .duration_trunc(TimeDelta::minutes(1)) + .expect("minute") + - TimeDelta::minutes(FAULT_BACKLOG_MINUTES) + + TimeDelta::milliseconds(250); + for trigger in &trigger_ids { + set_trigger_state(store, *trigger, + "UPDATE sqb_trigger_state SET started_at=$1, last_tick_for=NULL, next_due_at=$1 WHERE trigger_id=$2", + started).await; + } + let second = second_store(store).await; + let handles = [store, &second]; + let mut states = [ + SchedulerState::new(Uuid::now_v7()), + SchedulerState::new(Uuid::now_v7()), + ]; + let faults: Vec = tick_faults() + .into_iter() + .chain([TransactionFault { + description: "no fault", + table: "sqb_ticks", + operation: "INSERT", + condition: "WHEN 1=0", + sqlite_body: "SELECT 1", + postgres_body: "PERFORM 1", + }]) + .collect(); + let mut choices = seed; + let mut counts: BTreeMap<&'static str, usize> = BTreeMap::new(); + let mut anomalies = FaultAnomalies::default(); + for _ in 0..steps { + let step = draw_step(&mut choices); + step.restart + .into_iter() + .for_each(|index| states[index] = SchedulerState::new(Uuid::now_v7())); + // A lapsed lease leaves every current holder frozen on an epoch that must now be fenced. + let stale: Vec = states + .iter() + .filter_map(|state| state.lease.clone()) + .filter(|_| step.expire) + .collect(); + for _ in 0..usize::from(step.expire) { + expire_lease(store).await; + } + let fault = &faults[[point % 7, 7][usize::from(step.fault >= 7)]]; + install_fault(store, fault).await; + let cycle = |index: usize| { + let (handle, mut state, cycles) = + (handles[index], states[index].clone(), step.cycles[index]); + async move { + let mut crashed = 0; + for _ in 0..usize::from(cycles) { + (state, crashed) = cycle_or_restart(handle, state).await; + } + (state, crashed) + } + }; + let ((left, left_crashed), (right, right_crashed)) = tokio::join!(cycle(0), cycle(1)); + states = [left, right]; + remove_fault(store, fault).await; + for lease in &stale { + anomalies.unfenced_stale_writes += usize::from(!matches!( + evaluate_trigger(store, lease, trigger_id).await, + Err(ControlError::Fenced) + )); + } + *counts.entry(fault.description).or_default() += left_crashed + right_crashed; + point += usize::from(step.fault < 7 && left_crashed + right_crashed > 0); + for (name, count) in [ + ("restarts", usize::from(step.restart.is_some())), + ("lease expiries", usize::from(step.expire)), + ("armed faults", usize::from(step.fault < 7)), + ("crashes", left_crashed + right_crashed), + ( + "concurrent cycles", + usize::from(step.cycles == [true, true]), + ), + ("frozen leases fenced", stale.len()), + ] { + *counts.entry(name).or_default() += count; + } + } + // Recovery: faults are gone and leadership lapses; one survivor catches up on the latest due time. + expire_lease(store).await; + let mut survivor = SchedulerState::new(Uuid::now_v7()); + for _ in 0..60 { + survivor = scheduler_cycle(&second, survivor) + .await + .expect("recovery") + .0; + } + let settled = Utc::now(); + scheduler_cycle(&second, survivor) + .await + .expect("final recovery"); + let first = started + .duration_trunc(TimeDelta::minutes(1)) + .expect("minute") + + TimeDelta::minutes(1); + let expected: BTreeSet> = (0..) + .map(|minutes| first + TimeDelta::minutes(minutes)) + .take_while(|time| *time <= settled) + .collect(); + let mut ticks: Vec = Vec::new(); + for trigger in &trigger_ids { + let trigger_ticks = list_ticks(store, *trigger, 1000).await.expect("ticks"); + let times: BTreeSet> = trigger_ticks + .iter() + .map(|tick| tick.scheduled_for) + .collect(); + anomalies.duplicate_ticks += trigger_ticks.len() - times.len(); + anomalies.missing_latest_ticks += usize::from( + expected + .last() + .is_some_and(|latest| !times.contains(latest)), + ); + anomalies.off_schedule_ticks += times + .iter() + .filter(|time| **time < first || time.second() != 0 || time.nanosecond() != 0) + .count(); + ticks.extend(trigger_ticks); + } + let runs: BTreeSet = list_runs(store, (location, "development"), 5_000) + .await + .expect("runs") + .iter() + .map(|run| run.run_id) + .collect(); + let requesting: Vec = ticks.iter().filter_map(|tick| tick.run_id).collect(); + let tick_runs: BTreeSet = requesting.iter().copied().collect(); + anomalies.runs_without_exactly_one_tick = runs.symmetric_difference(&tick_runs).count() + + (requesting.len() - tick_runs.len()) + + ticks + .iter() + .filter(|tick| { + tick.run_id.is_some() == tick.covered_by_run_id.is_some() + || tick + .covered_by_run_id + .is_some_and(|covering| !runs.contains(&covering)) + }) + .count(); + let transaction = begin(store, TransactionMode::Read).await.expect("events"); + let (transaction, rows) = query( + transaction, + "SELECT COUNT(*) AS requested FROM recall_events WHERE event_type='run_requested'", + vec![], + ) + .await + .expect("request events"); + commit(transaction).await.expect("read events"); + let requested = + usize::try_from(rows[0]["requested"].as_i64().expect("count")).expect("nonnegative count"); + anomalies.request_event_mismatch = requested.abs_diff(requesting.len()); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("duplicate"); + let duplicate = query(transaction, + "INSERT INTO sqb_ticks (tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, run_id, covered_by_run_id) \ + VALUES ($1,$2,$3,$3,1,'covered',NULL,NULL)", + vec![Parameter::Id(Uuid::now_v7()), Parameter::Id(trigger_id), Parameter::Time(*expected.last().expect("a due time"))]).await; + anomalies.accepted_duplicate_ticks = usize::from(duplicate.is_ok()); + counts.insert("ticks", ticks.len()); + close_store(second).await; + close_test_store(fixture).await; + Some((anomalies, counts, point)) +} + +/// Run one real scheduler loop until `deadline`, then stop it and return the trigger's ticks. +pub(crate) async fn run_scheduler_until( + store: &ControlStore, + trigger_id: Uuid, + deadline: DateTime, +) -> Vec { + let (stop, shutdown) = tokio::sync::watch::channel(false); + let stopping = async { + tokio::time::sleep((deadline - Utc::now()).to_std().unwrap_or_default()).await; + stop.send(true).expect("stop scheduler"); + }; + let (finished, ()) = tokio::join!( + run_scheduler( + store, + SchedulerState::new(Uuid::now_v7()), + shutdown, + IdleLauncher + ), + stopping + ); + finished.expect("scheduler loop"); + list_ticks(store, trigger_id, 100).await.expect("ticks") +} + +pub(crate) async fn queued_times( + store: &ControlStore, + location: Uuid, +) -> BTreeMap> { + list_runs(store, (location, "development"), 100) + .await + .expect("runs") + .into_iter() + .map(|run| (run.run_id, run.queued_at)) + .collect() +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_contract.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_contract.rs new file mode 100644 index 0000000000..7c80aceab3 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_contract.rs @@ -0,0 +1,34 @@ +//! Crash points must not separate a tick from its queued work and durable cursor. + +use crate::scheduler::tests::contracts::ticks::helpers::transaction_contract; +use crate::scheduler::tests::contracts::ticks::test_types::TickTransactionTestCase; +use crate::store::tests::store_helpers::run; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_two_schedulers_and_faults_between_writes_when_restarting_then_ticks_runs_recall_and_cursor_commit_together() + { + let test_cases = [ + TickTransactionTestCase { + description: "SQLite fenced tick transaction", + backend: Backend::Sqlite, + expected_ticks: 1, + expected_runs: 1, + }, + TickTransactionTestCase { + description: "PostgreSQL fenced tick transaction", + backend: Backend::Postgres, + expected_ticks: 1, + expected_runs: 1, + }, + ]; + for test_case in test_cases { + let observed = run(transaction_contract(test_case.backend)); + assert!( + observed + .is_none_or(|counts| counts == (test_case.expected_ticks, test_case.expected_runs)), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_faults.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_faults.rs new file mode 100644 index 0000000000..be8b906210 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_faults.rs @@ -0,0 +1,67 @@ +//! Two schedulers under random crashes, faults, failovers and frozen leaders tick every due time once. + +use std::collections::BTreeMap; + +use crate::scheduler::tests::contracts::ticks::helpers::{fault_run, tick_faults}; +use crate::scheduler::tests::contracts::ticks::test_types::{ + FaultAnomalies, SchedulerFaultTestCase, +}; +use crate::store::tests::store_helpers::run; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_two_schedulers_under_random_faults_failovers_and_frozen_leaders_when_recovered_then_every_due_tick_happens_once() + { + let test_cases = [ + SchedulerFaultTestCase { + description: "SQLite two-scheduler fault injection", + backend: Backend::Sqlite, + expected_anomalies: FaultAnomalies::default(), + expected_points_without_a_crash: 0, + }, + SchedulerFaultTestCase { + description: "PostgreSQL two-scheduler fault injection", + backend: Backend::Postgres, + expected_anomalies: FaultAnomalies::default(), + expected_points_without_a_crash: 0, + }, + ]; + let seeds: u64 = std::env::var("SQB_SCHEDULER_FAULT_SEEDS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(5); + for test_case in test_cases { + let mut crashes: BTreeMap<&'static str, usize> = BTreeMap::new(); + let mut point = 0; + for seed in 0..seeds { + let Some((anomalies, counts, next)) = + run(fault_run(test_case.backend, seed, (60, point))) + else { + return; + }; + point = next; + println!("{} seed {seed}: {counts:?}", test_case.description); + assert_eq!( + anomalies, test_case.expected_anomalies, + "{} seed {seed}", + test_case.description + ); + for (point, count) in counts { + *crashes.entry(point).or_default() += count; + } + } + // Every crash point inside the tick transaction must actually fire across the seeds run. + let unexercised: Vec<&str> = tick_faults() + .iter() + .map(|fault| fault.description) + .filter(|point| crashes.get(point).copied().unwrap_or_default() == 0) + .collect(); + println!("{} crashes per point: {crashes:?}", test_case.description); + assert_eq!( + unexercised.len(), + test_case.expected_points_without_a_crash, + "{}: {unexercised:?}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs new file mode 100644 index 0000000000..71291a8feb --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_latency.rs @@ -0,0 +1,142 @@ +//! Bounds tick-to-decision latency, where a decision queues or covers a run, and standby catch-up. + +use chrono::{DurationRound, SecondsFormat, TimeDelta, Utc}; +use serde_json::json; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::scheduler::tests::contracts::ticks::helpers::{ + queued_times, run_scheduler_until, snapshot, +}; +use crate::scheduler::tests::contracts::ticks::test_types::SchedulerLatencyTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_running_scheduler_and_a_stopped_leader_when_times_fall_due_then_ticks_queue_within_their_latency_bounds() + { + let test_cases = [ + SchedulerLatencyTestCase { + description: "SQLite scheduler latency", + backend: Backend::Sqlite, + expected_ticks: 3, + expected_ticks_after_failover: 4, + expected_max_tick_to_decision: TimeDelta::milliseconds(500), + expected_max_failover_catch_up: TimeDelta::milliseconds(1_500), + }, + SchedulerLatencyTestCase { + description: "PostgreSQL scheduler latency", + backend: Backend::Postgres, + expected_ticks: 3, + expected_ticks_after_failover: 4, + expected_max_tick_to_decision: TimeDelta::milliseconds(500), + expected_max_failover_catch_up: TimeDelta::milliseconds(1_500), + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let base = Utc::now() + .duration_trunc(TimeDelta::seconds(1)) + .expect("second"); + let at = |seconds| base + TimeDelta::seconds(seconds); + let text = |seconds| at(seconds).to_rfc3339_opts(SecondsFormat::Secs, true); + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), + ); + input.documents.insert( + "triggers".into(), + json!({"calendar_orders": {"description": "Calendar orders", "job_key": "orders", + "schedule": {"kind": "calendar", "times": [text(2), text(3), text(4), text(8), text(9)], + "refresh_at": "2029-01-01T00:00:00Z"}}}), + ); + publish_snapshot(store, &input).await.expect("publication"); + let trigger_id = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + let ticks = run_scheduler_until(store, trigger_id, at(6)).await; + let queued = queued_times(store, location).await; + let earliest = ticks + .iter() + .min_by_key(|tick| tick.scheduled_for) + .expect("ticks"); + let requested = earliest.run_id.expect("the first due time queues a run"); + assert!( + queued[&requested] <= earliest.evaluated_at, + "{}: the run is queued with its tick", + test_case.description + ); + assert!( + ticks + .iter() + .filter(|tick| tick.tick_id != earliest.tick_id) + .all(|tick| tick.covered_by_run_id == Some(requested)), + "{}: later times are covered by the waiting run", + test_case.description + ); + let latencies: Vec = ticks + .iter() + .map(|tick| tick.evaluated_at - tick.scheduled_for) + .collect(); + println!("{} tick-to-decision: {latencies:?}", test_case.description); + assert_eq!( + latencies.len(), + test_case.expected_ticks, + "{}", + test_case.description + ); + assert!( + latencies + .iter() + .all(|latency| *latency <= test_case.expected_max_tick_to_decision), + "{}: {latencies:?}", + test_case.description + ); + // The first leader stopped without releasing its lease while two more times fell due; + // the standby catches up on the latest one only. + // A standby is already polling when that lease lapses, so its own retry wait counts. + tokio::time::sleep((at(10) - Utc::now()).to_std().unwrap_or_default()).await; + let second = second_store(store).await; + let lapse = async { + tokio::time::sleep((at(11) - Utc::now()).to_std().unwrap_or_default()).await; + expire_lease(store).await; + Utc::now() + }; + let (ticks, available) = + tokio::join!(run_scheduler_until(&second, trigger_id, at(14)), lapse); + assert_eq!( + ticks.len(), + test_case.expected_ticks_after_failover, + "{}: the standby ticks only the latest overdue time", + test_case.description + ); + let latest = ticks + .iter() + .find(|tick| tick.scheduled_for == at(9)) + .expect("the latest overdue time is decided"); + let catch_up = latest.evaluated_at - available; + println!("{} failover catch-up: {catch_up:?}", test_case.description); + assert!( + catch_up <= test_case.expected_max_failover_catch_up, + "{}: {catch_up:?}", + test_case.description + ); + close_store(second).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs new file mode 100644 index 0000000000..69106153cc --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs @@ -0,0 +1,101 @@ +//! Embedded and standalone schedulers share takeover and shutdown semantics. + +use std::time::Duration; + +use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; +use crate::scheduler::main::run_scheduler::run_scheduler; +use crate::scheduler::models::SchedulerState; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::scheduler::tests::contracts::ticks::helpers::snapshot; +use crate::scheduler::tests::contracts::ticks::test_types::{ + IdleLauncher, TickTransactionTestCase, +}; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_embedded_and_standalone_schedulers_when_taking_over_then_only_one_drains_due_work_and_idle_shutdown_is_prompt() + { + let test_cases = [ + TickTransactionTestCase { + description: "SQLite scheduler loop", + backend: Backend::Sqlite, + expected_ticks: 1, + expected_runs: 1, + }, + TickTransactionTestCase { + description: "PostgreSQL scheduler loop", + backend: Backend::Postgres, + expected_ticks: 1, + expected_runs: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate loop store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publish due schedule"); + let second = second_store(store).await; + let first_state = SchedulerState::new(Uuid::now_v7()); + let second_state = SchedulerState::new(Uuid::now_v7()); + let (first_state, first) = scheduler_cycle(store, first_state) + .await + .expect("initial cycle"); + assert!(first.leader, "{}", test_case.description); + assert_eq!(first.evaluated, 1); + assert_eq!(first.ticks, test_case.expected_ticks); + assert_eq!(first.wait_for, Duration::from_secs(5)); + let (second_state, follower) = scheduler_cycle(&second, second_state) + .await + .expect("follower cycle"); + assert!(!follower.leader); + assert_eq!(follower.ticks, 0); + expire_lease(store).await; + let (second_state, successor) = scheduler_cycle(&second, second_state) + .await + .expect("takeover cycle"); + assert!(successor.leader); + assert_eq!(successor.ticks, 0, "takeover never repeats durable work"); + assert!( + !scheduler_cycle(store, first_state) + .await + .expect("fenced cycle") + .1 + .leader + ); + let runs = + crate::runs::main::list_runs::list_runs(store, (location, "development"), 100) + .await + .expect("queued run"); + assert_eq!(runs.len(), test_case.expected_runs); + let (stop, shutdown) = tokio::sync::watch::channel(false); + let mut running = + Box::pin(run_scheduler(&second, second_state, shutdown, IdleLauncher)); + tokio::select! { + result = &mut running => panic!("loop ended before shutdown: {result:?}"), + _ = tokio::time::sleep(Duration::from_millis(50)) => {}, + } + stop.send(true).expect("signal shutdown"); + tokio::time::timeout(Duration::from_secs(2), running) + .await + .expect("idle shutdown does not wait for the five-second timer") + .expect("scheduler exits cleanly"); + close_store(second).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_precision.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_precision.rs new file mode 100644 index 0000000000..860c124377 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_precision.rs @@ -0,0 +1,136 @@ +//! A sub-second trigger start or cursor never moves a cron tick off its whole-second occurrence. + +use chrono::{DurationRound, TimeDelta, Timelike, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::scheduler::main::{_acquire_lease::acquire_lease, _evaluate_trigger::evaluate_trigger}; +use crate::scheduler::tests::contracts::ticks::helpers::{set_trigger_state, snapshot, tick_times}; +use crate::scheduler::tests::contracts::ticks::test_types::FirstTickPrecisionTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_sub_second_trigger_start_or_cursor_when_evaluated_then_cron_ticks_land_on_whole_seconds_once() + { + let test_cases = [ + FirstTickPrecisionTestCase { + description: "SQLite first and resumed cron ticks", + backend: Backend::Sqlite, + expected_first_tick_minutes_after_start: 1, + expected_fractional_ticks: 0, + expected_repeated_occurrences: 0, + }, + FirstTickPrecisionTestCase { + description: "PostgreSQL first and resumed cron ticks", + backend: Backend::Postgres, + expected_first_tick_minutes_after_start: 1, + expected_fractional_ticks: 0, + expected_repeated_occurrences: 0, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/first_tick_precision.json", + ); + let case: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("precision fixture")) + .expect("precision case"); + let number = |key: &str| case[key].as_i64().expect("fixture number"); + let mut input = snapshot(location); + input.documents.insert( + "triggers".into(), + json!({"minutely_orders": {"description": "Minutely orders", "job_key": "orders", + "starts_at": "2026-01-01T00:00:00Z", + "schedule": {"kind": "cron", "expression": case["expression"], "timezone": "UTC"}}}), + ); + publish_snapshot(store, &input).await.expect("publication"); + let trigger_id = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + // A sub-second start just after a whole minute must not tick that minute. + let now_minute = Utc::now() + .duration_trunc(TimeDelta::minutes(1)) + .expect("minute"); + let start_minute = now_minute + TimeDelta::minutes(number("start_minutes_ahead")); + let started = + start_minute + TimeDelta::microseconds(number("start_fraction_microseconds")); + set_trigger_state(store, trigger_id, + "UPDATE sqb_trigger_state SET started_at=$1, last_tick_for=NULL, next_due_at=NULL WHERE trigger_id=$2", + started).await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("election") + .expect("lease"); + let first = evaluate_trigger(store, &lease, trigger_id) + .await + .expect("first evaluation") + .expect("trigger"); + assert_eq!( + (first.actions.len(), first.next_due_at), + ( + 0, + Some( + start_minute + + TimeDelta::minutes(test_case.expected_first_tick_minutes_after_start) + ) + ), + "{}", + test_case.description + ); + // A cursor written by an earlier build sits just past the latest occurrence; resuming + // from it must not tick that occurrence again at the whole second. + set_trigger_state( + store, + trigger_id, + "UPDATE sqb_trigger_state SET started_at=$1 WHERE trigger_id=$2", + now_minute - TimeDelta::minutes(10), + ) + .await; + let legacy = + now_minute + TimeDelta::microseconds(number("legacy_cursor_fraction_microseconds")); + set_trigger_state( + store, + trigger_id, + "UPDATE sqb_trigger_state SET last_tick_for=$1 WHERE trigger_id=$2", + legacy, + ) + .await; + evaluate_trigger(store, &lease, trigger_id) + .await + .expect("resumed evaluation") + .expect("trigger"); + let resumed = tick_times(store, trigger_id).await; + assert_eq!( + resumed.iter().filter(|time| **time == now_minute).count(), + test_case.expected_repeated_occurrences, + "{}: {resumed:?}", + test_case.description + ); + assert_eq!( + resumed + .iter() + .filter(|time| time.second() != 0 || time.nanosecond() != 0) + .count(), + test_case.expected_fractional_ticks, + "{}: {resumed:?}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_progress.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_progress.rs new file mode 100644 index 0000000000..713f0abfa4 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_progress.rs @@ -0,0 +1,163 @@ +//! Reset and re-evaluation advance the cursor exactly even when every tick already exists. + +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::_helpers::operator_change::change_trigger_in; +use crate::declarations::main::{ + change_trigger::change_trigger, list_trigger_events::list_trigger_events, + list_triggers::list_triggers, +}; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; +use crate::scheduler::main::{_acquire_lease::acquire_lease, _evaluate_trigger::evaluate_trigger}; +use crate::scheduler::tests::contracts::ticks::helpers::{calendar_time, snapshot}; +use crate::scheduler::tests::contracts::ticks::test_types::TriggerControlsTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::{begin::begin, commit::commit, migrate_store::migrate_store}; +use crate::store::models::TransactionMode; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::logical_state::logical_state; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_reset_existing_ticks_when_re_evaluated_then_cursor_and_due_time_are_exact() { + let test_cases = [ + TriggerControlsTestCase { + description: "SQLite exact reset and re-evaluation", + backend: Backend::Sqlite, + expected_events: 2, + }, + TriggerControlsTestCase { + description: "PostgreSQL exact reset and re-evaluation", + backend: Backend::Postgres, + expected_events: 2, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend).await.expect("backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publication"); + let id = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("election") + .expect("lease"); + evaluate_trigger(store, &lease, id) + .await + .expect("first drain"); + let audit = AuditContext { + actor: "orders-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + change_trigger( + store, + &audit, + &TriggerChangeRequest { + event_id: Uuid::now_v7(), + trigger_id: id, + actor: audit.actor.clone(), + reason: "Replay retained calendar".into(), + change: TriggerChange::ResetCursor, + }, + ) + .await + .expect("reset"); + let reset = list_triggers(store, location, "development") + .await + .expect("reset state") + .remove(0); + assert!(reset.last_tick_for.is_none()); + assert_eq!(reset.next_due_at.expect("first due"), calendar_time(0)); + let fault = TransactionFault { + description: "trigger state failure", + table: "sqb_trigger_state", + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'synthetic trigger state failure')", + postgres_body: "RAISE EXCEPTION 'synthetic trigger state failure'", + }; + install_fault(store, &fault).await; + let before = logical_state(store).await; + let error = evaluate_trigger(store, &lease, id) + .await + .expect_err("fact failure rejects cursor update"); + assert!( + error + .to_string() + .contains("synthetic trigger state failure") + ); + assert_eq!( + logical_state(store).await, + before, + "entire transaction rolled back" + ); + remove_fault(store, &fault).await; + evaluate_trigger(store, &lease, id) + .await + .expect("re-evaluate retained ticks"); + let expected = list_triggers(store, location, "development") + .await + .expect("advanced state"); + assert_eq!( + expected[0].last_tick_for.expect("last due"), + calendar_time(2) + ); + assert_eq!( + expected[0].next_due_at.expect("future due").to_rfc3339(), + "2030-01-01T00:00:00+00:00" + ); + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("backward clock transition"); + transaction.now = "1999-01-01T00:00:00Z" + .parse() + .expect("synthetic database clock"); + let pause = TriggerChangeRequest { + event_id: Uuid::now_v7(), + trigger_id: id, + actor: audit.actor.clone(), + reason: "Pause after clock step".into(), + change: TriggerChange::Pause, + }; + let (transaction, applied) = change_trigger_in(transaction, &audit, &pause) + .await + .expect("causally later pause"); + assert!(applied); + commit(transaction) + .await + .expect("clock-step fact and state"); + let expected = list_triggers(store, location, "development") + .await + .expect("causally latest paused trigger"); + assert_eq!( + expected[0].status, + crate::declarations::models::TriggerStatus::Paused + ); + assert_eq!( + list_trigger_events(store, id, 100) + .await + .expect("operator history") + .len(), + test_case.expected_events, + "{}: reset and pause are the only operator facts", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs new file mode 100644 index 0000000000..f7dbf77a27 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_retention.rs @@ -0,0 +1,285 @@ +//! Expired covered ticks go; a cursor reset replays only the latest occurrence and recreates nothing. + +use chrono::{Datelike, DurationRound, TimeDelta, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::{change_trigger::change_trigger, list_triggers::list_triggers}; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::_helpers::tick_retention::prune_ticks; +use crate::scheduler::main::{ + _acquire_lease::acquire_lease, _evaluate_trigger::evaluate_trigger, list_ticks::list_ticks, +}; +use crate::scheduler::models::TickOutcome; +use crate::scheduler::tests::contracts::ticks::helpers::{ + append_covered_tick, calendar_time, shift_schedule, snapshot, +}; +use crate::scheduler::tests::contracts::ticks::test_types::{ + LatestCatchUpTestCase, TickRetentionTestCase, +}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_expired_covered_ticks_when_pruned_and_cursor_reset_then_run_linked_and_latest_ticks_survive_without_resurrection() + { + let test_cases = [ + TickRetentionTestCase { + description: "SQLite keeps the requested and latest ticks", + backend: Backend::Sqlite, + expected_survivors: 2, + }, + TickRetentionTestCase { + description: "PostgreSQL keeps the requested and latest ticks", + backend: Backend::Postgres, + expected_survivors: 2, + }, + ]; + for test_case in test_cases { + run(async { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/tick_retention.json"); + let case: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("tick fixture")) + .expect("tick case"); + let shift = case["shift_days"].as_i64().expect("shift days"); + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publication"); + let trigger_id = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("election") + .expect("lease"); + evaluate_trigger(store, &lease, trigger_id) + .await + .expect("catch up"); + let outcomes: Vec = list_ticks(store, trigger_id, 10) + .await + .expect("ticks") + .into_iter() + .rev() + .map(|tick| tick.outcome) + .collect(); + assert_eq!( + outcomes, + [TickOutcome::Requested], + "catch-up records only the latest missed time" + ); + let later = calendar_time(2) + TimeDelta::hours(1); + let backlog = case["backlog_ticks"].as_i64().expect("backlog"); + for second in 1..=backlog { + append_covered_tick( + store, + trigger_id, + calendar_time(2) + TimeDelta::seconds(second), + ) + .await; + } + append_covered_tick(store, trigger_id, later).await; + assert_eq!(prune_ticks(store, &lease).await.expect("recent ticks"), 0); + shift_schedule(store, trigger_id, shift).await; + assert_eq!( + prune_ticks(store, &lease).await.expect("bounded pass") as u64, + case["expected_first_pass"].as_u64().expect("first pass"), + "one pass stops at its batch cap so the lease renews before the rest" + ); + assert_eq!( + prune_ticks(store, &lease).await.expect("prune") as u64, + case["expected_pruned"].as_u64().expect("pruned") + ); + assert_eq!(prune_ticks(store, &lease).await.expect("idempotent"), 0); + let survivors = list_ticks(store, trigger_id, 10).await.expect("survivors"); + assert_eq!( + survivors.len(), + test_case.expected_survivors, + "{}", + test_case.description + ); + assert_eq!( + survivors + .iter() + .map(|tick| (tick.scheduled_for, tick.outcome)) + .collect::>(), + [ + (later - TimeDelta::days(shift), TickOutcome::Covered), + ( + calendar_time(2) - TimeDelta::days(shift), + TickOutcome::Requested + ), + ], + "latest and run-linked ticks survive" + ); + let audit = AuditContext { + actor: "orders-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + change_trigger( + store, + &audit, + &TriggerChangeRequest { + event_id: Uuid::now_v7(), + trigger_id, + actor: audit.actor.clone(), + reason: "Replay the retained calendar".into(), + change: TriggerChange::ResetCursor, + }, + ) + .await + .expect("reset"); + let plan = evaluate_trigger(store, &lease, trigger_id) + .await + .expect("re-evaluate") + .expect("trigger"); + let replayed = calendar_time(2) - TimeDelta::days(shift); + assert_eq!( + plan.actions + .iter() + .map(|action| action.scheduled_for) + .collect::>(), + [replayed], + "a reset replays only the latest occurrence" + ); + let ticks = list_ticks(store, trigger_id, 10) + .await + .expect("after reset"); + assert_eq!( + ticks.len(), + test_case.expected_survivors, + "the replayed tick already exists, so nothing is recreated" + ); + let trigger = list_triggers(store, location, "development") + .await + .expect("trigger") + .remove(0); + assert_eq!(trigger.last_tick_for, Some(replayed)); + assert_eq!( + trigger.next_due_at, + Some("2030-01-01T00:00:00Z".parse().expect("future")) + ); + assert!(ticks.iter().all(|tick| tick.scheduled_for < Utc::now())); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_an_old_yearly_schedule_when_first_evaluated_then_latest_catch_up_ticks_only_its_latest_occurrence() + { + let test_cases = [ + LatestCatchUpTestCase { + description: "SQLite latest catch-up ticks an occurrence older than the horizon", + backend: Backend::Sqlite, + expected_ticks: 1, + }, + LatestCatchUpTestCase { + description: "PostgreSQL latest catch-up ticks an occurrence older than the horizon", + backend: Backend::Postgres, + expected_ticks: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/tick_retention.json"); + let case: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("tick fixture")) + .expect("tick case"); + // A yearly date (day 28 at most) well past the horizon, so its latest occurrence is older than it. + let days_ago = case["yearly_latest_days_ago"].as_i64().expect("days ago"); + let past = Utc::now() - TimeDelta::days(days_ago); + let latest = past + .with_day(past.day().min(28)) + .expect("day every year has") + .duration_trunc(TimeDelta::days(1)) + .expect("midnight"); + let mut input = snapshot(location); + input.documents.insert( + "triggers".into(), + json!({"yearly_orders": {"description": "Yearly orders", "job_key": "orders", + "starts_at": case["yearly_starts_at"], + "schedule": {"kind": "cron", "expression": format!("0 0 {} {} *", latest.day(), latest.month()), "timezone": "UTC"}}}), + ); + publish_snapshot(store, &input).await.expect("publication"); + let trigger_id = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("election") + .expect("lease"); + for _ in 0..2 { + evaluate_trigger(store, &lease, trigger_id) + .await + .expect("catch up") + .expect("trigger"); + } + let ticks: Vec<_> = list_ticks(store, trigger_id, 10) + .await + .expect("ticks") + .into_iter() + .map(|tick| (tick.scheduled_for, tick.outcome)) + .collect(); + assert_eq!( + ticks.len(), + test_case.expected_ticks, + "{}", + test_case.description + ); + assert!( + ticks + .iter() + .all(|tick| *tick == (latest, TickOutcome::Requested)), + "only the latest occurrence is ticked: {}", + test_case.description + ); + let runs = list_runs(store, (location, "development"), 10) + .await + .expect("runs"); + assert_eq!( + runs.len(), + test_case.expected_ticks, + "{}", + test_case.description + ); + let trigger = list_triggers(store, location, "development") + .await + .expect("trigger") + .remove(0); + assert_eq!( + trigger.next_due_at, + Some(latest.with_year(latest.year() + 1).expect("next year")), + "{}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs new file mode 100644 index 0000000000..ca93c4cadf --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs @@ -0,0 +1,20 @@ +#[path = "helpers.rs"] +pub(crate) mod helpers; +#[path = "test_contract.rs"] +mod test_contract; +#[path = "test_faults.rs"] +mod test_faults; +#[path = "test_latency.rs"] +mod test_latency; +#[path = "test_loop.rs"] +mod test_loop; +#[path = "test_precision.rs"] +mod test_precision; +#[path = "test_progress.rs"] +mod test_progress; +#[path = "test_retention.rs"] +mod test_retention; +#[path = "test_triggers.rs"] +mod test_triggers; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs new file mode 100644 index 0000000000..b502477298 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs @@ -0,0 +1,366 @@ +//! Operator trigger facts remain consistent across publication and retry boundaries. + +use crate::credentials::models::AuditContext; +use crate::declarations::main::change_trigger::change_trigger; +use crate::declarations::main::list_trigger_events::list_trigger_events; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest, TriggerStatus}; +use crate::scheduler::main::_acquire_lease::acquire_lease; +use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::scheduler::tests::contracts::ticks::helpers::{ + calendar_text, calendar_time, snapshot, trigger_audits, +}; +use crate::scheduler::tests::contracts::ticks::test_types::{ + TickTransactionTestCase, TransactionFault, TriggerControlsTestCase, +}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_state_remain_consistent() + { + let test_cases = [ + TriggerControlsTestCase { + description: "SQLite operator facts", + backend: Backend::Sqlite, + expected_events: 5, + }, + TriggerControlsTestCase { + description: "PostgreSQL operator facts", + backend: Backend::Postgres, + expected_events: 5, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate trigger store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + publish_snapshot(store, &input) + .await + .expect("initial publish"); + let trigger_id = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + let pause = TriggerChangeRequest { + event_id: Uuid::now_v7(), + trigger_id, + actor: "orders-operator".into(), + reason: "maintenance".into(), + change: TriggerChange::Pause, + }; + let audit = AuditContext { + actor: "orders-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + assert!(change_trigger(store, &audit, &pause).await.expect("pause")); + let paused = list_triggers(store, location, "development") + .await + .expect("paused trigger") + .remove(0); + assert_eq!( + paused.status, + TriggerStatus::Paused, + "{}", + test_case.description + ); + assert!(paused.next_due_at.is_none()); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("election") + .expect("lease"); + assert!( + evaluate_trigger(store, &lease, trigger_id) + .await + .expect("paused evaluation") + .is_none() + ); + let enable = TriggerChangeRequest { + event_id: Uuid::now_v7(), + change: TriggerChange::Enable, + ..pause.clone() + }; + for fault in [ + TransactionFault { + description: "operator event and projection", + table: "sqb_trigger_state", + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected state failure')", + postgres_body: "RAISE EXCEPTION 'injected state failure'", + }, + TransactionFault { + description: "operator security audit", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected audit failure')", + postgres_body: "RAISE EXCEPTION 'injected audit failure'", + }, + ] { + install_fault(store, &fault).await; + assert!(change_trigger(store, &audit, &enable).await.is_err()); + remove_fault(store, &fault).await; + assert_eq!( + list_trigger_events(store, trigger_id, 100) + .await + .expect("rolled-back fact") + .len(), + 1 + ); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("rolled back projection")[0] + .status, + TriggerStatus::Paused + ); + } + assert!( + change_trigger(store, &audit, &enable) + .await + .expect("retry enable") + ); + assert!( + !change_trigger(store, &audit, &pause) + .await + .expect("retry old pause is inert") + ); + let conflict = TriggerChangeRequest { + change: TriggerChange::ResetDefault, + ..pause.clone() + }; + assert!( + change_trigger(store, &audit, &conflict).await.is_err(), + "same event ID cannot change content" + ); + input.generation = 2; + input + .documents + .get_mut("triggers") + .expect("trigger document")["daily_orders"]["default_status"] = + serde_json::json!("paused"); + publish_snapshot(store, &input) + .await + .expect("publish a changed code default"); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("retained override")[0] + .status, + TriggerStatus::Running + ); + let reset = TriggerChangeRequest { + event_id: Uuid::now_v7(), + change: TriggerChange::ResetDefault, + ..pause.clone() + }; + change_trigger(store, &audit, &reset) + .await + .expect("restore new code default"); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("default restored")[0] + .status, + TriggerStatus::Paused + ); + let time = calendar_time(1); + let cursor = TriggerChangeRequest { + event_id: Uuid::now_v7(), + change: TriggerChange::SetCursor { time }, + ..pause.clone() + }; + change_trigger(store, &audit, &cursor) + .await + .expect("set durable cursor"); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("cursor set")[0] + .last_tick_for, + Some(time) + ); + let reset_cursor = TriggerChangeRequest { + event_id: Uuid::now_v7(), + change: TriggerChange::ResetCursor, + ..pause.clone() + }; + change_trigger(store, &audit, &reset_cursor) + .await + .expect("reset cursor"); + assert!( + list_triggers(store, location, "development") + .await + .expect("cursor reset")[0] + .last_tick_for + .is_none() + ); + let events = list_trigger_events(store, trigger_id, 100) + .await + .expect("operator history"); + assert_eq!(events.len(), test_case.expected_events); + assert!( + events + .iter() + .all(|event| event.request_json.actor == "orders-operator") + ); + let other_actor = AuditContext { + actor: "customers-operator".into(), + ..audit.clone() + }; + assert!(change_trigger(store, &other_actor, &pause).await.is_err()); + let audits = trigger_audits(store).await; + assert_eq!(audits.len(), test_case.expected_events); + assert!( + audits + .iter() + .all(|row| row["actor"] == "orders-operator" + && row["granting_role"] == "operator") + ); + let cleared = audits + .iter() + .find(|row| row["event_id"] == reset_cursor.event_id.to_string()) + .expect("cursor reset audit"); + assert_eq!(cleared["before_json"]["last_tick_for"], calendar_text(1)); + assert!(cleared["after_json"]["last_tick_for"].is_null()); + assert_eq!(cleared["after_json"]["reason"], "maintenance"); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_calendar_with_unprocessed_due_times_when_refreshed_then_due_ticks_survive_and_only_future_times_are_replaced() + { + use crate::runs::main::list_runs::list_runs; + use crate::scheduler::main::list_ticks::list_ticks; + use serde_json::json; + + let test_cases = [ + TickTransactionTestCase { + description: "SQLite retains due calendar ticks", + backend: Backend::Sqlite, + expected_ticks: 1, + expected_runs: 1, + }, + TickTransactionTestCase { + description: "PostgreSQL retains due calendar ticks", + backend: Backend::Postgres, + expected_ticks: 1, + expected_runs: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + publish_snapshot(store, &input) + .await + .expect("publish pending calendar"); + input.generation = 2; + let future: chrono::DateTime = + "2031-01-01T00:00:00Z".parse().expect("future"); + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = json!( + (0..10_000) + .map(|seconds| future + chrono::TimeDelta::seconds(seconds)) + .collect::>() + ); + assert!( + publish_snapshot(store, &input) + .await + .expect_err("pending ticks cannot be truncated to fit a refreshed horizon") + .to_string() + .contains("let the scheduler drain pending ticks") + ); + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = + json!(["2031-01-01T00:00:00Z"]); + publish_snapshot(store, &input) + .await + .expect("refresh while scheduler is offline"); + let trigger = list_triggers(store, location, "development") + .await + .expect("refreshed trigger") + .remove(0); + assert_eq!( + serde_json::to_value(&trigger.definition_json).expect("calendar")["schedule"]["times"], + json!([ + calendar_text(0), + calendar_text(1), + calendar_text(2), + "2031-01-01T00:00:00Z" + ]), + "{}", + test_case.description + ); + assert_eq!(trigger.next_due_at, Some(calendar_time(0))); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("election") + .expect("lease"); + let plan = evaluate_trigger(store, &lease, trigger.trigger_id) + .await + .expect("catch up") + .expect("trigger"); + assert_eq!(plan.actions.len(), test_case.expected_ticks); + assert_eq!( + list_ticks(store, trigger.trigger_id, 100) + .await + .expect("ticks") + .len(), + test_case.expected_ticks + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("coalesced run") + .len(), + test_case.expected_runs + ); + input.generation = 3; + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = + json!(["2032-01-01T00:00:00Z"]); + publish_snapshot(store, &input) + .await + .expect("refresh after catch-up"); + let trigger = list_triggers(store, location, "development") + .await + .expect("drained trigger") + .remove(0); + assert_eq!( + serde_json::to_value(&trigger.definition_json).expect("calendar")["schedule"]["times"], + json!(["2032-01-01T00:00:00Z"]) + ); + assert_eq!( + list_ticks(store, trigger.trigger_id, 100) + .await + .expect("immutable tick history") + .len(), + test_case.expected_ticks + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs new file mode 100644 index 0000000000..f6cb23edc0 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -0,0 +1,93 @@ +//! Database cases and expected scheduler tick outcomes. + +pub(crate) use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct TickTransactionTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_ticks: usize, + pub(crate) expected_runs: usize, +} + +pub(crate) struct TriggerControlsTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_events: usize, +} + +pub(crate) struct IdleLauncher; + +impl crate::scheduler::types::RunLauncher for IdleLauncher { + async fn poll( + &mut self, + _store: &crate::store::models::ControlStore, + _lease: Option<&crate::scheduler::models::SchedulerLease>, + ) -> Result<(), crate::store::errors::ControlError> { + Ok(()) + } + + async fn shutdown( + &mut self, + _store: &crate::store::models::ControlStore, + _lease: Option<&crate::scheduler::models::SchedulerLease>, + ) -> Result<(), crate::store::errors::ControlError> { + Ok(()) + } +} + +pub(crate) struct TickRetentionTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_survivors: usize, +} + +pub(crate) struct LatestCatchUpTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_ticks: usize, +} + +pub(crate) struct FirstTickPrecisionTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_first_tick_minutes_after_start: i64, + pub(crate) expected_fractional_ticks: usize, + pub(crate) expected_repeated_occurrences: usize, +} + +/// Ways a fault run breaks exactly-once; times passed during an outage are skipped by design. +#[derive(Debug, Default, PartialEq, Eq)] +pub(crate) struct FaultAnomalies { + pub(crate) duplicate_ticks: usize, + pub(crate) missing_latest_ticks: usize, + pub(crate) off_schedule_ticks: usize, + pub(crate) runs_without_exactly_one_tick: usize, + pub(crate) request_event_mismatch: usize, + pub(crate) unfenced_stale_writes: usize, + pub(crate) accepted_duplicate_ticks: usize, +} + +pub(crate) struct SchedulerFaultTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_anomalies: FaultAnomalies, + pub(crate) expected_points_without_a_crash: usize, +} + +/// One randomized step: who restarts, whether the lease lapses, which fault is armed, who cycles. +pub(crate) struct FaultStep { + pub(crate) restart: Option, + pub(crate) expire: bool, + pub(crate) fault: usize, + pub(crate) cycles: [bool; 2], +} + +pub(crate) struct SchedulerLatencyTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_ticks: usize, + pub(crate) expected_ticks_after_failover: usize, + pub(crate) expected_max_tick_to_decision: chrono::TimeDelta, + pub(crate) expected_max_failover_catch_up: chrono::TimeDelta, +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs new file mode 100644 index 0000000000..4960f0ab8f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs @@ -0,0 +1,257 @@ +use chrono::{DateTime, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::declarations::main::change_trigger::change_trigger; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; +use crate::platform::models::PlatformEvent; +use crate::runs::main::{ + advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, + request_job_run::request_job_run, +}; +use crate::runs::models::{ + JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, +}; +use crate::scheduler::models::SchedulerLease; +use crate::sensors::models::SensorEvaluationFact; +use crate::snapshots::models::SnapshotInput; +use crate::store::main::append_control_event::append_control_event; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlStore, TransactionMode}; + +use crate::runs::tests::helpers::snapshot as base_snapshot; + +pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { + let mut input = base_snapshot(location_id); + input.documents.insert( + "jobs".into(), + json!({ + "orders": {"description": "Refresh orders", "selection": ["orders"]} + }), + ); + input.documents.insert( + "triggers".into(), + json!({ + "health_orders": { + "description": "Refresh reporting on overdue ticks", "job_key": "orders", + "when": {"kind": "platform_event", "kinds": ["tick_overdue", "location_broken", "launch_failed"]}, + "minimum_interval_seconds": 1 + } + }), + ); + input +} + +pub(crate) async fn append_events( + store: &ControlStore, + indices: std::ops::Range, + occurred_at: DateTime, +) { + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("event transaction"); + let payloads = [ + PlatformEvent::TickOverdue { + trigger_id: Uuid::from_u128(7), + scheduled_for: occurred_at, + }, + PlatformEvent::Heartbeat { + replica_id: Uuid::from_u128(8), + }, + PlatformEvent::LocationBroken { + location_id: Uuid::from_u128(9), + target: "development".into(), + last_seen_at: occurred_at, + }, + PlatformEvent::Heartbeat { + replica_id: Uuid::from_u128(10), + }, + PlatformEvent::LaunchFailed { + run_id: Uuid::from_u128(11), + claim_id: Uuid::from_u128(12), + code: "local_spawn_failed".into(), + }, + PlatformEvent::Heartbeat { + replica_id: Uuid::from_u128(13), + }, + ]; + for index in indices { + let platform = &payloads[(index % 6) as usize]; + let (next, appended) = append_control_event( + transaction, + ControlEvent { + event_id: Uuid::from_u128(1000 + index), + invocation_id: "platform/test-health".into(), + event_type: platform.event_type(), + producer: "platform", + occurred_at, + run_id: None, + payload: json!({"platform": platform}), + }, + ) + .await + .expect("append platform observation"); + assert!(appended.inserted); + transaction = next; + } + commit(transaction).await.expect("commit observations"); +} + +pub(crate) async fn evaluations(store: &ControlStore) -> Vec { + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await.expect("read"), + "SELECT fact_json FROM sqb_sensor_evaluation_events ORDER BY occurred_at, event_id", + vec![], + ) + .await + .expect("evaluation history"); + commit(transaction).await.expect("commit"); + rows.into_iter() + .map(|row| serde_json::from_value(row["fact_json"].clone()).expect("typed attempt fact")) + .collect() +} + +pub(crate) async fn change(store: &ControlStore, trigger_id: Uuid, change: TriggerChange) { + let event_id = Uuid::now_v7(); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: event_id, + ip_address: None, + }; + let request = TriggerChangeRequest { + event_id, + trigger_id, + actor: context.actor.clone(), + reason: "Test operator change".into(), + change, + }; + assert!( + change_trigger(store, &context, &request) + .await + .expect("apply operator fact") + ); + assert!( + !change_trigger(store, &context, &request) + .await + .expect("retry operator fact") + ); +} + +/// Real request, claim, node facts and finish, including the virtual group parent. +pub(crate) async fn finish_run( + store: &ControlStore, + lease: &SchedulerLease, + scope: (Uuid, &str), + job_key: &str, + outcome: RunOutcome, + nodes: [NodeOutcome; 2], + started: [bool; 2], +) -> Uuid { + let run_id = Uuid::now_v7(); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + request_job_run( + store, + &context, + &JobRunRequest { + run_id, + location_id: scope.0, + target: scope.1.into(), + job_key: job_key.into(), + reason: "Exercise terminal sensor scope".into(), + priority: 10, + window: None, + }, + ) + .await + .expect("request source run"); + finish_claimed_run(store, lease, scope, run_id, outcome, nodes, started).await; + run_id +} + +/// Execute only the frozen selection, including a from-failure rerun's reduced scope. +pub(crate) async fn finish_claimed_run( + store: &ControlStore, + lease: &SchedulerLease, + scope: (Uuid, &str), + run_id: Uuid, + outcome: RunOutcome, + nodes: [NodeOutcome; 2], + started: [bool; 2], +) { + let limits = std::collections::BTreeMap::new(); + advance_groups(store, lease, (&limits, scope)) + .await + .expect("advance parent"); + let claim = claim_next_run(store, lease, &limits, Some(scope)) + .await + .expect("claim source") + .expect("source ready"); + assert!(claim.request.run_id == run_id || claim.request.parent_run_id == Some(run_id)); + let worker = RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }; + let mut changes = vec![RunChange::Started]; + for (index, key) in ["customers", "orders"] + .into_iter() + .enumerate() + .filter(|(_, key)| claim.request.selection.contains(*key)) + { + changes.extend(started[index].then_some(RunChange::NodeStarted { + node_key: key.into(), + })); + changes.push(RunChange::NodeFinished { + node_key: key.into(), + outcome: nodes[index], + rows_affected: None, + error_code: None, + skipped_because_of: None, + }); + } + for change in changes { + change_run( + store, + worker, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await + .expect("publish worker evidence"); + } + let terminal = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Finished { + outcome, + reason: "Terminal outcome fixture".into(), + exit_code: None, + }, + }; + assert!( + change_run(store, RunAuthority::Scheduler(lease), &terminal) + .await + .expect("finish") + ); + assert!( + !change_run(store, RunAuthority::Scheduler(lease), &terminal) + .await + .expect("retry finish") + ); + advance_groups(store, lease, (&limits, scope)) + .await + .expect("complete parent"); +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs new file mode 100644 index 0000000000..e97b015a8c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs @@ -0,0 +1,22 @@ +//! Real stores prove atomic sensor consumption and recovery across scheduler ownership changes. + +#[path = "helpers.rs"] +pub(crate) mod helpers; +#[path = "test_custom.rs"] +mod test_custom; +#[path = "test_delivery.rs"] +mod test_delivery; +#[path = "test_execution_bounds.rs"] +mod test_execution_bounds; +#[path = "test_nodes.rs"] +mod test_nodes; +#[path = "test_recovery.rs"] +mod test_recovery; +#[path = "test_runs.rs"] +mod test_runs; +#[path = "test_sources.rs"] +mod test_sources; +#[path = "test_transitions.rs"] +mod test_transitions; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs b/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs new file mode 100644 index 0000000000..ae19f87689 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs @@ -0,0 +1,442 @@ +//! The real outbound HTTP boundary returns data, never scheduler authority. + +use std::time::Duration; + +use chrono::{TimeDelta, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::credentials::main::tokens::issue_token::issue_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::platform::main::connect_location_work::connect_location_work; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::models::{ + LocationHealth, LocationHealthReport, LocationWork, LocationWorkOutcome, LocationWorkPoll, + LocationWorkResult, +}; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; +use crate::sensors::main::complete_custom_sensor::complete_custom_sensor; +use crate::sensors::models::{CustomSensorOutcome, SensorEvaluationOutcome}; +use crate::sensors::tests::helpers::{evaluations, snapshot}; +use crate::sensors::tests::test_types::{CustomSensorResultCase, SensorTestCase}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_custom_sensor_work_when_results_retry_then_cursor_runs_and_outcome_commit_atomically() { + let test_cases = [ + SensorTestCase { + description: "SQLite custom sensor protocol", + backend: Backend::Sqlite, + expected_runs: 2, + }, + SensorTestCase { + description: "PostgreSQL custom sensor protocol", + backend: Backend::Postgres, + expected_runs: 2, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.documents.get_mut("triggers").expect("triggers")["health_orders"]["when"] = + json!({"kind": "custom", "function": "orders_ready", "code_hash": "orders_v1"}); + publish_snapshot(store, &input) + .await + .expect("publish custom sensor data"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let token = issue_token( + store, + &context, + CredentialScope::Location { + location_id: location, + }, + Utc::now() + TimeDelta::hours(1), + ) + .await + .expect("scoped token"); + let api = start_api(store).await; + let client = + connect_location_work((&api.url, &token.bearer), location).expect("client"); + let poll = LocationWorkPoll { + session_id: Uuid::now_v7(), + target: "development".into(), + }; + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: poll.session_id, + target: poll.target.clone(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("register location instance"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let point = TransactionFault { + description: "request and outbound work are atomic", + table: "sqb_location_work", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected sensor request failure')", + postgres_body: "RAISE EXCEPTION 'injected sensor request failure'", + }; + install_fault(store, &point).await; + assert!( + begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .is_err() + ); + remove_fault(store, &point).await; + assert!( + evaluations(store).await.is_empty(), + "request fact rolled back with work" + ); + let requested = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("begin") + .expect("due"); + assert_eq!( + complete_custom_sensor(store, &lease, &requested) + .await + .expect("still waiting"), + 0 + ); + assert_eq!( + evaluations(store).await.len(), + 1, + "no invented terminal phase while waiting" + ); + let claim = client.poll(&poll).await.expect("poll").expect("work"); + assert_eq!(claim.request.request_id, requested.evaluation_id); + assert_eq!( + claim.request.work, + LocationWork::EvaluateSensor { + evaluation: Box::new(requested.clone()) + } + ); + assert_eq!( + client.poll(&poll).await.expect("lost poll response"), + Some(claim.clone()) + ); + let wrong = LocationWorkResult { + claim_id: claim.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::Connection { healthy: true }, + }; + assert!( + client + .complete(requested.evaluation_id, &wrong) + .await + .is_err(), + "wrong result kind" + ); + let empty_key = LocationWorkResult { + outcome: LocationWorkOutcome::SensorEvaluation { + outcome: CustomSensorOutcome::Completed { + cursor: Some("unsafe".into()), + run_keys: vec![" ".into()], + }, + }, + ..wrong.clone() + }; + assert!( + client + .complete(requested.evaluation_id, &empty_key) + .await + .is_err(), + "empty keys fail closed" + ); + let result = LocationWorkResult { + outcome: LocationWorkOutcome::SensorEvaluation { + outcome: CustomSensorOutcome::Completed { + cursor: Some("orders/page-2".into()), + run_keys: vec!["orders-2".into(), "orders-1".into(), "orders-2".into()], + }, + }, + ..wrong + }; + assert!( + client + .complete(requested.evaluation_id, &result) + .await + .expect("durable location result") + ); + assert!( + !client + .complete(requested.evaluation_id, &result) + .await + .expect("uncertain acknowledgement retry") + ); + assert!( + list_runs(store, (location, "development"), 100) + .await + .expect("runs") + .is_empty(), + "location result alone has no queue authority" + ); + for table in [ + "sqb_runs", + "sqb_run_requests", + "sqb_sensor_evaluation_events", + ] { + let point = TransactionFault { + description: "custom sensor atomic publication", + table, + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected sensor failure')", + postgres_body: "RAISE EXCEPTION 'injected sensor failure'", + }; + install_fault(store, &point).await; + assert!( + complete_custom_sensor(store, &lease, &requested) + .await + .is_err() + ); + remove_fault(store, &point).await; + assert!( + list_runs(store, (location, "development"), 100) + .await + .expect("rollback runs") + .is_empty() + ); + assert_eq!( + evaluations(store).await.len(), + 1, + "outcome rolled back with cursor and queue" + ); + assert_eq!( + begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("unchanged cursor"), + Some(requested.clone()) + ); + } + let replica = second_store(store).await; + assert_eq!( + complete_custom_sensor(&replica, &lease, &requested) + .await + .expect("replica commit"), + test_case.expected_runs + ); + assert_eq!( + complete_custom_sensor(store, &lease, &requested) + .await + .expect("retry commit"), + 0 + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("dedup runs") + .len(), + test_case.expected_runs, + "{}", + test_case.description + ); + let facts = evaluations(store).await; + let SensorEvaluationOutcome::Completed { cursor, run_ids } = + &facts.last().expect("completed").outcome + else { + panic!("completed phase required"); + }; + assert_eq!(cursor.custom.as_deref(), Some("orders/page-2")); + assert_eq!(cursor.event_storage_id, requested.cursor.event_storage_id); + assert_eq!(run_ids.len(), test_case.expected_runs); + for result_case in [ + CustomSensorResultCase { + outcome: CustomSensorOutcome::Completed { + cursor: Some("orders/page-2".into()), + run_keys: vec!["orders-2".into(), "orders-1".into()], + }, + expected_phase: "completed", + expected_cursor: Some("orders/page-2"), + }, + CustomSensorResultCase { + outcome: CustomSensorOutcome::Failed, + expected_phase: "failed", + expected_cursor: None, + }, + CustomSensorResultCase { + outcome: CustomSensorOutcome::TimedOut, + expected_phase: "timed_out", + expected_cursor: None, + }, + CustomSensorResultCase { + outcome: CustomSensorOutcome::Skipped { + cursor: Some("orders/page-3".into()), + reason: "No new orders".into(), + }, + expected_phase: "completed", + expected_cursor: Some("orders/page-3"), + }, + ] { + tokio::time::sleep(Duration::from_secs(1)).await; + let pending = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("next attempt") + .expect("due"); + assert_eq!( + pending.cursor.custom.as_deref(), + Some("orders/page-2"), + "failed and timeout leave cursor unchanged" + ); + let claim = client + .poll(&poll) + .await + .expect("next poll") + .expect("next work"); + assert_eq!(claim.request.request_id, pending.evaluation_id); + let result = LocationWorkResult { + claim_id: claim.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::SensorEvaluation { + outcome: result_case.outcome, + }, + }; + assert!( + client + .complete(pending.evaluation_id, &result) + .await + .expect("complete callback") + ); + complete_custom_sensor(store, &lease, &pending) + .await + .expect("apply phase"); + let facts = evaluations(store).await; + let outcome = &facts.last().expect("terminal evaluation phase").outcome; + assert_eq!(outcome.phase(), result_case.expected_phase); + let payload = serde_json::to_value(outcome).expect("outcome payload"); + assert_eq!( + payload["cursor"]["custom"].as_str(), + result_case.expected_cursor + ); + } + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("still deduped") + .len(), + test_case.expected_runs + ); + tokio::time::sleep(Duration::from_secs(1)).await; + let stale = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request before takeover") + .expect("due"); + let stale_claim = client + .poll(&poll) + .await + .expect("claim before takeover") + .expect("work"); + assert_eq!(stale_claim.request.request_id, stale.evaluation_id); + expire_lease(store).await; + assert!( + client + .poll(&poll) + .await + .expect("no expired-epoch dispatch") + .is_none() + ); + assert!(matches!( + complete_custom_sensor(store, &lease, &stale).await, + Err(ControlError::Fenced) + )); + let successor = acquire_lease(&replica, Uuid::now_v7()) + .await + .expect("takeover") + .expect("leader"); + let late = LocationWorkResult { + claim_id: stale_claim.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::SensorEvaluation { + outcome: CustomSensorOutcome::Completed { + cursor: Some("uncommitted/stale".into()), + run_keys: vec!["must-not-queue".into()], + }, + }, + }; + assert!( + client + .complete(stale.evaluation_id, &late) + .await + .expect("late location result") + ); + assert!(matches!( + complete_custom_sensor(&replica, &successor, &stale).await, + Err(ControlError::Fenced) + )); + tokio::time::sleep(Duration::from_secs(1)).await; + let recovered = begin_sensor_evaluation(&replica, &successor, trigger.trigger_id) + .await + .expect("recover abandoned callback") + .expect("due after minimum interval"); + assert_eq!(recovered.cursor.custom.as_deref(), Some("orders/page-3")); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("no stale queued request") + .len(), + test_case.expected_runs + ); + assert_eq!( + evaluations(store) + .await + .iter() + .filter(|fact| fact.request.evaluation_id == stale.evaluation_id + && fact.outcome == SensorEvaluationOutcome::Abandoned) + .count(), + 1 + ); + assert_eq!( + client + .poll(&poll) + .await + .expect("new epoch work") + .expect("new claim") + .request + .request_id, + recovered.evaluation_id, + "retired evaluation must not block or execute before current work" + ); + close_store(replica).await; + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs b/crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs new file mode 100644 index 0000000000..1e132090e2 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs @@ -0,0 +1,182 @@ +use std::collections::BTreeSet; +use std::time::Duration; + +use chrono::{TimeDelta, Utc}; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::TriggerChange; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; +use crate::sensors::main::complete_builtin_sensor::complete_builtin_sensor; +use crate::sensors::models::SensorEvaluationOutcome; +use crate::sensors::tests::helpers::{append_events, change, evaluations, snapshot}; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_paged_platform_events_when_sensor_publication_fails_then_retry_is_atomic_and_reset_deduplicates() + { + let test_cases = [ + SensorTestCase { + description: "SQLite sensor delivery", + backend: Backend::Sqlite, + expected_runs: 54, + }, + SensorTestCase { + description: "PostgreSQL sensor delivery", + backend: Backend::Postgres, + expected_runs: 54, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publish sensor"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let now = Utc::now(); + append_events(store, 0..105, now).await; + let requested = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request") + .expect("due"); + let replica = second_store(store).await; + assert_eq!( + begin_sensor_evaluation(&replica, &lease, trigger.trigger_id) + .await + .expect("recover pending"), + Some(requested.clone()) + ); + for table in [ + "sqb_runs", + "sqb_run_requests", + "sqb_sensor_evaluation_events", + ] { + let point = TransactionFault { + description: "fail sensor transaction", + table, + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected sensor failure')", + postgres_body: "RAISE EXCEPTION 'injected sensor failure'", + }; + install_fault(store, &point).await; + assert!( + complete_builtin_sensor(store, &lease, &requested) + .await + .is_err(), + "{table}" + ); + remove_fault(store, &point).await; + assert!( + list_runs(store, (location, "development"), 1000) + .await + .expect("rolled back runs") + .is_empty(), + "{table}" + ); + assert_eq!( + evaluations(store).await.len(), + 1, + "only requested survives {table}" + ); + } + assert_eq!( + complete_builtin_sensor(&replica, &lease, &requested) + .await + .expect("retry first page"), + 50 + ); + assert_eq!( + complete_builtin_sensor(store, &lease, &requested) + .await + .expect("retry completion"), + 0 + ); + tokio::time::sleep(Duration::from_secs(1)).await; + let second = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("next page") + .expect("due"); + assert!(second.cursor.event_storage_id > 0); + assert_eq!( + complete_builtin_sensor(store, &lease, &second) + .await + .expect("remaining page"), + 3 + ); + // Committed later, but its occurrence precedes the entire first page. + append_events(store, 106..107, now - TimeDelta::days(1)).await; + tokio::time::sleep(Duration::from_secs(1)).await; + let late = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("late request") + .expect("due"); + assert!(late.cursor.event_storage_id > second.cursor.event_storage_id); + assert_eq!( + complete_builtin_sensor(store, &lease, &late) + .await + .expect("late arrival"), + 1 + ); + change(store, trigger.trigger_id, TriggerChange::ResetCursor).await; + tokio::time::sleep(Duration::from_secs(1)).await; + let replay = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("replay") + .expect("due"); + assert_eq!(replay.cursor.event_storage_id, 0); + assert_eq!( + complete_builtin_sensor(store, &lease, &replay) + .await + .expect("replay retained first page"), + 50 + ); + let runs = list_runs(store, (location, "development"), 1000) + .await + .expect("queued runs"); + assert_eq!( + runs.len(), + test_case.expected_runs, + "{}: reset never requeues an existing run key", + test_case.description + ); + assert!( + runs.iter() + .all(|run| run.selection_json == BTreeSet::from(["orders".into()])) + ); + let facts = evaluations(store).await; + assert_eq!(facts.len(), 8); + assert!(facts.iter().all(|fact| matches!( + fact.outcome, + SensorEvaluationOutcome::Requested | SensorEvaluationOutcome::Completed { .. } + ))); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs b/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs new file mode 100644 index 0000000000..87340bf331 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs @@ -0,0 +1,187 @@ +//! A wide group forces multiple predicate chunks; the last node is not optional. + +use std::time::Duration; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::runs::main::{ + advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, + request_job_run::request_job_run, +}; +use crate::runs::models::{ + JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, +}; +use crate::scheduler::main::renew_lease::renew_lease; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::{ + begin_sensor_evaluation::begin_sensor_evaluation, + complete_builtin_sensor::complete_builtin_sensor, +}; +use crate::sensors::tests::helpers::snapshot; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_wide_group_when_only_the_last_node_has_not_started_then_all_chunks_must_match_within_a_bounded_evaluation() + { + let test_cases = [ + SensorTestCase { + description: "SQLite wide executed-node predicate", + backend: Backend::Sqlite, + expected_runs: 1, + }, + SensorTestCase { + description: "PostgreSQL wide executed-node predicate", + backend: Backend::Postgres, + expected_runs: 1, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + let node = input.nodes["orders"].clone(); + let selection: Vec = (0..501) + .map(|index| format!("model:orders_{index:04}")) + .collect(); + input + .nodes + .extend(selection.iter().map(|key| (key.clone(), node.clone()))); + input.documents.insert( + "jobs".into(), + json!({ + "source": {"description": "Wide source", "selection": selection}, + "destination": {"description": "Destination", "selection": ["orders"]} + }), + ); + input.documents.insert("groups".into(), json!({ + "wide_orders": {"description": "Wide group", "selection": selection, "jobs": {"source": []}} + })); + input.documents.insert("triggers".into(), json!({ + "executed": {"description": "All source nodes must execute", "job_key": "destination", + "when": {"kind": "run_event", "job_key": "wide_orders", "kinds": ["completed"], "executed_successfully": selection}, + "minimum_interval_seconds": 1} + })); + publish_snapshot(store, &input) + .await + .expect("publish wide predicate"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = std::collections::BTreeMap::new(); + let scope = (location, "development"); + for include_last in [true, false] { + let parent = Uuid::now_v7(); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + request_job_run( + store, + &context, + &JobRunRequest { + run_id: parent, + location_id: location, + target: "development".into(), + job_key: "wide_orders".into(), + reason: "Exercise wide execution evidence".into(), + priority: 10, + window: None, + }, + ) + .await + .expect("request"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("admit parent"); + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim") + .expect("child"); + assert_eq!(claim.request.parent_run_id, Some(parent)); + let changes = std::iter::once(RunChange::Started) + .chain( + selection + .iter() + .filter(|key| include_last || key.as_str() != "model:orders_0500") + .map(|key| RunChange::NodeStarted { + node_key: key.clone(), + }), + ) + .chain(selection.iter().map(|key| RunChange::NodeFinished { + node_key: key.clone(), + outcome: NodeOutcome::Success, + rows_affected: None, + error_code: None, + skipped_because_of: None, + })) + .chain(std::iter::once(RunChange::Finished { + outcome: RunOutcome::Success, + reason: "All logical nodes succeed".into(), + exit_code: None, + })) + .collect::>(); + for batch in changes.chunks(100) { + // Evidence setup must keep its scheduler alive, just like the real loop. + renew_lease(store, &lease) + .await + .expect("renew during setup"); + for change in batch { + change_run( + store, + RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change: change.clone(), + }, + ) + .await + .expect("real node evidence"); + } + } + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish parent"); + } + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + let evaluation = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request evaluation") + .expect("due"); + let count = tokio::time::timeout( + Duration::from_secs(5), + complete_builtin_sensor(store, &lease, &evaluation), + ) + .await + .expect("wide predicate must not hold the write lock unboundedly") + .expect("complete"); + assert_eq!(count, test_case.expected_runs, "{}", test_case.description); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs b/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs new file mode 100644 index 0000000000..a4297e759c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs @@ -0,0 +1,375 @@ +//! Real lifecycle facts exercise any-run rebuilds, early delivery and paged atomic recovery. + +use std::collections::BTreeMap; +use std::time::Duration; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::TriggerChange; +use crate::runs::main::{ + change_run::change_run, claim_next_run::claim_next_run, list_runs::list_runs, + request_job_run::request_job_run, +}; +use crate::runs::models::{ + JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, +}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::{ + begin_sensor_evaluation::begin_sensor_evaluation, + complete_builtin_sensor::complete_builtin_sensor, +}; +use crate::sensors::tests::helpers::{change, finish_run, snapshot}; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_any_run_rebuilds_when_consuming_then_only_genuine_scoped_successes_queue_once_per_finish() +{ + let test_cases = [ + SensorTestCase { + description: "SQLite any-run node successes", + backend: Backend::Sqlite, + expected_runs: 8, + }, + SensorTestCase { + description: "PostgreSQL any-run node successes", + backend: Backend::Postgres, + expected_runs: 8, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + migrate_store(store).await.expect("warm migration"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "products", "test-admin") + .await + .expect("other location"); + let mut input = snapshot(location); + input.documents.insert("jobs".into(), json!({ + "orders": {"description": "Source orders", "selection": ["orders", "customers"]}, + "inventory": {"description": "Sensor destination", "selection": ["orders", "customers"]} + })); + input.documents.insert("groups".into(), json!({ + "daily_orders": {"description": "Source group", "selection": ["orders", "customers"], "jobs": {"orders": []}} + })); + input.documents.insert("triggers".into(), json!({ + "rebuilt_orders": {"description": "Each genuine rebuild", "job_key": "inventory", + "when": {"kind": "node_event", "selection": ["orders", "customers"], "kinds": ["success"]}, + "minimum_interval_seconds": 1} + })); + publish_snapshot(store, &input).await.expect("publish"); + let mut isolated = input.clone(); + isolated.location_id = other; + publish_snapshot(store, &isolated) + .await + .expect("other location snapshot"); + isolated.location_id = location; + isolated.target = "staging".into(); + publish_snapshot(store, &isolated) + .await + .expect("other target snapshot"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + for (outcome, nodes, started) in [ + (RunOutcome::Success, [NodeOutcome::Success; 2], [true; 2]), + (RunOutcome::Success, [NodeOutcome::Success; 2], [false; 2]), + ( + RunOutcome::Warn, + [NodeOutcome::Warn, NodeOutcome::Success], + [true; 2], + ), + (RunOutcome::Failed, [NodeOutcome::Failed; 2], [true; 2]), + ( + RunOutcome::Partial, + [NodeOutcome::Skipped, NodeOutcome::Success], + [false, true], + ), + ( + RunOutcome::Cancelled, + [NodeOutcome::Cancelled; 2], + [true; 2], + ), + ] { + finish_run( + store, + &lease, + (location, "development"), + "orders", + outcome, + nodes, + started, + ) + .await; + } + for scope in [(other, "development"), (location, "staging")] { + finish_run( + store, + &lease, + scope, + "orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + [true; 2], + ) + .await; + } + finish_run( + store, + &lease, + (location, "development"), + "daily_orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + [true; 2], + ) + .await; + for _ in 0..46 { + finish_run( + store, + &lease, + (location, "development"), + "orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + [false; 2], + ) + .await; + } + let run_id = Uuid::now_v7(); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + request_job_run( + store, + &context, + &JobRunRequest { + run_id, + location_id: location, + target: "development".into(), + job_key: "orders".into(), + reason: "Deliver before whole-run completion".into(), + priority: 10, + window: None, + }, + ) + .await + .expect("request running source"); + let claim = claim_next_run( + store, + &lease, + &BTreeMap::new(), + Some((location, "development")), + ) + .await + .expect("claim") + .expect("ready"); + assert_eq!(claim.request.run_id, run_id); + let authority = RunAuthority::Worker { + run_id, + claim_id: claim.claim_id, + }; + let changes = [ + RunChange::Started, + RunChange::NodeStarted { + node_key: "orders".into(), + }, + RunChange::NodeFinished { + node_key: "orders".into(), + outcome: NodeOutcome::Success, + rows_affected: None, + error_code: None, + skipped_because_of: None, + }, + ]; + for change in changes { + let request = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id, + claim_id: Some(claim.claim_id), + change, + }; + let mut wrong = request.clone(); + wrong.claim_id = Some(Uuid::now_v7()); + assert!( + change_run(store, authority, &wrong).await.is_err(), + "another claim cannot supply start/finish evidence" + ); + assert!( + change_run(store, authority, &request) + .await + .expect("worker fact") + ); + assert!( + !change_run(store, authority, &request) + .await + .expect("duplicate worker fact") + ); + } + let pending = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request evaluation") + .expect("due"); + let replica = second_store(store).await; + let fault = TransactionFault { + description: "node cursor rollback", + table: "sqb_trigger_state", + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected node cursor failure')", + postgres_body: "RAISE EXCEPTION 'injected node cursor failure'", + }; + install_fault(store, &fault).await; + assert!( + complete_builtin_sensor(store, &lease, &pending) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert!( + list_runs(store, (location, "development"), 1000) + .await + .expect("rollback") + .iter() + .all(|run| run.job_key.as_deref() != Some("inventory")) + ); + assert_eq!( + begin_sensor_evaluation(&replica, &lease, trigger.trigger_id) + .await + .expect("recover pending"), + Some(pending.clone()) + ); + assert_eq!( + tokio::time::timeout( + Duration::from_secs(5), + complete_builtin_sensor(&replica, &lease, &pending), + ) + .await + .expect("bounded 100-row page") + .expect("first page"), + 7 + ); + assert_eq!( + complete_builtin_sensor(store, &lease, &pending) + .await + .expect("duplicate completion"), + 0 + ); + tokio::time::sleep(Duration::from_secs(1)).await; + let next = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("second evaluation") + .expect("due"); + assert!(next.cursor.event_storage_id > 0); + assert_eq!( + complete_builtin_sensor(store, &lease, &next) + .await + .expect("second page"), + 0, + "unstarted finishes never fill a page" + ); + for change in [ + RunChange::NodeStarted { + node_key: "customers".into(), + }, + RunChange::NodeFinished { + node_key: "customers".into(), + outcome: NodeOutcome::Success, + rows_affected: None, + error_code: None, + skipped_because_of: None, + }, + ] { + change_run( + store, + authority, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await + .expect("later node in the same run"); + } + tokio::time::sleep(Duration::from_secs(1)).await; + let later = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("later evaluation") + .expect("due"); + assert_eq!( + complete_builtin_sensor(store, &lease, &later) + .await + .expect("later finish"), + 1, + "distinct successful finishes in the same run must not collapse" + ); + let runs = list_runs(store, (location, "development"), 1000) + .await + .expect("runs"); + assert_eq!( + runs.iter() + .filter(|run| run.job_key.as_deref() == Some("inventory")) + .count(), + test_case.expected_runs, + "{}", + test_case.description + ); + assert_eq!( + runs.iter() + .find(|run| run.run_id == run_id) + .expect("source retained") + .status, + crate::runs::models::RunStatus::Running + ); + change(store, trigger.trigger_id, TriggerChange::ResetCursor).await; + for _ in 0..2 { + tokio::time::sleep(Duration::from_secs(1)).await; + let replay = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("replay") + .expect("due"); + complete_builtin_sensor(store, &lease, &replay) + .await + .expect("deduplicated replay"); + } + assert_eq!( + list_runs(store, (location, "development"), 1000) + .await + .expect("replayed runs") + .iter() + .filter(|run| run.job_key.as_deref() == Some("inventory")) + .count(), + test_case.expected_runs + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs b/crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs new file mode 100644 index 0000000000..7b8ba79ce8 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs @@ -0,0 +1,252 @@ +use std::time::Duration; + +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::{TriggerChange, TriggerStatus}; +use crate::platform::_helpers::retention::retain; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; +use crate::sensors::main::complete_builtin_sensor::complete_builtin_sensor; +use crate::sensors::models::SensorEvaluationOutcome; +use crate::sensors::tests::helpers::{append_events, change, evaluations, snapshot}; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_crashed_sensor_when_leadership_or_snapshot_changes_then_stale_results_cannot_queue_runs() +{ + let test_cases = [ + SensorTestCase { + description: "SQLite sensor recovery", + backend: Backend::Sqlite, + expected_runs: 1, + }, + SensorTestCase { + description: "PostgreSQL sensor recovery", + backend: Backend::Postgres, + expected_runs: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + publish_snapshot(store, &input) + .await + .expect("publish sensor"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + append_events(store, 0..1, chrono::Utc::now()).await; + let first = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let pending = begin_sensor_evaluation(store, &first, trigger.trigger_id) + .await + .expect("request") + .expect("due"); + expire_lease(store).await; + let replica = second_store(store).await; + let successor = acquire_lease(&replica, Uuid::now_v7()) + .await + .expect("takeover") + .expect("new leader"); + assert!(matches!( + complete_builtin_sensor(store, &first, &pending).await, + Err(ControlError::Fenced) + )); + assert!( + begin_sensor_evaluation(&replica, &successor, trigger.trigger_id) + .await + .expect("recover abandoned attempt") + .is_none(), + "a crash still counts toward the minimum interval" + ); + assert!(matches!( + evaluations(store) + .await + .last() + .expect("abandoned fact") + .outcome, + SensorEvaluationOutcome::Abandoned + )); + tokio::time::sleep(Duration::from_secs(1)).await; + let requested = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("new request") + .expect("due"); + assert_ne!(pending.evaluation_id, requested.evaluation_id); + let mut forged = requested.clone(); + forged.cursor.event_storage_id = 999; + assert!(matches!( + complete_builtin_sensor(store, &successor, &forged).await, + Err(ControlError::Fenced) + )); + input.generation += 1; + publish_snapshot(store, &input).await.expect("redeploy"); + assert_eq!( + complete_builtin_sensor(store, &successor, &requested) + .await + .expect("abandon stale snapshot"), + 0 + ); + assert!( + list_runs(store, (location, "development"), 100) + .await + .expect("no stale runs") + .is_empty() + ); + change(store, trigger.trigger_id, TriggerChange::Pause).await; + input.generation += 1; + publish_snapshot(store, &input) + .await + .expect("redeploy paused sensor"); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("preserved override")[0] + .status, + TriggerStatus::Paused + ); + assert!( + begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("paused") + .is_none() + ); + change(store, trigger.trigger_id, TriggerChange::ResetDefault).await; + tokio::time::sleep(Duration::from_secs(1)).await; + let current = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("code default") + .expect("running"); + assert_eq!( + complete_builtin_sensor(store, &successor, ¤t) + .await + .expect("catch up retained health"), + test_case.expected_runs, + "{}", + test_case.description + ); + tokio::time::sleep(Duration::from_secs(1)).await; + let before_reset = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("request before reset") + .expect("due"); + change(store, trigger.trigger_id, TriggerChange::ResetCursor).await; + assert_eq!( + complete_builtin_sensor(store, &successor, &before_reset) + .await + .expect("reset fences old cursor"), + 0 + ); + assert!(matches!( + evaluations(store) + .await + .last() + .expect("reset abandonment") + .outcome, + SensorEvaluationOutcome::Abandoned + )); + tokio::time::sleep(Duration::from_secs(1)).await; + let before_pause = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("request before pause") + .expect("due"); + change(store, trigger.trigger_id, TriggerChange::Pause).await; + assert_eq!( + complete_builtin_sensor(store, &successor, &before_pause) + .await + .expect("pause fences pending result"), + 0 + ); + assert!(matches!( + evaluations(store) + .await + .last() + .expect("pause abandonment") + .outcome, + SensorEvaluationOutcome::Abandoned + )); + change(store, trigger.trigger_id, TriggerChange::ResetDefault).await; + tokio::time::sleep(Duration::from_secs(1)).await; + let delayed = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("request delayed evaluation") + .expect("due"); + tokio::time::sleep(Duration::from_secs(1)).await; + complete_builtin_sensor(store, &successor, &delayed) + .await + .expect("complete delayed evaluation"); + let immediate = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("interval measured from start, not completion") + .expect("already due"); + assert_ne!(immediate.evaluation_id, delayed.evaluation_id); + complete_builtin_sensor(store, &successor, &immediate) + .await + .expect("complete next attempt"); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("deduplicated runs") + .len(), + test_case.expected_runs, + "{}", + test_case.description + ); + let transaction = begin(store, TransactionMode::Write).await.expect("age"); + let aged = transaction.now - TimeDelta::days(8); + let (transaction, _) = query( + transaction, + "UPDATE sqb_sensor_evaluation_events SET occurred_at=$1 WHERE evaluation_id<>$2", + vec![ + Parameter::Time(aged), + Parameter::Id(immediate.evaluation_id), + ], + ) + .await + .expect("aged evaluations"); + commit(transaction) + .await + .expect("aged evaluations committed"); + retain(store, Default::default(), &[]) + .await + .expect("retention pass"); + assert_eq!( + evaluations(store) + .await + .iter() + .map(|fact| fact.request.evaluation_id) + .collect::>(), + [immediate.evaluation_id, immediate.evaluation_id], + "{}: retention keeps only the latest attempt's phases", + test_case.description + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs new file mode 100644 index 0000000000..f513e2f7aa --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -0,0 +1,511 @@ +//! Actual lifecycle facts feed scoped outcome selectors and atomic recovery on both stores. + +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::TriggerChange; +use crate::runs::main::list_runs::list_runs; +use crate::runs::main::request_rerun::request_rerun; +use crate::runs::models::{NodeOutcome, RerunRequest, RunOutcome}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::{ + begin_sensor_evaluation::begin_sensor_evaluation, + complete_builtin_sensor::complete_builtin_sensor, +}; +use crate::sensors::tests::helpers::{change, finish_claimed_run, finish_run, snapshot}; +use crate::sensors::tests::test_types::{SensorExecutionTestCase, SensorTestCase}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_run_keys_are_preserved() + { + let test_cases = [ + SensorTestCase { + description: "SQLite terminal outcome sensor recovery", + backend: Backend::Sqlite, + expected_runs: 104, + }, + SensorTestCase { + description: "PostgreSQL terminal outcome sensor recovery", + backend: Backend::Postgres, + expected_runs: 104, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + migrate_store(store) + .await + .expect("warm migration preserves index"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "products", "test-admin") + .await + .expect("other location"); + let mut input = snapshot(location); + input.documents.insert("jobs".into(), json!({ + "orders": {"description": "Source orders", "selection": ["orders", "customers"]}, + "inventory": {"description": "Sensor destination", "selection": ["orders", "customers"]} + })); + input.documents.insert("groups".into(), json!({ + "daily_orders": {"description": "Source group", "selection": ["orders", "customers"], "jobs": {"orders": []}} + })); + let selectors = [ + "completed", + "success", + "warn", + "partial", + "failed", + "cancelled", + ]; + let mut sensors = serde_json::Map::new(); + for job_key in ["orders", "daily_orders"] { + for selector in selectors { + sensors.insert(format!("{job_key}_{selector}"), json!({ + "description": "Consume exact terminal outcomes", "job_key": "inventory", + "when": {"kind": "run_event", "job_key": job_key, "kinds": [selector]}, + "minimum_interval_seconds": 1 + })); + } + for (name, kind, nodes) in [ + ("both_completed", "completed", vec!["customers", "orders"]), + ("customers_completed", "completed", vec!["customers"]), + ("customers_success", "success", vec!["customers"]), + ("customers_partial", "partial", vec!["customers"]), + ("orders_completed", "completed", vec!["orders"]), + ] { + sensors.insert(format!("{job_key}_executed_{name}"), json!({ + "description": "Require every selected node to actually succeed", "job_key": "inventory", + "when": {"kind": "run_event", "job_key": job_key, "kinds": [kind], "executed_successfully": nodes}, + "minimum_interval_seconds": 1 + })); + } + } + input.documents.insert("triggers".into(), json!(sensors)); + publish_snapshot(store, &input) + .await + .expect("publish sensors"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let mut expected: BTreeMap> = BTreeMap::new(); + let outcomes = [ + ( + "success", + RunOutcome::Success, + [NodeOutcome::Success, NodeOutcome::Success], + ), + ( + "warn", + RunOutcome::Warn, + [NodeOutcome::Warn, NodeOutcome::Success], + ), + ( + "partial", + RunOutcome::Partial, + [NodeOutcome::Success, NodeOutcome::Failed], + ), + ( + "failed", + RunOutcome::Failed, + [NodeOutcome::Failed, NodeOutcome::Failed], + ), + ( + "cancelled", + RunOutcome::Cancelled, + [NodeOutcome::Cancelled, NodeOutcome::Cancelled], + ), + ]; + for (selector, outcome, nodes) in outcomes { + let id = finish_run( + store, + &lease, + (location, "development"), + "orders", + outcome, + nodes, + [false; 2], + ) + .await + .to_string(); + expected + .entry(format!("orders_{selector}")) + .or_default() + .insert(id.clone()); + expected + .entry("orders_completed".into()) + .or_default() + .insert(id); + } + let group = finish_run( + store, + &lease, + (location, "development"), + "daily_orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + [false; 2], + ) + .await; + let sources = list_runs(store, (location, "development"), 100) + .await + .expect("sources"); + let child = sources + .iter() + .find(|record| { + record.run_id != group + && record.job_key.as_deref() == Some("orders") + && !expected["orders_completed"].contains(&record.run_id.to_string()) + }) + .expect("group child"); + for selector in ["completed", "success"] { + expected + .entry(format!("daily_orders_{selector}")) + .or_default() + .insert(group.to_string()); + expected + .entry(format!("orders_{selector}")) + .or_default() + .insert(child.run_id.to_string()); + } + let executions = [ + SensorExecutionTestCase { + description: "freshness-only success cannot borrow another execution's starts", + outcome: RunOutcome::Success, + nodes: [NodeOutcome::Success; 2], + started: [false; 2], + expected_selector: "success", + expected_filters: &[], + }, + SensorExecutionTestCase { + description: "all selected nodes must start, not merely any node", + outcome: RunOutcome::Success, + nodes: [NodeOutcome::Success; 2], + started: [true, false], + expected_selector: "success", + expected_filters: &["customers_completed", "customers_success"], + }, + SensorExecutionTestCase { + description: "both selected nodes actually succeed", + outcome: RunOutcome::Success, + nodes: [NodeOutcome::Success; 2], + started: [true; 2], + expected_selector: "success", + expected_filters: &[ + "both_completed", + "customers_completed", + "customers_success", + "orders_completed", + ], + }, + SensorExecutionTestCase { + description: "partial outcome is independent of selected-node success", + outcome: RunOutcome::Partial, + nodes: [NodeOutcome::Success, NodeOutcome::Failed], + started: [true; 2], + expected_selector: "partial", + expected_filters: &["customers_completed", "customers_partial"], + }, + SensorExecutionTestCase { + description: "warning is not successful execution", + outcome: RunOutcome::Warn, + nodes: [NodeOutcome::Warn, NodeOutcome::Success], + started: [true; 2], + expected_selector: "warn", + expected_filters: &["orders_completed"], + }, + SensorExecutionTestCase { + description: "start without success cannot borrow success from another run", + outcome: RunOutcome::Failed, + nodes: [NodeOutcome::Failed; 2], + started: [true; 2], + expected_selector: "failed", + expected_filters: &[], + }, + SensorExecutionTestCase { + description: "skipped selected node is not successful", + outcome: RunOutcome::Partial, + nodes: [NodeOutcome::Skipped, NodeOutcome::Success], + started: [false, true], + expected_selector: "partial", + expected_filters: &["orders_completed"], + }, + SensorExecutionTestCase { + description: "a partial run's freshness-only node still does not qualify", + outcome: RunOutcome::Partial, + nodes: [NodeOutcome::Success, NodeOutcome::Failed], + started: [false, true], + expected_selector: "partial", + expected_filters: &[], + }, + SensorExecutionTestCase { + description: "a genuinely started warning is not success", + outcome: RunOutcome::Warn, + nodes: [NodeOutcome::Warn, NodeOutcome::Success], + started: [true, false], + expected_selector: "warn", + expected_filters: &[], + }, + ]; + let mut source_ids: BTreeMap<(&str, &str), Uuid> = BTreeMap::new(); + for execution in executions { + for job_key in ["orders", "daily_orders"] { + let id = finish_run( + store, + &lease, + (location, "development"), + job_key, + execution.outcome, + execution.nodes, + execution.started, + ) + .await; + source_ids.insert((execution.description, job_key), id); + let transaction = begin(store, TransactionMode::Read) + .await + .expect("read lineage"); + let (transaction, children) = query( + transaction, + "SELECT run_id FROM sqb_runs WHERE parent_run_id=$1", + vec![Parameter::Id(id)], + ) + .await + .expect(execution.description); + commit(transaction).await.expect("read lineage committed"); + for (source, id) in std::iter::once((job_key, id.to_string())).chain( + children.into_iter().map(|row| { + ( + "orders", + row["run_id"].as_str().expect("child id").to_owned(), + ) + }), + ) { + for kind in ["completed", execution.expected_selector] { + expected + .entry(format!("{source}_{kind}")) + .or_default() + .insert(id.clone()); + } + for filter in execution.expected_filters { + expected + .entry(format!("{source}_executed_{filter}")) + .or_default() + .insert(id.clone()); + } + } + } + } + let retry = RerunRequest { + run_id: Uuid::now_v7(), + source_run_id: source_ids[&( + "partial outcome is independent of selected-node success", + "daily_orders", + )], + from_failure: true, + reason: "Retry only failed orders without borrowing earlier customer success" + .into(), + priority: 10, + }; + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + assert!( + request_rerun(store, (&context, None), &retry) + .await + .expect("request rerun") + ); + assert!( + !request_rerun(store, (&context, None), &retry) + .await + .expect("retry request") + ); + finish_claimed_run( + store, + &lease, + (location, "development"), + retry.run_id, + RunOutcome::Success, + [NodeOutcome::Success; 2], + [true; 2], + ) + .await; + let transaction = begin(store, TransactionMode::Read) + .await + .expect("read rerun lineage"); + let (transaction, children) = query( + transaction, + "SELECT run_id FROM sqb_runs WHERE parent_run_id=$1", + vec![Parameter::Id(retry.run_id)], + ) + .await + .expect("rerun child"); + commit(transaction).await.expect("read rerun committed"); + for (source, id) in std::iter::once(("daily_orders", retry.run_id.to_string())).chain( + children.into_iter().map(|row| { + ( + "orders", + row["run_id"].as_str().expect("child id").to_owned(), + ) + }), + ) { + for suffix in ["completed", "success", "executed_orders_completed"] { + expected + .entry(format!("{source}_{suffix}")) + .or_default() + .insert(id.clone()); + } + } + let sources = list_runs(store, (location, "development"), 500) + .await + .expect("all source executions"); + // Same key in another target/location and a different job in this scope must not match. + let mut other_input = input.clone(); + other_input.location_id = other; + publish_snapshot(store, &other_input) + .await + .expect("other publication"); + other_input.location_id = location; + other_input.target = "preview".into(); + publish_snapshot(store, &other_input) + .await + .expect("other target"); + for scope in [(other, "development"), (location, "preview")] { + finish_run( + store, + &lease, + scope, + "orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + [false; 2], + ) + .await; + } + finish_run( + store, + &lease, + (location, "development"), + "inventory", + RunOutcome::Success, + [NodeOutcome::Success; 2], + [false; 2], + ) + .await; + let replica = second_store(store).await; + let fault = TransactionFault { + description: "fail atomic sensor cursor", + table: "sqb_trigger_state", + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected sensor failure')", + postgres_body: "RAISE EXCEPTION 'injected sensor failure'", + }; + let triggers = list_triggers(store, location, "development") + .await + .expect("triggers"); + for trigger in &triggers { + let requested = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request") + .expect("due"); + assert_eq!( + begin_sensor_evaluation(&replica, &lease, trigger.trigger_id) + .await + .expect("recover"), + Some(requested.clone()) + ); + install_fault(store, &fault).await; + assert!( + complete_builtin_sensor(store, &lease, &requested) + .await + .is_err() + ); + remove_fault(store, &fault).await; + let wanted = expected.get(&trigger.name).cloned().unwrap_or_default(); + assert_eq!( + complete_builtin_sensor(&replica, &lease, &requested) + .await + .expect("retry"), + wanted.len(), + "{}", + trigger.name + ); + assert_eq!( + complete_builtin_sensor(store, &lease, &requested) + .await + .expect("duplicate completion"), + 0 + ); + let transaction = begin(store, TransactionMode::Read) + .await + .expect("read requests"); + let (transaction, rows) = query( + transaction, + "SELECT run_key FROM sqb_run_requests WHERE trigger_id=$1", + vec![Parameter::Id(trigger.trigger_id)], + ) + .await + .expect("keys"); + commit(transaction).await.expect("read commit"); + let actual: BTreeSet = rows + .iter() + .map(|row| row["run_key"].as_str().expect("run key").into()) + .collect(); + assert_eq!(actual, wanted, "{}", trigger.name); + change(store, trigger.trigger_id, TriggerChange::ResetCursor).await; + } + tokio::time::sleep(Duration::from_secs(1)).await; + for trigger in &triggers { + let replay = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("reset request") + .expect("due"); + complete_builtin_sensor(store, &lease, &replay) + .await + .expect("replay deduplicates"); + } + let after = list_runs(store, (location, "development"), 500) + .await + .expect("queued sensors"); + assert_eq!( + after.len(), + sources.len() + 1 + test_case.expected_runs, + "{}: reset adds no runs", + test_case.description + ); + for source in sources { + assert_eq!( + after.iter().find(|record| record.run_id == source.run_id), + Some(&source), + "sensor never changes source state" + ); + } + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs new file mode 100644 index 0000000000..2f4831ba58 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs @@ -0,0 +1,703 @@ +//! Warehouse imports, atomic requests and durable receipt cursors run against both real stores. + +use std::time::Duration; + +use chrono::{FixedOffset, TimeDelta}; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::models::{LocationHealth, LocationHealthReport}; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::{ + begin_sensor_evaluation::begin_sensor_evaluation, + complete_builtin_sensor::complete_builtin_sensor, +}; +use crate::sensors::tests::helpers::snapshot; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::SnapshotNode; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::TransactionMode; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::warehouse_metadata::main::ingest_source_freshness::ingest_source_freshness; +use crate::warehouse_metadata::models::{ + MetadataBatch, MetadataScan, WarehouseFreshnessRecord, WarehouseHistoryKind, + WarehouseStateLocation, +}; + +#[test] +fn given_imported_source_versions_when_consuming_then_late_facts_queue_once_and_rebuilds_do_not_repeat() + { + let test_cases = [ + SensorTestCase { + description: "SQLite source versions", + backend: Backend::Sqlite, + expected_runs: 2, + }, + SensorTestCase { + description: "PostgreSQL source versions", + backend: Backend::Postgres, + expected_runs: 2, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + migrate_store(store).await.expect("warm migration"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.nodes.insert( + "source:raw_orders".into(), + SnapshotNode { + kind: "source".into(), + definition: json!({"name": "raw_orders", "database": null, "schema": "main", "table": "raw_orders"}), + raw_sql: None, + compiled_sql: None, + python_source: None, + }, + ); + input.documents.insert( + "triggers".into(), + json!({"orders_changed": { + "description": "Refresh orders after source data changes.", "job_key": "orders", + "when": {"kind": "source_new_data", "selection": ["source:raw_orders"]}, + "minimum_interval_seconds": 1 + }}), + ); + publish_snapshot(store, &input) + .await + .expect("publish source sensor"); + let session_id = Uuid::now_v7(); + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: "development".into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("registered reader"); + let fixture_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/source_data_sensor.json"); + let data: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(fixture_path).expect("DuckShop source sensor"), + ) + .expect("synthetic source versions"); + let mut records: Vec = Vec::new(); + for index in 0..data["source_rows"].as_u64().expect("row count") { + let mut record = data["record"].clone(); + record["event_id"] = json!(format!("orders-source-{index:03}")); + records.push( + serde_json::from_value::(record).expect("record"), + ); + } + for index in data["null_indices"].as_array().expect("null observations") { + let record = &mut records[index.as_u64().expect("index") as usize]; + record.data_version = None; + record.data_version_hash = "orders-null".into(); + } + records[data["unselected_index"].as_u64().expect("index") as usize].source_name = + "raw_products".into(); + let receipt_fault = TransactionFault { + description: "source receipt rollback", + table: "sqb_warehouse_freshness", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected source receipt failure')", + postgres_body: "RAISE EXCEPTION 'injected source receipt failure'", + }; + install_fault(store, &receipt_fault).await; + assert!( + ingest_source_freshness( + store, + (location, "development"), + &MetadataBatch { + kind: WarehouseHistoryKind::SourceFreshness, + session_id, + generation: 1, + records: records[..1].iter().map(|record| json!(record)).collect(), + scan: None, + } + ) + .await + .is_err() + ); + remove_fault(store, &receipt_fault).await; + let (transaction, stored) = query( + begin(store, TransactionMode::Read) + .await + .expect("observation read"), + "SELECT COUNT(*) AS count FROM sqb_warehouse_freshness", + vec![], + ) + .await + .expect("rolled back observations"); + commit(transaction).await.expect("finish read"); + assert_eq!(stored[0]["count"], 0); + for page in records.chunks(100) { + let batch = MetadataBatch { + kind: WarehouseHistoryKind::SourceFreshness, + session_id, + generation: 1, + records: page.iter().map(|record| json!(record)).collect(), + scan: None, + }; + assert!( + ingest_source_freshness(store, (location, "development"), &batch) + .await + .expect("import") + .applied + ); + assert!( + !ingest_source_freshness(store, (location, "development"), &batch) + .await + .expect("exact retry") + .applied + ); + } + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + let pending = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("evaluation") + .expect("due"); + let fault = TransactionFault { + description: "source cursor rollback", + table: "sqb_trigger_state", + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected source cursor failure')", + postgres_body: "RAISE EXCEPTION 'injected source cursor failure'", + }; + install_fault(store, &fault).await; + assert!( + complete_builtin_sensor(store, &lease, &pending) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert!( + list_runs(store, (location, "development"), 100) + .await + .expect("rollback") + .is_empty() + ); + assert_eq!( + begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("retry"), + Some(pending.clone()) + ); + assert_eq!( + tokio::time::timeout( + Duration::from_secs(5), + complete_builtin_sensor(store, &lease, &pending) + ) + .await + .expect("bounded source page") + .expect("complete"), + 1 + ); + assert_eq!( + complete_builtin_sensor(store, &lease, &pending) + .await + .expect("lost acknowledgement retry"), + 0 + ); + tokio::time::sleep(Duration::from_secs(1)).await; + let second = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("second page") + .expect("due"); + assert_eq!( + complete_builtin_sensor(store, &lease, &second) + .await + .expect("unchanged source versions"), + 0 + ); + let mut late = data["record"].as_object().expect("record").clone(); + late.extend( + data["late_record"] + .as_object() + .expect("late source observation") + .clone(), + ); + let batch = MetadataBatch { + kind: WarehouseHistoryKind::SourceFreshness, + session_id, + generation: 1, + records: vec![json!(late)], + scan: None, + }; + ingest_source_freshness(store, (location, "development"), &batch) + .await + .expect("late commit"); + tokio::time::sleep(Duration::from_secs(1)).await; + let later = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("late evaluation") + .expect("due"); + assert_eq!( + complete_builtin_sensor(store, &lease, &later) + .await + .expect("late warehouse time must not skip new receipt"), + 1 + ); + let (transaction, _) = query( + begin(store, TransactionMode::Write) + .await + .expect("re-ingested observations"), + "DELETE FROM sqb_warehouse_freshness", + vec![], + ) + .await + .expect("clear observations"); + commit(transaction) + .await + .expect("clear observations committed"); + for page in records.chunks(100) { + ingest_source_freshness( + store, + (location, "development"), + &MetadataBatch { + kind: WarehouseHistoryKind::SourceFreshness, + session_id, + generation: 1, + records: page.iter().map(|record| json!(record)).collect(), + scan: None, + }, + ) + .await + .expect("re-ingest observations"); + } + ingest_source_freshness(store, (location, "development"), &batch) + .await + .expect("rebuild late fact"); + tokio::time::sleep(Duration::from_secs(1)).await; + let rebuilt = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("rebuilt evaluation") + .expect("due"); + assert_eq!( + complete_builtin_sensor(store, &lease, &rebuilt) + .await + .expect("re-ingesting the same observations changes no version"), + 0 + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("queued requests") + .len(), + test_case.expected_runs, + "{}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_source_version_that_returns_after_a_complete_scan_prunes_history_when_consuming_then_each_change_queues_one_run() + { + let test_cases = [ + SensorTestCase { + description: "SQLite returning source version", + backend: Backend::Sqlite, + expected_runs: 3, + }, + SensorTestCase { + description: "PostgreSQL returning source version", + backend: Backend::Postgres, + expected_runs: 3, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.nodes.insert( + "source:raw_orders".into(), + SnapshotNode { + kind: "source".into(), + definition: json!({"name": "raw_orders", "database": null, "schema": "main", "table": "raw_orders"}), + raw_sql: None, + compiled_sql: None, + python_source: None, + }, + ); + input.documents.insert( + "triggers".into(), + json!({"orders_changed": { + "description": "Refresh orders after source data changes.", "job_key": "orders", + "when": {"kind": "source_new_data", "selection": ["source:raw_orders"]}, + "minimum_interval_seconds": 1 + }}), + ); + publish_snapshot(store, &input) + .await + .expect("publish source sensor"); + let session_id = Uuid::now_v7(); + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: "development".into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("registered reader"); + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/source_version_returns.json", + ); + let data: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("DuckShop case")) + .expect("synthetic source versions"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + let steps = data["observations"] + .as_array() + .expect("observations") + .iter() + .zip(data["expected_runs_after_each"].as_array().expect("runs")); + for (observation, expected) in steps { + let mut record = data["record"].as_object().expect("record").clone(); + record.extend(observation.as_object().expect("observation").clone()); + let complete = record.remove("complete_scan").expect("scan flag"); + let scan = complete.as_bool().expect("flag").then(|| MetadataScan { + scan_id: Uuid::now_v7(), + location: WarehouseStateLocation { + database: None, + schema: "main".into(), + }, + complete: true, + }); + ingest_source_freshness( + store, + (location, "development"), + &MetadataBatch { + kind: WarehouseHistoryKind::SourceFreshness, + session_id, + generation: 1, + records: vec![json!(record)], + scan, + }, + ) + .await + .expect("ingest observation"); + tokio::time::sleep(Duration::from_secs(1)).await; + let request = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("evaluation") + .expect("due"); + complete_builtin_sensor(store, &lease, &request) + .await + .expect("complete"); + assert_eq!( + complete_builtin_sensor(store, &lease, &request) + .await + .expect("lost acknowledgement"), + 0 + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("runs") + .len() as u64, + expected.as_u64().expect("run count"), + "{}: {observation}", + test_case.description + ); + } + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("runs") + .len(), + test_case.expected_runs, + "{}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_timestamp_observations_when_age_crosses_the_boundary_then_scoped_pages_queue_each_version_once() + { + let test_cases = [ + SensorTestCase { + description: "SQLite timestamp age", + backend: Backend::Sqlite, + expected_runs: 4, + }, + SensorTestCase { + description: "PostgreSQL timestamp age", + backend: Backend::Postgres, + expected_runs: 4, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + migrate_store(store).await.expect("warm migration"); + let data: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join( + "../../benchmarks/duckshop/edge_cases/orchestration/source_stale_sensor.json", + )) + .expect("DuckShop stale-source fixture"), + ) + .expect("fixture"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + let keys: Vec = (0..data["source_count"].as_u64().expect("source count")) + .map(|index| format!("source:raw_orders_{index:03}")) + .collect(); + for key in &keys { + input.nodes.insert( + key.clone(), + SnapshotNode { + kind: "source".into(), + definition: data["source_definitions"].get(key).cloned().unwrap_or_else( + || json!({"database": null, "schema": "main", "table": "raw_orders"}), + ), + raw_sql: None, + compiled_sql: None, + python_source: None, + }, + ); + } + input.documents.insert("triggers".into(), json!({"stale_orders": { + "description": "Refresh stale timestamp sources", "job_key": "orders", + "when": {"kind": "source_stale", "selection": keys, "after_seconds": data["after_seconds"]}, + "minimum_interval_seconds": 1 + }})); + publish_snapshot(store, &input).await.expect("publish"); + let session_id = Uuid::now_v7(); + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: "development".into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("registered reader"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + let first = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("evaluation") + .expect("due"); + let mut records = Vec::new(); + for (ordinal, case) in data["cases"].as_array().expect("cases").iter().enumerate() { + let index = case["index"].as_u64().expect("index"); + let instant = first.requested_at + - TimeDelta::seconds(case["age_seconds"].as_i64().expect("age")); + let version = case["data_version"] + .as_str() + .map(str::to_owned) + .unwrap_or_else(|| { + instant + .with_timezone( + &FixedOffset::east_opt( + case["offset_seconds"].as_i64().unwrap_or(0) as i32, + ) + .expect("offset"), + ) + .format(case["format"].as_str().unwrap_or("%Y-%m-%dT%H:%M:%S%.f%:z")) + .to_string() + }); + let record = WarehouseFreshnessRecord { + event_id: Some(format!("stale-source-{ordinal:03}")), + state_database: None, + state_schema: "main".into(), + source_name: format!("raw_orders_{index:03}"), + target_database: case["target_database"].as_str().map(str::to_owned), + target_schema: Some(case["target_schema"].as_str().unwrap_or("main").into()), + target_name: Some(case["target_name"].as_str().unwrap_or("raw_orders").into()), + run_id: "external-orders".into(), + strategy: "max_column".into(), + value_kind: case["value_kind"].as_str().unwrap_or("timestamp").into(), + data_version: (!case["null"].as_bool().unwrap_or(false)).then_some(version), + data_version_hash: format!("orders-version-{index}"), + observed_at: first.requested_at + + TimeDelta::microseconds( + case["observed_offset_microseconds"].as_i64().unwrap_or(0), + ), + }; + records.push(record); + } + let batch = MetadataBatch { + kind: WarehouseHistoryKind::SourceFreshness, + session_id, + generation: 1, + records: records.iter().map(|record| json!(record)).collect(), + scan: None, + }; + ingest_source_freshness(store, (location, "development"), &batch) + .await + .expect("store observations"); + let fault = TransactionFault { + description: "stale cursor rollback", + table: "sqb_trigger_state", + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected stale cursor failure')", + postgres_body: "RAISE EXCEPTION 'injected stale cursor failure'", + }; + install_fault(store, &fault).await; + assert!( + complete_builtin_sensor(store, &lease, &first) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert!( + list_runs(store, (location, "development"), 100) + .await + .expect("rollback") + .is_empty() + ); + tokio::time::timeout( + Duration::from_secs(5), + complete_builtin_sensor(store, &lease, &first), + ) + .await + .expect("bounded first page") + .expect("complete"); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("first page") + .len(), + 1, + "{}: strict age threshold and current source binding", + test_case.description + ); + for page in data["later_pages"].as_array().expect("page outcomes") { + tokio::time::sleep(Duration::from_secs(1)).await; + let request = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("next evaluation") + .expect("due"); + assert_eq!( + request.cursor.custom.as_deref(), + page["expected_cursor"].as_str(), + "{}", + test_case.description + ); + tokio::time::timeout( + Duration::from_secs(5), + complete_builtin_sensor(store, &lease, &request), + ) + .await + .expect("bounded indexed source lookup") + .expect("complete"); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("queued runs") + .len(), + page["expected_runs"].as_u64().expect("run count") as usize, + "{}", + test_case.description + ); + assert_eq!( + complete_builtin_sensor(store, &lease, &request) + .await + .expect("lost acknowledgement"), + 0 + ); + } + let (transaction, _) = query( + begin(store, TransactionMode::Write) + .await + .expect("clear observations"), + "DELETE FROM sqb_warehouse_freshness", + vec![], + ) + .await + .expect("delete observations"); + commit(transaction).await.expect("commit clear"); + ingest_source_freshness(store, (location, "development"), &batch) + .await + .expect("re-ingest observations"); + tokio::time::sleep(Duration::from_secs(1)).await; + let request = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("rebuilt evaluation") + .expect("due"); + complete_builtin_sensor(store, &lease, &request) + .await + .expect("replay stale conditions"); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("deduplicated requests") + .len(), + test_case.expected_runs, + "{}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_transitions.rs b/crates/sqlbuild-orchestration/tests/sensors/test_transitions.rs new file mode 100644 index 0000000000..02beda4259 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_transitions.rs @@ -0,0 +1,184 @@ +//! Independent check health survives non-decisive outcomes, other scopes and cursor replay. + +use std::time::Duration; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::TriggerChange; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{NodeOutcome, RunOutcome}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::{ + begin_sensor_evaluation::begin_sensor_evaluation, + complete_builtin_sensor::complete_builtin_sensor, +}; +use crate::sensors::tests::helpers::{change, finish_run, snapshot}; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_independent_check_failures_when_nondecisive_results_arrive_then_only_executed_success_recovers() + { + let test_cases = [ + SensorTestCase { + description: "SQLite check transitions", + backend: Backend::Sqlite, + expected_runs: 5, + }, + SensorTestCase { + description: "PostgreSQL check transitions", + backend: Backend::Postgres, + expected_runs: 5, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + migrate_store(store).await.expect("warm migration"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "products", "test-admin") + .await + .expect("other location"); + let mut input = snapshot(location); + input.nodes.get_mut("orders").expect("orders check").kind = "check".into(); + input + .nodes + .get_mut("customers") + .expect("customers check") + .kind = "check".into(); + input.documents.insert("jobs".into(), json!({ + "orders": {"description": "Source checks", "selection": ["orders", "customers"]}, + "inventory": {"description": "React to a transition", "selection": ["inventory"]} + })); + input.documents.insert("triggers".into(), json!({ + "failed_checks": {"description": "Actual failures", "job_key": "inventory", + "when": {"kind": "node_event", "selection": ["orders", "customers"], "kinds": ["failed"]}, "minimum_interval_seconds": 1}, + "recovered_checks": {"description": "Actual recoveries", "job_key": "inventory", + "when": {"kind": "node_event", "selection": ["orders", "customers"], "kinds": ["recovered"]}, "minimum_interval_seconds": 1} + })); + publish_snapshot(store, &input).await.expect("publish"); + for scope in [(other, "development"), (location, "staging")] { + let mut isolated = input.clone(); + isolated.location_id = scope.0; + isolated.target = scope.1.into(); + publish_snapshot(store, &isolated) + .await + .expect("isolated snapshot"); + } + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let fixture_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/check_transitions.json"); + let transitions: Vec = serde_json::from_str( + &std::fs::read_to_string(fixture_path).expect("DuckShop fixture"), + ) + .expect("DuckShop independent check lifecycle"); + for transition in transitions { + finish_run( + store, + &lease, + (location, "development"), + "orders", + serde_json::from_value(transition["outcome"].clone()).expect("run outcome"), + serde_json::from_value(transition["nodes"].clone()).expect("check outcomes"), + serde_json::from_value(transition["started"].clone()).expect("actual starts"), + ) + .await; + // Another location/target cannot clear either check's actual failure. + for scope in [(other, "development"), (location, "staging")] { + finish_run( + store, + &lease, + scope, + "orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + [true; 2], + ) + .await; + } + } + let history = read_recall_events(store, 0, 1000).await.expect("history"); + assert_eq!( + history + .iter() + .filter(|event| event["event_type"] == "run_node_finished" + && event["payload"]["change"]["change"]["outcome"] == "cancelled") + .count(), + 1 + ); + let triggers = list_triggers(store, location, "development") + .await + .expect("triggers"); + let mut queued = 0; + for trigger in &triggers { + let pending = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("evaluate") + .expect("due"); + let count = tokio::time::timeout( + Duration::from_secs(5), + complete_builtin_sensor(store, &lease, &pending), + ) + .await + .expect("bounded transition page") + .expect("complete"); + assert_eq!( + count, + if trigger.name == "failed_checks" { + 3 + } else { + 2 + }, + "{}", + test_case.description + ); + queued += count; + change(store, trigger.trigger_id, TriggerChange::ResetCursor).await; + } + assert_eq!(queued, test_case.expected_runs); + tokio::time::sleep(Duration::from_secs(1)).await; + for trigger in triggers { + let replay = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("replay") + .expect("due"); + assert_eq!( + complete_builtin_sensor(store, &lease, &replay) + .await + .expect("deduplicated replay"), + if trigger.name == "failed_checks" { + 3 + } else { + 2 + } + ); + } + assert_eq!( + list_runs(store, (location, "development"), 1000) + .await + .expect("replayed runs") + .iter() + .filter(|run| run.job_key.as_deref() == Some("inventory")) + .count(), + test_case.expected_runs + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_types.rs b/crates/sqlbuild-orchestration/tests/sensors/test_types.rs new file mode 100644 index 0000000000..af27d0d35e --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_types.rs @@ -0,0 +1,24 @@ +use crate::runs::models::{NodeOutcome, RunOutcome}; +use crate::sensors::models::CustomSensorOutcome; +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct SensorTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_runs: usize, +} + +pub(crate) struct CustomSensorResultCase { + pub(crate) outcome: CustomSensorOutcome, + pub(crate) expected_phase: &'static str, + pub(crate) expected_cursor: Option<&'static str>, +} + +pub(crate) struct SensorExecutionTestCase { + pub(crate) description: &'static str, + pub(crate) outcome: RunOutcome, + pub(crate) nodes: [NodeOutcome; 2], + pub(crate) started: [bool; 2], + pub(crate) expected_selector: &'static str, + pub(crate) expected_filters: &'static [&'static str], +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs new file mode 100644 index 0000000000..17e4465ba8 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -0,0 +1,333 @@ +//! Real control-store integration coverage; both engines exercise the same public contract. + +use std::collections::BTreeMap; + +use crate::snapshots::main::current_snapshot::current_snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::store::errors::ControlError; +use crate::store::main::open_store::open_store; +use crate::store::main::{begin::begin, commit::commit, migrate_store::migrate_store}; +use crate::store::models::{ControlStore, TransactionMode}; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::snapshots::tests::test_types::SchemaRejectionCase; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store}; +use crate::store::tests::store_test_types::Backend; + +pub(super) fn snapshot(location_id: Uuid) -> SnapshotInput { + SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: true, + git_author: None, + image_id: None, + environment_hash: "synthetic-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::from([ + ( + "customers".into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"materialization": "table", "columns": ["customer_id"]}), + raw_sql: Some("select 1 as customer_id".into()), + compiled_sql: Some("select 1 as customer_id".into()), + python_source: None, + }, + ), + ( + "orders".into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"dependencies": ["customers"], "materialization": "view"}), + raw_sql: Some("select customer_id from ref('customers')".into()), + compiled_sql: Some("select customer_id from main.customers".into()), + python_source: Some("# display only: orders".into()), + }, + ), + ]), + documents: BTreeMap::from([("lineage".into(), json!({"orders": ["customers"]}))]), + } +} + +async fn exercise_snapshot_contract(store: &ControlStore) -> i64 { + let second = second_store(store).await; + let (first, second_migration) = tokio::join!(migrate_store(store), migrate_store(&second)); + first.expect("first migration succeeds"); + second_migration.expect("concurrent migration succeeds"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("create test location"); + assert!( + current_snapshot(store, location, "development") + .await + .expect("read empty target") + .is_none() + ); + let mut input = snapshot(location); + let initial = publish_snapshot(store, &input) + .await + .expect("publish initial snapshot"); + let loaded = current_snapshot(store, location, "development") + .await + .expect("read initial snapshot") + .expect("initial snapshot exists"); + assert_eq!(loaded.record, initial); + assert_eq!(loaded.nodes, input.nodes); + assert_eq!(loaded.documents, input.documents); + assert_eq!( + loaded + .changes + .iter() + .map(|change| (change.subject_key.as_str(), change.change_kind.as_str())) + .collect::>(), + [("customers", "added"), ("orders", "added")] + ); + + // Container insertion order and the generation do not affect the content address. + input.generation = 2; + input + .nodes + .get_mut("orders") + .expect("orders node exists") + .definition = json!({"materialization": "view", "dependencies": ["customers"]}); + let unchanged = publish_snapshot(store, &input) + .await + .expect("publish reordered definition"); + assert_eq!(unchanged.manifest_digest, initial.manifest_digest); + assert!( + current_snapshot(store, location, "development") + .await + .expect("read unchanged snapshot") + .expect("unchanged snapshot exists") + .changes + .is_empty() + ); + assert!(matches!( + publish_snapshot(store, &input).await, + Err(ControlError::StaleGeneration) + )); + + input.generation = 3; + input + .nodes + .get_mut("orders") + .expect("orders node exists") + .compiled_sql = Some("select customer_id + 5 from main.customers".into()); + let changed = publish_snapshot(store, &input) + .await + .expect("publish changed SQL"); + assert_ne!(changed.manifest_digest, unchanged.manifest_digest); + let loaded = current_snapshot(store, location, "development") + .await + .expect("read changed snapshot") + .expect("changed snapshot exists"); + assert_eq!(loaded.nodes, input.nodes); + assert_eq!(loaded.changes.len(), 1); + assert_eq!(loaded.changes[0].subject_key, "orders"); + assert_eq!(loaded.changes[0].change_kind, "changed"); + assert_ne!(loaded.changes[0].before_hash, loaded.changes[0].after_hash); + + // One target's generation does not suppress another target's first publication. + input.target = "testing".into(); + input.generation = 1; + publish_snapshot(store, &input) + .await + .expect("publish another target"); + assert_eq!( + current_snapshot(store, location, "development") + .await + .expect("read development target") + .expect("development target exists") + .record + .generation, + 3 + ); + assert_eq!( + current_snapshot(store, location, "testing") + .await + .expect("read testing target") + .expect("testing target exists") + .record + .generation, + 1 + ); + + input.target = "development".into(); + input.generation = 4; + input.nodes.remove("orders"); + publish_snapshot(store, &input) + .await + .expect("publish node removal"); + let loaded = current_snapshot(store, location, "development") + .await + .expect("read node removal") + .expect("snapshot exists after removal"); + assert_eq!(loaded.changes.len(), 1); + assert_eq!(loaded.changes[0].change_kind, "removed"); + assert_eq!(loaded.changes[0].subject_key, "orders"); + assert!(loaded.changes[0].after_hash.is_none()); + + // The newer compiler wins regardless of which publisher takes the transaction lock first. + let mut older = input.clone(); + older.generation = 5; + input.generation = 6; + input + .nodes + .get_mut("customers") + .expect("customers node exists") + .compiled_sql = Some("select 6 as customer_id".into()); + let (old_result, new_result) = tokio::join!( + publish_snapshot(store, &older), + publish_snapshot(&second, &input) + ); + assert!(old_result.is_ok() || matches!(old_result, Err(ControlError::StaleGeneration))); + new_result.expect("new generation wins"); + let loaded = current_snapshot(store, location, "development") + .await + .expect("read winning snapshot") + .expect("winning snapshot exists"); + assert_eq!(loaded.record.generation, 6); + assert_eq!(loaded.nodes, input.nodes); + close_store(second).await; + loaded.record.generation +} + +pub(super) async fn snapshot_contract(backend: Backend) -> Option { + let test_store = open_test_store(backend).await?; + let generation = exercise_snapshot_contract(&test_store.store).await; + verify_sqlite_deduplication(&test_store.store); + close_test_store(test_store).await; + Some(generation) +} + +fn verify_sqlite_deduplication(store: &ControlStore) { + let ControlStore::Sqlite(path) = store else { + return; + }; + let connection = rusqlite::Connection::open(path).expect("inspect control store"); + let definitions: i64 = connection + .query_row("SELECT count(*) FROM sqb_node_definitions", [], |row| { + row.get(0) + }) + .expect("count immutable definitions"); + // Two initial nodes, one changed source, and one changed customer; unchanged targets reuse them. + assert_eq!(definitions, 4); + let sources: i64 = connection + .query_row("SELECT count(*) FROM sqb_source_texts", [], |row| { + row.get(0) + }) + .expect("count shared sources"); + assert_eq!(sources, 6); +} + +pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { + let directory = tempfile::tempdir().expect("create test directory"); + let path = directory.path().join("control.sqlite3"); + let store = open_store(path.to_str().expect("UTF-8 test path")) + .await + .expect("open test store"); + migrate_store(&store).await.expect("migrate test store"); + let location = create_location(&store, "orders", "test-admin") + .await + .expect("create test location"); + let mut input = snapshot(location); + let good = publish_snapshot(&store, &input) + .await + .expect("publish good snapshot"); + let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); + migrate_store(&store) + .await + .expect("repeat schema creation without changes"); + assert_eq!( + current_snapshot(&store, location, "development") + .await + .expect("read store after repeated startup") + .expect("snapshot survives repeated startup") + .record, + good + ); + // Fail after immutable bodies were inserted but before switching the current pointer. + connection.execute_batch("CREATE TRIGGER fail_snapshot_switch BEFORE UPDATE ON sqb_current_snapshots BEGIN SELECT RAISE(ABORT, 'synthetic publication failure'); END").expect("install publication failure"); + input.generation = 2; + input + .nodes + .get_mut("orders") + .expect("orders node exists") + .compiled_sql = Some("select 99".into()); + assert!(publish_snapshot(&store, &input).await.is_err()); + assert_eq!( + current_snapshot(&store, location, "development") + .await + .expect("read preserved snapshot") + .expect("last good snapshot exists") + .record, + good + ); + let count: i64 = connection + .query_row("SELECT count(*) FROM sqb_snapshots", [], |row| row.get(0)) + .expect("count committed snapshots"); + assert_eq!(count, 1); + connection + .execute_batch("DROP TRIGGER fail_snapshot_switch") + .expect("remove injected failure"); + let recovered = publish_snapshot(&store, &input) + .await + .expect("retry publication"); + for case in [SchemaRejectionCase { + description: "schema checksum changed", + update_sql: "UPDATE sqb_schema_version SET checksum='different'", + }] { + connection + .execute_batch(case.update_sql) + .expect("apply synthetic schema incompatibility"); + assert!( + matches!(migrate_store(&store).await, Err(ControlError::Schema(_))), + "{}", + case.description + ); + } + recovered.generation +} + +/// Retained generations for one location and target, and counts that expose orphaned content. +pub(super) async fn retained_snapshot_state(store: &ControlStore, location_id: Uuid) -> Vec { + let transaction = begin(store, TransactionMode::Read) + .await + .expect("snapshot state"); + let (transaction, rows) = crate::store::main::query::query( + transaction, + "SELECT generation FROM sqb_snapshots WHERE location_id=$1 ORDER BY generation", + vec![crate::store::models::Parameter::Id(location_id)], + ) + .await + .expect("generations"); + commit(transaction).await.expect("snapshot state read"); + rows.iter() + .map(|row| row["generation"].as_i64().expect("generation")) + .collect() +} + +/// Source texts and unreferenced source texts. +pub(super) async fn retention_counts(store: &ControlStore) -> (i64, i64) { + let transaction = begin(store, TransactionMode::Read) + .await + .expect("retention counts"); + let (transaction, rows) = crate::store::main::query::query( + transaction, + "SELECT (SELECT COUNT(*) FROM sqb_source_texts) AS texts, \ + (SELECT COUNT(*) FROM sqb_source_texts t WHERE NOT EXISTS \ + (SELECT 1 FROM sqb_snapshot_sources s WHERE s.text_hash=t.text_hash)) AS orphans", + vec![], + ) + .await + .expect("counts"); + commit(transaction).await.expect("retention counts read"); + let count = |name: &str| rows[0][name].as_i64().expect("count"); + (count("texts"), count("orphans")) +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs new file mode 100644 index 0000000000..708a6aff32 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs @@ -0,0 +1,48 @@ +use crate::snapshots::tests::helpers::{snapshot_contract, sqlite_failed_publish_and_schema_floor}; +use crate::snapshots::tests::test_types::{RecoveryTestCase, SnapshotTestCase}; +use crate::store::tests::store_helpers::run; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_control_store_when_snapshots_publish_then_generations_sources_and_changes_are_consistent() +{ + let test_cases = [ + SnapshotTestCase { + description: "SQLite snapshot publication", + backend: Backend::Sqlite, + expected_generation: 6, + }, + SnapshotTestCase { + description: "PostgreSQL snapshot publication", + backend: Backend::Postgres, + expected_generation: 6, + }, + ]; + for test_case in test_cases { + let Some(observed) = run(snapshot_contract(test_case.backend)) else { + continue; + }; + assert_eq!( + observed, test_case.expected_generation, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_sqlite_when_a_publish_fails_then_last_good_snapshot_survives_and_schema_drift_is_rejected() +{ + let test_cases = [RecoveryTestCase { + description: "atomic rollback and recovery", + expected_generation: 2, + }]; + for test_case in test_cases { + let observed = run(sqlite_failed_publish_and_schema_floor()); + assert_eq!( + observed, test_case.expected_generation, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs new file mode 100644 index 0000000000..b18477bf2f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs @@ -0,0 +1,10 @@ +//! Snapshot contracts compiled within the snapshot owner. + +#[path = "test_contract.rs"] +mod contract; +#[path = "helpers.rs"] +mod helpers; +#[path = "test_retention.rs"] +mod retention; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_retention.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_retention.rs new file mode 100644 index 0000000000..9e9e2e5d5f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_retention.rs @@ -0,0 +1,127 @@ +//! Snapshot GC keeps the newest generations and every referenced snapshot, and collects content. + +use serde_json::json; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::runs::tests::helpers::{request, request_run}; +use crate::scheduler::tests::contracts::ticks::helpers::snapshot; +use crate::snapshots::main::prune_snapshots::prune_snapshots; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::SnapshotRetention; +use crate::snapshots::tests::helpers::{retained_snapshot_state, retention_counts}; +use crate::snapshots::tests::test_types::SnapshotRetentionTestCase; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_twelve_snapshots_when_collected_then_ten_newest_and_run_referenced_snapshots_survive() { + let test_cases = [ + SnapshotRetentionTestCase { + description: "SQLite keeps ten newest plus the run-pinned first generation", + backend: Backend::Sqlite, + expected_generations: &[1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], + }, + SnapshotRetentionTestCase { + description: "PostgreSQL keeps ten newest plus the run-pinned first generation", + backend: Backend::Postgres, + expected_generations: &[1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], + }, + ]; + for test_case in test_cases { + run(async { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../benchmarks/duckshop/edge_cases/orchestration/snapshot_retention.json"); + let case: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(path).expect("snapshot fixture")) + .expect("snapshot case"); + let setting = |name: &str| { + SnapshotRetention::count( + u16::try_from(case[name].as_u64().expect("count")).expect("u16"), + ) + .expect("retention") + }; + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let generation_input = |generation: i64| { + let mut input = snapshot(location); + input.generation = generation; + let node = input.nodes.get_mut("orders").expect("orders node"); + node.raw_sql = Some(format!("select {generation} as order_id")); + node.compiled_sql = node.raw_sql.clone(); + node.definition = json!({"materialization": "table", "revision": generation}); + input + }; + let first = publish_snapshot(store, &generation_input(1)) + .await + .expect("publish first"); + assert!( + request_run( + store, + &request((location, first.snapshot_id), 1, 0, &["orders"]) + ) + .await + .expect("pinned run") + ); + for generation in 2..=case["publications"].as_i64().expect("publications") { + publish_snapshot(store, &generation_input(generation)) + .await + .expect("publish"); + } + let triggers = list_triggers(store, location, "development") + .await + .expect("triggers before collection"); + assert_eq!(retention_counts(store).await, (12, 0)); + assert_eq!( + prune_snapshots(store, setting("keep")) + .await + .expect("collect") as u64, + case["expected_collected"].as_u64().expect("collected") + ); + assert_eq!( + retained_snapshot_state(store, location).await, + test_case.expected_generations, + "{}", + test_case.description + ); + assert_eq!( + retention_counts(store).await, + (11, 0), + "the expired snapshot's text goes with it" + ); + assert_eq!( + prune_snapshots(store, setting("keep")) + .await + .expect("idempotent"), + 0 + ); + assert!(SnapshotRetention::count(0).is_err()); + assert_eq!( + prune_snapshots(store, setting("shorter_keep")) + .await + .expect("shorter retention"), + 9 + ); + assert_eq!( + serde_json::json!(retained_snapshot_state(store, location).await), + case["expected_after_shorter"] + ); + assert_eq!(retention_counts(store).await.1, 0); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("triggers after collection"), + triggers, + "collection never changes live trigger state" + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs new file mode 100644 index 0000000000..997d311376 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs @@ -0,0 +1,29 @@ +//! Cases that alter a disposable schema to verify startup rejection. + +use crate::store::tests::store_test_types::Backend; + +#[derive(Debug)] +pub(super) struct SchemaRejectionCase { + pub(super) description: &'static str, + pub(super) update_sql: &'static str, +} + +#[derive(Debug)] +pub(super) struct SnapshotTestCase { + pub(super) description: &'static str, + pub(super) backend: Backend, + pub(super) expected_generation: i64, +} + +#[derive(Debug)] +pub(super) struct RecoveryTestCase { + pub(super) description: &'static str, + pub(super) expected_generation: i64, +} + +#[derive(Debug)] +pub(super) struct SnapshotRetentionTestCase { + pub(super) description: &'static str, + pub(super) backend: Backend, + pub(super) expected_generations: &'static [i64], +} diff --git a/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs b/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs new file mode 100644 index 0000000000..d56af653d7 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs @@ -0,0 +1,58 @@ +//! Database-trigger fault injection at durable publication boundaries. + +use crate::store::models::ControlStore; + +use crate::store::tests::fault_types::TransactionFault; + +pub(crate) async fn install_fault(store: &ControlStore, point: &TransactionFault) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + let path = path.clone(); + let sql = format!( + "CREATE TRIGGER sqb_test_fault BEFORE {} ON {} {} BEGIN {}; END;", + point.operation, point.table, point.condition, point.sqlite_body + ); + tokio::task::spawn_blocking(move || { + rusqlite::Connection::open(path) + .expect("fault connection") + .execute_batch(&sql) + }) + .await + .expect("join fault setup") + .expect(point.description); + return; + }; + sqlx_core::query::query(&format!("CREATE FUNCTION sqb_test_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN {}; RETURN NEW; END $$", point.postgres_body)) + .execute(pool).await.expect("install PostgreSQL fault function"); + sqlx_core::query::query(&format!("CREATE TRIGGER sqb_test_fault BEFORE {} ON {} FOR EACH ROW {} EXECUTE FUNCTION sqb_test_fault()", + point.operation, point.table, point.condition.replace("WHEN ", "WHEN (") + [")", ""][usize::from(point.condition.is_empty())])) + .execute(pool).await.expect(point.description); +} + +pub(crate) async fn remove_fault(store: &ControlStore, point: &TransactionFault) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + let path = path.clone(); + tokio::task::spawn_blocking(move || { + rusqlite::Connection::open(path) + .expect("fault connection") + .execute_batch("DROP TRIGGER sqb_test_fault") + }) + .await + .expect("join fault removal") + .expect("remove SQLite transaction fault"); + return; + }; + sqlx_core::query::query(&format!("DROP TRIGGER sqb_test_fault ON {}", point.table)) + .execute(pool) + .await + .expect("remove PostgreSQL fault trigger"); + sqlx_core::query::query("DROP FUNCTION sqb_test_fault()") + .execute(pool) + .await + .expect("remove PostgreSQL fault function"); +} diff --git a/crates/sqlbuild-orchestration/tests/store/faults/test_types.rs b/crates/sqlbuild-orchestration/tests/store/faults/test_types.rs new file mode 100644 index 0000000000..e25282c3cc --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/faults/test_types.rs @@ -0,0 +1,10 @@ +//! Transaction failure cases shared by credential, tick and run contracts. + +pub(crate) struct TransactionFault { + pub(crate) description: &'static str, + pub(crate) table: &'static str, + pub(crate) operation: &'static str, + pub(crate) condition: &'static str, + pub(crate) sqlite_body: &'static str, + pub(crate) postgres_body: &'static str, +} diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs new file mode 100644 index 0000000000..6258f469fb --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -0,0 +1,87 @@ +//! Real-database setup shared by storage, scheduling, and queue contracts. + +use crate::store::main::open_store::open_store; +use crate::store::models::ControlStore; +use uuid::Uuid; + +use crate::store::tests::store_test_types::{Backend, TestStore}; + +pub(crate) fn run(future: impl std::future::Future) -> T { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("create control-store test runtime") + .block_on(future) +} + +pub(crate) async fn open_test_store(backend: Backend) -> Option { + let Backend::Postgres = backend else { + let directory = tempfile::tempdir().expect("create isolated SQLite directory"); + let store = open_store( + directory + .path() + .join("control.sqlite3") + .to_str() + .expect("UTF-8 test path"), + ) + .await + .expect("open isolated SQLite store"); + return Some(TestStore { + store, + directory: Some(directory), + postgres: None, + }); + }; + let url = std::env::var("SQLBUILD_TEST_CONTROL_POSTGRES_URL").expect( + "PostgreSQL integration requires the disposable database harness; run make test-native", + ); + let admin = sqlx_postgres::PgPool::connect(&url) + .await + .expect("connect to test PostgreSQL"); + let schema = format!("sqb_test_{}", Uuid::now_v7().simple()); + // Writer advisory locks are database-wide, so schemas alone do not isolate contracts. + sqlx_core::query::query(&format!("CREATE DATABASE {schema} TEMPLATE template0")) + .execute(&admin) + .await + .expect("create isolated test database"); + let mut isolated_url = reqwest::Url::parse(&url).expect("test PostgreSQL URL"); + isolated_url.set_path(&schema); + let url = isolated_url.as_str(); + let setup = sqlx_postgres::PgPool::connect(url) + .await + .expect("connect to isolated test database"); + sqlx_core::query::query(&format!("CREATE SCHEMA {schema}")) + .execute(&setup) + .await + .expect("create isolated test schema"); + setup.close().await; + let separator = ['?', '&'][usize::from(url.contains('?'))]; + let store = open_store(&format!("{url}{separator}options=-csearch_path%3D{schema}")) + .await + .expect("open isolated PostgreSQL store"); + Some(TestStore { + store, + directory: None, + postgres: Some((admin, schema)), + }) +} + +pub(crate) async fn close_test_store(test_store: TestStore) { + close_store(test_store.store).await; + let Some((admin, schema)) = test_store.postgres else { + return; + }; + sqlx_core::query::query(&format!("DROP DATABASE {schema}")) + .execute(&admin) + .await + .expect("remove isolated test database"); + admin.close().await; + drop(test_store.directory); +} + +pub(crate) async fn close_store(store: ControlStore) { + let ControlStore::Postgres(pool) = store else { + return; + }; + pool.close().await; +} diff --git a/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs b/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs new file mode 100644 index 0000000000..59f067dc23 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs @@ -0,0 +1,24 @@ +//! Expire the persisted lease without introducing a fake control-store clock. + +use crate::store::models::ControlStore; + +pub(crate) async fn expire_lease(store: &ControlStore) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + rusqlite::Connection::open(path) + .expect("open lease expiry connection") + .execute_batch( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ) + .expect("expire SQLite scheduler lease"); + return; + }; + sqlx_core::query::query( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ) + .execute(pool) + .await + .expect("expire PostgreSQL scheduler lease"); +} diff --git a/crates/sqlbuild-orchestration/tests/store/logical_state/helpers.rs b/crates/sqlbuild-orchestration/tests/store/logical_state/helpers.rs new file mode 100644 index 0000000000..e13e1321dc --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/logical_state/helpers.rs @@ -0,0 +1,42 @@ +//! Hash complete logical state without printing credential or evidence rows on a failure. + +use std::collections::BTreeMap; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub(crate) async fn logical_state(store: &ControlStore) -> BTreeMap { + let transaction = begin(store, TransactionMode::Read) + .await + .expect("state cut"); + let sql = [ + "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name", + "SELECT table_name AS name FROM information_schema.tables WHERE table_schema=current_schema() AND table_type='BASE TABLE' ORDER BY table_name", + ][usize::from(matches!(store, ControlStore::Postgres(..)))]; + let (mut transaction, tables) = query(transaction, sql, vec![]).await.expect("tables"); + let mut result = BTreeMap::new(); + for table in tables { + let name = table["name"].as_str().expect("table name"); + let rows; + (transaction, rows) = query( + transaction, + &format!("SELECT * FROM \"{}\"", name.replace('"', "\"\"")), + vec![], + ) + .await + .expect("logical rows"); + let mut rows: Vec = rows + .iter() + .map(|row| serde_json::to_string(row).expect("canonical row")) + .collect(); + rows.sort(); + result.insert( + name.to_owned(), + sha256_hex(&serde_json::to_vec(&rows).expect("rows")), + ); + } + commit(transaction).await.expect("finish state cut"); + result +} diff --git a/crates/sqlbuild-orchestration/tests/store/recall/helpers.rs b/crates/sqlbuild-orchestration/tests/store/recall/helpers.rs new file mode 100644 index 0000000000..a530b60e5c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/recall/helpers.rs @@ -0,0 +1,179 @@ +//! Recall publication fixtures exercise the control store's durable append boundary. + +use crate::store::errors::ControlError; +use crate::store::main::append_recall::append_recall; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, RecallAppendOutcome, TransactionMode}; +use crate::store::types::Row; +use serde_json::json; +use sqlbuild_recall::records::main::artifact_rows::artifact_rows; +use sqlbuild_recall::records::main::event_object_row::event_object_row; +use sqlbuild_recall::records::models::{ArtifactInput, RecallRow, RowEnvelope}; + +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store}; +use crate::store::tests::store_test_types::Backend; + +pub(crate) async fn publish_recall( + store: &ControlStore, + rows: &[RecallRow], +) -> Result, ControlError> { + let mut transaction = begin(store, TransactionMode::Write).await?; + let mut outcomes = Vec::with_capacity(rows.len()); + for row in rows { + let outcome; + (transaction, outcome) = append_recall(transaction, row).await?; + outcomes.push(outcome); + } + commit(transaction).await?; + Ok(outcomes) +} + +pub(crate) async fn read_recall_events( + store: &ControlStore, + after: i64, + limit: u32, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT * FROM recall_events WHERE storage_id > $1 ORDER BY storage_id LIMIT $2", + vec![ + Parameter::Integer(after), + Parameter::Integer(i64::from(limit.min(1000))), + ], + ) + .await?; + commit(transaction).await?; + Ok(rows) +} + +pub(crate) fn event(id: &str, occurred_at: &str, duration_ms: f64) -> RecallRow { + event_object_row(&RowEnvelope { invocation_id: "orders-build".into(), + invocation_started_at: "2026-03-01T00:30:00+01:00".into(), producer_version: "test".into() }, + json!({ "event_id": id, "event_type": "invocation_completed", "schema_version": 1, + "occurred_at": occurred_at, "producer": "worker", "invocation_sequence": 1, + "payload": { "duration_ms": duration_ms, "exit_code": 0, "nested": [null, true, "orders"] } }) + .as_object().expect("event object")).expect("valid lifecycle event") +} + +pub(crate) async fn recall_contract(backend: Backend) -> Option> { + let fixture = open_test_store(backend).await?; + let store = &fixture.store; + migrate_store(store) + .await + .expect("migrate control and Recall schema"); + let second = second_store(store).await; + let first = event("orders-first", "2026-03-01T00:00:00Z", 3.25); + let first_batch = [first.clone()]; + let (left, right) = tokio::join!( + publish_recall(store, &first_batch), + publish_recall(&second, &first_batch) + ); + let left = left.expect("first writer").remove(0); + let right = right.expect("second writer").remove(0); + assert_ne!(left.inserted, right.inserted, "exactly one writer inserts"); + assert_eq!( + left.storage_id, right.storage_id, + "duplicates preserve cursor" + ); + let cursor = left.storage_id.expect("event storage cursor"); + let conflict = publish_recall(store, &[event("orders-first", "2026-03-01T00:00:00Z", 7.5)]) + .await + .expect("record conflicting retry") + .remove(0); + assert!( + conflict.conflict.is_some(), + "different content records conflict" + ); + let later = event("orders-later-arrival", "2026-02-01T00:00:00Z", 2.5); + let malformed = event("orders-malformed", "2026-04-01T00:00:00Z", 1.0); + let mut malformed = malformed; + malformed.values.retain(|(name, _)| *name != "event_id"); + assert!( + publish_recall(store, &[later.clone(), malformed]) + .await + .is_err(), + "whole malformed batch rolls back" + ); + assert!( + read_recall_events(store, cursor, 100) + .await + .expect("rolled-back page") + .is_empty() + ); + publish_recall(store, &[later]) + .await + .expect("retry rolled-back event"); + let page = read_recall_events(store, cursor, 1) + .await + .expect("late arrival page"); + assert_eq!(page.len(), 1); + assert_eq!( + page[0]["event_id"], "orders-later-arrival", + "page follows commit order, not producer time" + ); + let rows = read_recall_events(store, 0, 100).await.expect("all events"); + assert_eq!( + rows[0]["duration_ms"], + json!(3.25), + "first content wins with fractional precision" + ); + let payload = rows[0]["payload"].as_str().map_or_else( + || rows[0]["payload"].clone(), + |text| serde_json::from_str(text).expect("JSON payload"), + ); + assert_eq!(payload["nested"], json!([null, true, "orders"])); + let artifact = ArtifactInput { + digest: "sha256:orders".into(), + kind: "sql".into(), + media_type: "application/octet-stream".into(), + content: vec![0, 255, 1, 0], + seen_at: "2026-01-01T00:00:00Z".into(), + ref_id: "orders".into(), + role: "input".into(), + statement_id: None, + resource_id: None, + }; + let first_artifact = artifact_rows( + &RowEnvelope { + invocation_id: "first-artifact".into(), + invocation_started_at: artifact.seen_at.clone(), + producer_version: "test".into(), + }, + &artifact, + ); + publish_recall(store, &first_artifact) + .await + .expect("first binary artifact"); + let artifact = ArtifactInput { + seen_at: "2026-04-01T00:00:00Z".into(), + ..artifact + }; + let second_artifact = artifact_rows( + &RowEnvelope { + invocation_id: "second-artifact".into(), + invocation_started_at: artifact.seen_at.clone(), + producer_version: "test".into(), + }, + &artifact, + ); + let outcomes = publish_recall(store, &second_artifact) + .await + .expect("same artifact across months"); + assert!( + !outcomes[0].inserted && outcomes[0].conflict.is_none(), + "artifact identity is global, not partition-local" + ); + assert!(outcomes[1].inserted, "each invocation keeps its reference"); + let ids = rows + .iter() + .map(|row| row["event_id"].as_str().expect("event ID").to_owned()) + .collect(); + close_store(second).await; + close_test_store(fixture).await; + Some(ids) +} diff --git a/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs new file mode 100644 index 0000000000..7f0f9098dd --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs @@ -0,0 +1,123 @@ +//! Idempotency and commit-order paging through real store transactions. + +use crate::store::tests::helpers::recall_contract; +use crate::store::tests::store_helpers::run; +use crate::store::tests::store_test_types::Backend; +use crate::store::tests::test_types::RecallTestCase; + +use std::time::Duration; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, + store_source_text::store_source_text, subscribe_changes::subscribe_changes, +}; +use crate::store::models::{Change, ChangeMessage, ChangeTopic, Parameter, TransactionMode}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store}; +use crate::store::tests::test_types::CommitCancellationTestCase; + +#[test] +fn given_native_recall_facts_when_retried_and_paged_then_content_and_commit_order_are_preserved() { + let test_cases = [ + RecallTestCase { + description: "SQLite Recall transactions", + backend: Backend::Sqlite, + expected_event_ids: vec!["orders-first", "orders-later-arrival"], + }, + RecallTestCase { + description: "PostgreSQL Recall transactions", + backend: Backend::Postgres, + expected_event_ids: vec!["orders-first", "orders-later-arrival"], + }, + ]; + for test_case in test_cases { + let observed = run(recall_contract(test_case.backend)); + assert!( + observed.is_none_or(|ids| ids == test_case.expected_event_ids), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_sqlite_commit_cancellation_when_blocking_work_finishes_then_persisted_changes_still_notify() + { + let test_cases = [CommitCancellationTestCase { + description: "cancelled async caller cannot strand committed invalidations", + expected_body: b"SELECT 7 AS order_id", + }]; + for test_case in test_cases { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .max_blocking_threads(1) + .build() + .expect("single blocking worker runtime") + .block_on(async { + let fixture = open_test_store(Backend::Sqlite) + .await + .expect("isolated SQLite"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let (sender, _) = subscribe_changes(store).await.expect("change subscription"); + let mut receiver = sender.subscribe(); + let hash = sha256_hex(test_case.expected_body); + let mut transaction = store_source_text( + begin(store, TransactionMode::Write) + .await + .expect("write transaction"), + (hash.clone(), test_case.expected_body.to_vec()), + ) + .await + .expect("insert source"); + transaction + .changes + .insert(Change::NodeStatus(Uuid::now_v7(), "development".into())); + + let (started, started_signal) = tokio::sync::oneshot::channel(); + let (release, wait) = std::sync::mpsc::channel(); + let blocker = tokio::task::spawn_blocking(move || { + started.send(()).expect("blocking worker occupied"); + let _ = wait.recv(); + }); + started_signal.await.expect("worker ready"); + let mut committing = Box::pin(commit(transaction)); + assert!( + std::future::poll_fn(|context| std::task::Poll::Ready( + committing.as_mut().poll(context).is_pending() + )) + .await + ); + drop(committing); + release.send(()).expect("release queued commit"); + blocker.await.expect("blocking worker released"); + let notice = tokio::time::timeout(Duration::from_secs(3), receiver.recv()) + .await + .expect("committed write still notifies") + .expect("live notification channel"); + assert!(matches!( + notice, + ChangeMessage::Changed { + topic: ChangeTopic::NodeStatus, + .. + } + )); + let (transaction, rows) = query( + begin(store, TransactionMode::Read) + .await + .expect("read committed source"), + "SELECT body FROM sqb_source_texts WHERE text_hash=$1", + vec![Parameter::Text(hash)], + ) + .await + .expect("query committed bytes"); + commit(transaction).await.expect("finish read"); + let body: Vec = + serde_json::from_value(rows[0]["body"].clone()).expect("stored bytes"); + assert_eq!(body, test_case.expected_body, "{}", test_case.description); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs new file mode 100644 index 0000000000..490a7f2c0c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs @@ -0,0 +1,20 @@ +//! Expected Recall storage outcomes for both supported databases. + +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct RecallTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_event_ids: Vec<&'static str>, +} + +pub(crate) struct RecallViewTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_complete: i64, +} + +pub(crate) struct CommitCancellationTestCase { + pub(crate) description: &'static str, + pub(crate) expected_body: &'static [u8], +} diff --git a/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs new file mode 100644 index 0000000000..a157ecf787 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs @@ -0,0 +1,194 @@ +//! Durable Recall views preserve the published execution records. + +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::read_recall_view::read_recall_view; +use serde_json::json; +use sqlbuild_recall::records::main::artifact_rows::artifact_rows; +use sqlbuild_recall::records::main::delivery_row::delivery_row; +use sqlbuild_recall::records::main::event_object_row::event_object_row; +use sqlbuild_recall::records::main::heartbeat_row::heartbeat_row; +use sqlbuild_recall::records::main::invocation_row::invocation_row; +use sqlbuild_recall::records::main::output_row::output_row; +use sqlbuild_recall::records::models::{ + ArtifactInput, DeliveryCounts, InvocationContext, OutputChunk, OutputStream, RowEnvelope, +}; + +use crate::store::tests::helpers::publish_recall; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::store::tests::test_types::RecallViewTestCase; + +#[test] +fn given_native_context_output_and_audits_when_reading_views_then_both_databases_preserve_their_meaning() + { + let test_cases = [ + RecallViewTestCase { + description: "SQLite supported views", + backend: Backend::Sqlite, + expected_complete: 1, + }, + RecallViewTestCase { + description: "PostgreSQL supported views", + backend: Backend::Postgres, + expected_complete: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate view store"); + let envelope = RowEnvelope { + invocation_id: "orders-output".into(), + invocation_started_at: "2026-01-02T00:00:00Z".into(), + producer_version: "test".into(), + }; + let context = InvocationContext { + command: Some("build".into()), + selection: vec!["orders".into()], + external_context: Some( + json!({"integration": {"name": "orders", "retry_number": 2}}).to_string(), + ), + ..InvocationContext::default() + }; + let mut rows = vec![ + invocation_row(&envelope, &context), + heartbeat_row( + &envelope, + ("2026-01-02T00:00:01Z", "2026-01-02T00:00:31Z"), + (None, None), + (Some("build"), None), + ), + ]; + for (sequence, (stream, bytes)) in [ + (OutputStream::Stdout, b"orders\x00\xff".as_slice()), + (OutputStream::Stderr, b"warning\n".as_slice()), + (OutputStream::Stdout, b"\xc3\x28 tail".as_slice()), + ] + .into_iter() + .enumerate() + { + rows.push(output_row( + &envelope, + &OutputChunk { + sequence: sequence as u64, + stream, + occurred_at: "2026-01-02T00:00:02Z".into(), + data: bytes.to_vec(), + byte_offset: 0, + dropped_before: 0, + omitted: None, + }, + )); + } + rows.push(delivery_row( + &envelope, + "output", + ( + &DeliveryCounts { + accepted: 3, + delivered: 3, + ..DeliveryCounts::default() + }, + true, + ), + "2026-01-02T00:00:03Z", + )); + for (event_id, event_type, payload) in [ + ( + "orders-audit", + "audit_completed", + json!({"audit_name":"unique_orders", "violation_count": 7, "measured_value": 3.5, "target_name":"orders", "evidence_truncated": true}), + ), + ( + "orders-complete", + "invocation_completed", + json!({"duration_ms": 12.75, "exit_code": 0}), + ), + ] { + rows.push(event_object_row(&envelope, json!({"event_id": event_id, "event_type": event_type, "schema_version": 1, "occurred_at": "2026-01-02T00:00:03Z", "producer": "worker", "invocation_sequence": 1, "payload": payload}).as_object().expect("event object")).expect("lifecycle fact")); + } + rows.extend(artifact_rows( + &envelope, + &ArtifactInput { + digest: "sha256:orders-display".into(), + kind: "sql".into(), + media_type: "text/plain".into(), + content: b"orders\x00\xff".to_vec(), + seen_at: envelope.invocation_started_at.clone(), + ref_id: "orders-statement".into(), + role: "input".into(), + statement_id: None, + resource_id: None, + }, + )); + publish_recall(store, &rows) + .await + .expect("write view inputs"); + let output = + read_recall_view(store, ("output_invocations", &envelope.invocation_id), 100) + .await + .expect("output accounting"); + assert_eq!(output[0]["complete"], json!(test_case.expected_complete)); + assert_eq!(output[0]["chunks"], json!(3)); + assert_eq!(output[0]["missing_sequences"], json!(0)); + let transcript = + read_recall_view(store, ("output_transcripts", &envelope.invocation_id), 100) + .await + .expect("binary transcript"); + assert_eq!( + transcript[0]["stdout"], + json!([b"orders\x00\xff".as_slice(), b"\xc3\x28 tail".as_slice()].concat()) + ); + assert_eq!( + transcript[0]["combined"], + json!( + [ + b"orders\x00\xff".as_slice(), + b"warning\n".as_slice(), + b"\xc3\x28 tail".as_slice() + ] + .concat() + ) + ); + let invocation = read_recall_view(store, ("invocations", &envelope.invocation_id), 100) + .await + .expect("invocation projection"); + assert_eq!(invocation[0]["status"], "completed"); + assert_eq!(invocation[0]["duration_ms"], json!(12.75)); + assert_eq!(invocation[0]["integration_retry"], json!(2)); + let audit = read_recall_view(store, ("audit_results", &envelope.invocation_id), 100) + .await + .expect("audit projection"); + assert_eq!(audit[0]["violation_count"], json!(7)); + assert_eq!(audit[0]["measured_value"], json!(3.5)); + assert_eq!(audit[0]["evidence_truncated"], json!(1)); + let tail = read_recall_view(store, ("output_tail", &envelope.invocation_id), 100) + .await + .expect("lossy output display"); + assert_eq!( + tail.iter() + .find(|row| row["sequence"] == 0) + .expect("first chunk")["text"], + json!("orders\u{0}\u{fffd}"), + "{}", + test_case.description + ); + assert_eq!( + tail.iter() + .find(|row| row["sequence"] == 2) + .expect("last chunk")["text"], + json!("\u{fffd}( tail") + ); + let artifacts = read_recall_view(store, ("artifacts", &envelope.invocation_id), 100) + .await + .expect("lossy artifact display"); + assert_eq!(artifacts[0]["text"], json!("orders\u{0}\u{fffd}")); + assert_eq!(artifacts[0]["content"], json!(b"orders\x00\xff".to_vec())); + assert_eq!(artifacts[0]["available"], json!(1)); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs b/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs new file mode 100644 index 0000000000..0357fa71b6 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs @@ -0,0 +1,26 @@ +//! Independent database handles for multi-replica integration contracts. + +use crate::store::main::open_store::open_store; +use crate::store::models::ControlStore; +use sqlx_core::connection::ConnectOptions; + +pub(crate) async fn second_store(store: &ControlStore) -> ControlStore { + open_store(&store_url(store)) + .await + .expect("open independent control-store handle") +} + +pub(crate) fn store_url(store: &ControlStore) -> String { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + return path.to_str().expect("UTF-8 test path").to_owned(); + }; + let options = pool.connect_options(); + let mut url = options.to_url_lossy(); + // SQLx's lossy URL omits startup options, including the isolated schema search path. + url.query_pairs_mut() + .extend_pairs(options.get_options().map(|startup| ("options", startup))); + url.to_string() +} diff --git a/crates/sqlbuild-orchestration/tests/store/test_contracts.rs b/crates/sqlbuild-orchestration/tests/store/test_contracts.rs new file mode 100644 index 0000000000..868430a934 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/test_contracts.rs @@ -0,0 +1,24 @@ +//! Recall storage integration contracts for SQLite and PostgreSQL. + +#[path = "faults/helpers.rs"] +pub(crate) mod fault_helpers; +#[path = "faults/test_types.rs"] +pub(crate) mod fault_types; +#[path = "recall/helpers.rs"] +pub(crate) mod helpers; +#[path = "leases/helpers.rs"] +pub(crate) mod lease_helpers; +#[path = "logical_state/helpers.rs"] +pub(crate) mod logical_state; +#[path = "replicas/helpers.rs"] +pub(crate) mod replica_helpers; +#[path = "helpers.rs"] +pub(crate) mod store_helpers; +#[path = "test_types.rs"] +pub(crate) mod store_test_types; +#[path = "recall/test_contract.rs"] +mod test_contract; +#[path = "recall/test_types.rs"] +mod test_types; +#[path = "recall/test_views.rs"] +mod test_views; diff --git a/crates/sqlbuild-orchestration/tests/store/test_types.rs b/crates/sqlbuild-orchestration/tests/store/test_types.rs new file mode 100644 index 0000000000..d58d5beb56 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/test_types.rs @@ -0,0 +1,16 @@ +//! Store fixtures retain their isolated SQLite directory or PostgreSQL database until cleanup. + +use crate::store::models::ControlStore; + +#[derive(Clone, Copy, Debug)] +pub(crate) enum Backend { + Sqlite, + Postgres, +} + +#[derive(Debug)] +pub(crate) struct TestStore { + pub(crate) store: ControlStore, + pub(crate) directory: Option, + pub(crate) postgres: Option<(sqlx_postgres::PgPool, String)>, +} diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 2219287ac5..b8eaec53bb 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -16,6 +16,8 @@ default = [] extension-module = ["pyo3/extension-module"] [dependencies] +chrono.workspace = true +chrono-tz.workspace = true fensu-policy.workspace = true mimalloc.workspace = true pyo3.workspace = true @@ -24,6 +26,7 @@ rusqlite.workspace = true rustc-hash.workspace = true serde.workspace = true serde_json.workspace = true +sha2.workspace = true signal-hook.workspace = true sqlbuild-analysis.workspace = true sqlbuild-attachments.workspace = true @@ -35,6 +38,7 @@ sqlbuild-cursor.workspace = true sqlbuild-discovery.workspace = true sqlbuild-host.workspace = true sqlbuild-model-config.workspace = true +sqlbuild-orchestration.workspace = true sqlbuild-project.workspace = true sqlbuild-pysource.workspace = true sqlbuild-recall.workspace = true @@ -46,6 +50,8 @@ sqlbuild-sqltext.workspace = true sqlbuild-state.workspace = true sqlclean.workspace = true stacker.workspace = true +tokio.workspace = true +uuid.workspace = true [lints] workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/config_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/config_values.rs index 5f2f304c6b..0a444023a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/boundary/config_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/config_values.rs @@ -104,3 +104,35 @@ pub(crate) fn config_value_to_python(py: Python<'_>, value: ConfigValue) -> PyRe } } } + +/// JSON-backed configuration keeps integer widths and recursively constructs native Python values. +pub(crate) fn json_value_to_python<'py>( + py: Python<'py>, + value: &serde_json::Value, +) -> PyResult> { + use serde_json::Value; + match value { + Value::Null => Ok(py.None().into_bound(py)), + Value::Bool(value) => value.into_bound_py_any(py), + Value::String(value) => value.into_bound_py_any(py), + Value::Number(value) => match (value.as_i64(), value.as_u64()) { + (Some(value), _) => value.into_bound_py_any(py), + (None, Some(value)) => value.into_bound_py_any(py), + (None, None) => value.as_f64().unwrap_or(f64::NAN).into_bound_py_any(py), + }, + Value::Array(items) => { + let values = items + .iter() + .map(|item| json_value_to_python(py, item)) + .collect::>>()?; + Ok(PyList::new(py, values)?.into_any()) + } + Value::Object(entries) => { + let table = PyDict::new(py); + for (key, item) in entries { + table.set_item(key, json_value_to_python(py, item)?)?; + } + Ok(table.into_any()) + } + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 8dbd45566b..3f45271352 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -1,5 +1,6 @@ //! `sqb build`: resolve, compile, gate, execute and report one direct build. +use std::sync::Arc; use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -17,6 +18,7 @@ use crate::bindings::_helpers::build_command::planning::{ compile_build_plan, enforce_build_plan_policies, run_defer_clone_prephase, write_compile_target, write_plan_text, }; +use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::diagnostics::trackers::BuildPhaseTimingTracker; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; @@ -38,6 +40,14 @@ fn run_with_providers(context: &BuildContext<'_>) -> PyResult { } fn run_planned_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) -> PyResult { + if let Some(worker) = &context.worker { + worker.check_cancelled(context.py())?; + if !context.get("effective_defer_clone_from")?.is_none() { + return Err(pyo3::exceptions::PyValueError::new_err( + "worker defer-clone requires claimed boundary writers and is not yet supported", + )); + } + } run_defer_clone_prephase(context)?; let cache_scope: Bound<'_, PyAny> = call_with( &host::entry(context.py(), AdapterHost::RelationMetadataCacheScope)?, @@ -49,6 +59,9 @@ fn run_planned_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) - /// Plan and run the build, sharing one relation metadata cache from planning to the run's end. fn run_cached_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) -> PyResult { let pipeline: Bound<'_, PyAny> = compile_build_plan(context)?; + if let Some(worker) = &context.worker { + worker.enforce_plan_writers(&pipeline)?; + } write_plan_text(context, &pipeline)?; enforce_build_plan_policies(context, &pipeline)?; write_compile_target(context, &pipeline)?; @@ -74,9 +87,11 @@ fn run_within_limits( request: &Bound<'_, PyAny>, tracker: &Bound<'_, PyAny>, started: Instant, + worker: Option>, ) -> PyResult { let py: Python<'_> = request.py(); - let context: BuildContext<'_> = resolve_build_context(request, tracker, started)?; + let mut context: BuildContext<'_> = resolve_build_context(request, tracker, started)?; + context.worker = worker; let limits: Bound<'_, PyAny> = context.get("execution_limits")?; let deadline: Bound<'_, PyAny> = call_with( &host::entry(py, ExecutorHost::ExecutionDeadlineScope)?, @@ -97,6 +112,15 @@ fn run_within_limits( #[pyfunction] #[pyo3(signature = (*, request))] pub(crate) fn run_build_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + run_build(py, request, None) +} + +/// A claim owner survives planning, execution and connection cleanup before terminal reporting. +pub(crate) fn run_build( + py: Python<'_>, + request: &Bound<'_, PyAny>, + worker: Option>, +) -> PyResult { let tracker: Bound<'_, PyAny> = BuildPhaseTimingTracker::new(py, None) .and_then(|value| pyo3::Bound::new(py, value)) .map(pyo3::Bound::into_any)?; @@ -107,7 +131,7 @@ pub(crate) fn run_build_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> P within(&reporting, |_| { within(&tracker.call_method0("scope")?, |_| { let started: Instant = Instant::now(); - run_within_limits(request, &tracker, started).inspect_err(|_| { + run_within_limits(request, &tracker, started, worker.clone()).inspect_err(|_| { let verbose: bool = native_field(request, "verbose") .and_then(|value| value.is_truthy()) .unwrap_or(false) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index f0a0f55d12..e92e997318 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -20,7 +20,7 @@ use crate::bindings::_helpers::build_command::support::{ boolean, display_text, emit, frozenset, joined, keyword_relay, line_writer, none, text, tuple, }; use crate::bindings::_helpers::build_progress::build_callbacks::{ - BuildProgressCallbacks, batch_complete_callback, + BuildProgressCallbacks, batch_complete_callback, check_callback, }; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; @@ -225,6 +225,10 @@ pub(crate) fn prepare_build_execution<'py>( ], )?), ) + .map(|mut value| { + value.worker = context.worker.clone(); + value + }) .and_then(|value| pyo3::Bound::new(py, value)) .map(pyo3::Bound::into_any)?; let concurrency = context.concurrency()?.limit; @@ -592,10 +596,16 @@ pub(crate) fn run_post_build_python_checks<'py>( )?; let callbacks: Bound<'py, PyAny> = call_with( &host::entry(py, ExecutorHost::PythonCheckCallbacks)?, - &[( - "on_check_complete", - native_field(&preparation.callbacks, "write_execution_event")?, - )], + &[ + ( + "on_check_start", + check_callback(&preparation.callbacks, false)?, + ), + ( + "on_check_complete", + check_callback(&preparation.callbacks, true)?, + ), + ], )?; call_with( &host::entry(py, ExecutorHost::ExecutePythonChecks)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index 1f1b394cb0..a2a5d45df6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -1,5 +1,6 @@ //! Resolve discovery, target, adapter, limits, connection and reporters for one build. +use std::sync::Arc; use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -19,6 +20,7 @@ use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_settings_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; @@ -38,6 +40,7 @@ pub(crate) struct BuildContext<'py> { pub(crate) verbose: bool, pub(crate) started: Instant, pub(crate) tracker: Bound<'py, PyAny>, + pub(crate) worker: Option>, } impl<'py> BuildContext<'py> { @@ -241,5 +244,6 @@ pub(crate) fn resolve_build_context<'py>( verbose: native_field(request, "verbose")?.is_truthy()? || debug, started, tracker: tracker.clone(), + worker: None, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs index ec6bbad153..86948474e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs @@ -1,7 +1,7 @@ //! Live progress rows, completion rows and runtime diagnostics of `sqb build`. use std::collections::HashMap; -use std::sync::{Mutex, MutexGuard, PoisonError}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use std::time::Instant; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -31,6 +31,7 @@ use crate::bindings::_helpers::build_progress::terminal::{ consume_terminal, flush, join_spinner, projector, release, shared_lines, start_spinner, terminal_columns, tty_flag, write, }; +use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced_keywords; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; @@ -85,11 +86,8 @@ fn sql_test_resource_id(result: &Bound<'_, PyAny>) -> PyResult { } /// The resource name and projector identity of a completion. -fn resource_identity( - result: &Bound<'_, PyAny>, - kind: ResultKind, -) -> PyResult> { - Ok(match kind { +pub(crate) fn resource_identity(result: &Bound<'_, PyAny>) -> PyResult> { + Ok(match result_kind(result)? { ResultKind::Model => { let name: String = text(result, "model_name")?; Some((name.clone(), format!("model:{name}"))) @@ -141,6 +139,7 @@ struct CallbackState { /// Encapsulates live build progress output state and callbacks. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct BuildProgressCallbacks { + pub(crate) worker: Option>, plan: Py, stream: Py, lines: Py, @@ -453,6 +452,7 @@ impl BuildProgressCallbacks { } let lines: Bound<'_, PyAny> = shared_lines(py)?; Ok(Self { + worker: None, plan: plan.unbind(), is_tty: tty_flag(&stdout)? && !debug, stream: stream.unbind(), @@ -484,6 +484,9 @@ impl BuildProgressCallbacks { ) -> PyResult<()> { let py: Python<'_> = slf.py(); let callbacks: &Self = slf.get(); + if let Some(worker) = &callbacks.worker { + worker.start(py, &name, resource_kind)?; + } if let Some(projector) = &callbacks.projector { let options: Bound<'_, PyDict> = PyDict::new(py); options.set_item("resource_name", name)?; @@ -579,10 +582,15 @@ impl BuildProgressCallbacks { fn on_node_complete(slf: &Bound<'_, Self>, node_result: &Bound<'_, PyAny>) -> PyResult<()> { let py: Python<'_> = slf.py(); let callbacks: &Self = slf.get(); + if let Some(worker) = &callbacks.worker + && let Some((_, key)) = resource_identity(node_result)? + { + worker.completed(node_result, &key)?; + } let kind: ResultKind = result_kind(node_result)?; let mut result: Bound<'_, PyAny> = node_result.clone(); if let Some(projector) = &callbacks.projector - && let Some((name, identity)) = resource_identity(node_result, kind)? + && let Some((name, identity)) = resource_identity(node_result)? { let Some(duration_ms) = consume_terminal(projector.bind(py), &name, Some(&identity))? else { @@ -607,6 +615,20 @@ impl BuildProgressCallbacks { self.write_event(py, result) } + fn on_check_start(&self, py: Python<'_>, name: &str) -> PyResult<()> { + if let Some(worker) = &self.worker { + worker.start_check(py, name)?; + } + Ok(()) + } + + fn on_check_complete(&self, py: Python<'_>, result: &Bound<'_, PyAny>) -> PyResult<()> { + if let Some(worker) = &self.worker { + worker.completed_check(result)?; + } + self.write_event(py, result) + } + /// Close the optional structured execution event stream. fn close(&self, py: Python<'_>) -> PyResult<()> { let (scheduler, query) = { @@ -660,6 +682,28 @@ pub(crate) fn batch_complete_callback<'py>( Ok(PyCFunction::new_closure(callbacks.py(), None, None, forward)?.into_any()) } +/// Forward a Python check lifecycle callback directly to its native progress owner. +pub(crate) fn check_callback<'py>( + callbacks: &Bound<'py, PyAny>, + completion: bool, +) -> PyResult> { + let owned: Py = callbacks + .downcast::()? + .clone() + .unbind(); + let forward = move |arguments: &Bound<'_, PyTuple>, + _keywords: Option<&Bound<'_, PyDict>>| + -> PyResult<()> { + let value = arguments.get_item(0)?; + if completion { + owned.get().on_check_complete(arguments.py(), &value) + } else { + owned.get().on_check_start(arguments.py(), value.extract()?) + } + }; + Ok(PyCFunction::new_closure(callbacks.py(), None, None, forward)?.into_any()) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index f92b9941c0..7c0ebc870f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -23,11 +23,13 @@ use crate::bindings::_helpers::init_command::command::run_init_command; use crate::bindings::_helpers::janitor::janitor_command::run_janitor_command; use crate::bindings::_helpers::lineage::lineage_command::run_lineage_command; use crate::bindings::_helpers::load::load_command::run_load_command; +use crate::bindings::_helpers::location_command::command::run_location; use crate::bindings::_helpers::plan_command::plan::run_plan_command; use crate::bindings::_helpers::plan_command::rules::run_rules_command; use crate::bindings::_helpers::query_command::dag::run_dag_command; use crate::bindings::_helpers::query_command::query::run_query_command; use crate::bindings::_helpers::recall::command::run_recall_command; +use crate::bindings::_helpers::recall::session::InvocationRecall; use crate::bindings::_helpers::records::construct::{new_record_with, record_class}; use crate::bindings::_helpers::refactor::refactor_command::run_refactor_command; use crate::bindings::_helpers::review_command::command::run_review_command; @@ -74,7 +76,7 @@ const RAW_QUERY_ATTRIBUTES: [&str; 4] = [ ]; /// One invocation being dispatched: its namespace and the values derived from it once. -struct Invocation<'py> { +pub(crate) struct Invocation<'py> { args: Bound<'py, PyAny>, project_dir: Bound<'py, PyAny>, select: Bound<'py, PyTuple>, @@ -82,6 +84,27 @@ struct Invocation<'py> { } impl<'py> Invocation<'py> { + pub(crate) fn new(args: &Bound<'py, PyAny>) -> PyResult { + let raw_project_dir = native_field(args, "project_dir")?; + let project_dir = if raw_project_dir.is_none() { + raw_project_dir + } else { + path_class(args.py())? + .call1((raw_project_dir,))? + .call_method0("resolve")? + }; + let (select, selector_files) = selector_inputs(args)?; + if native_field(args, "select_file")?.is_truthy()? { + withhold_launch_record(args.py())?; + } + Ok(Self { + args: args.clone(), + project_dir, + select, + selector_files, + }) + } + fn py(&self) -> Python<'py> { self.args.py() } @@ -95,7 +118,7 @@ impl<'py> Invocation<'py> { } /// The value of one handler argument or request field. - fn value(&self, source: Source) -> PyResult> { + pub(crate) fn value(&self, source: Source) -> PyResult> { let py: Python<'py> = self.py(); match source { Source::Attr(name) => self.attr(name), @@ -312,6 +335,7 @@ fn call_route<'py>( invocation: &Invocation<'py>, route: &Route, handler: Option>, + recall: &InvocationRecall, ) -> PyResult> { let py: Python<'py> = invocation.py(); let mut positional: Vec> = Vec::new(); @@ -339,7 +363,7 @@ fn call_route<'py>( } match handler { Some(handler) => handler.call(PyTuple::new(py, positional)?, Some(&keywords)), - None => run_implementation(route.implementation, (positional, keywords)), + None => run_implementation(route.implementation, (positional, keywords), recall), } } @@ -358,6 +382,7 @@ fn argument<'py>( fn run_implementation<'py>( name: &str, arguments: (Vec>, Bound<'py, PyDict>), + recall: &InvocationRecall, ) -> PyResult> { let py: Python<'py> = arguments.1.py(); let request = || -> PyResult> { @@ -387,6 +412,7 @@ fn run_implementation<'py>( "run_janitor_command" => run_janitor_command(py, &request()?)?, "run_lineage_command" => run_lineage_command(py, &request()?)?, "run_load_command" => run_load_command(py, &request()?)?, + "run_location_command" => run_location(py, options(), Some(recall))?, "run_plan_command" => run_plan_command(&request()?)?, "run_playground_command" => run_playground_command(py, &request()?)?, "run_query_command" => run_query_command(py, options())?, @@ -417,26 +443,10 @@ fn run_implementation<'py>( fn routed_command<'py>( args: &Bound<'py, PyAny>, handlers: Option<&Bound<'py, PyAny>>, + recall: &InvocationRecall, ) -> PyResult> { let py: Python<'py> = args.py(); - let raw_project_dir: Bound<'py, PyAny> = native_field(args, "project_dir")?; - let project_dir: Bound<'py, PyAny> = if raw_project_dir.is_none() { - raw_project_dir - } else { - path_class(py)? - .call1((raw_project_dir,))? - .call_method0("resolve")? - }; - let (select, selector_files) = selector_inputs(args)?; - if native_field(args, "select_file")?.is_truthy()? { - withhold_launch_record(py)?; - } - let invocation = Invocation { - args: args.clone(), - project_dir, - select, - selector_files, - }; + let invocation = Invocation::new(args)?; match command_dispatch(&command_facts(args)?) { Dispatch::Nothing => 0.into_bound_py_any(py), Dispatch::Fail(failure) => Err(user_error(py, failure)), @@ -456,7 +466,7 @@ fn routed_command<'py>( if let Some(failure) = failure { return Err(user_error(py, failure)); } - call_route(&invocation, route, handler) + call_route(&invocation, route, handler, recall) } } } @@ -465,7 +475,14 @@ fn routed_command<'py>( pub(crate) fn dispatch_command<'py>( args: &Bound<'py, PyAny>, handlers: Option>, + recall: &InvocationRecall, ) -> PyResult> { + within(&warehouse_scope(args)?, |_| { + routed_command(args, handlers.as_ref(), recall) + }) +} + +pub(crate) fn warehouse_scope<'py>(args: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = args.py(); let warehouse: Bound<'py, PyAny> = native_field(args, "warehouse")?; if !warehouse.is_none() { @@ -484,7 +501,7 @@ pub(crate) fn dispatch_command<'py>( ) .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?; } - let scope: Bound<'py, PyAny> = Bound::new( + Ok(Bound::new( py, command_warehouse_scope( &native_field(args, "command")?, @@ -492,8 +509,7 @@ pub(crate) fn dispatch_command<'py>( Some(&native_field(args, "vars")?), )?, )? - .into_any(); - within(&scope, |_| routed_command(args, handlers.as_ref())) + .into_any()) } /// The parsed values that select a route, read from the namespace. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 9e2a381159..d684b4bd04 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use std::time::{Instant, SystemTime}; -use pyo3::exceptions::PySystemExit; +use pyo3::exceptions::{PyKeyboardInterrupt, PySystemExit}; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -17,6 +17,7 @@ use sqlbuild_cli::dispatch::main::invocation_policy::invocation_policy; use sqlbuild_cli::dispatch::models::{CommandFacts, CommandFailure, InvocationPolicy}; use sqlbuild_host::entries::models::{ExecutorHost, FacadeHost}; +use sqlbuild_orchestration::runs::models::RunOutcome; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, keywords, within}; use crate::bindings::_helpers::build_progress::transient_lines::shared_transient_line_coordinator; @@ -32,10 +33,12 @@ use crate::bindings::_helpers::cli::termination::{ CommandTerminated, TERMINATED_MESSAGE, TerminationHandler, handle_sigterm, interrupting_signal, signal_exit_code, terminated, terminated_exit, }; +use crate::bindings::_helpers::compile_command::constants::COMPILE_COMMAND_NAME; use crate::bindings::_helpers::compile_command::report::format_compile_error_json; use crate::bindings::_helpers::cost_report::attribution::take_cost_status; use crate::bindings::_helpers::diff_report::output::render_diff_error_json; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::interrupt::{InterruptHandler, interrupted}; use crate::bindings::_helpers::output_capture::records::invocation_context; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::recall::constants::{JSON_FORMAT, RECALL_SETTINGS_CODE}; @@ -123,9 +126,7 @@ fn print_object(value: &Bound<'_, PyAny>, stream: Option<&str>, end: Option<&str if let Some(end) = end { keywords.set_item("end", end)?; } - py.import("builtins")? - .getattr("print")? - .call((value,), Some(&keywords))?; + imported(py, "builtins", "print")?.call((value,), Some(&keywords))?; Ok(()) } @@ -564,9 +565,18 @@ fn observed_dispatch<'py>( if let Some(session) = &invocation.recall.session { session.publish_context(false); } - return dispatch_command(&invocation.args, invocation.handlers.clone()); + return dispatch_command( + &invocation.args, + invocation.handlers.clone(), + &invocation.recall, + ); } let command: String = invocation.command_name(); + let _interrupt: Option = if command == COMPILE_COMMAND_NAME { + InterruptHandler::install(py)? + } else { + None + }; drop(take_failure_cause()); drop(take_cost_status()); let started: Instant = Instant::now(); @@ -620,11 +630,15 @@ fn observed_dispatch<'py>( "invocation_started", started_payload()?, )?)?; - let dispatched: PyResult> = - match dispatch_command(&invocation.args, invocation.handlers.clone()) { - Ok(_) if terminated() => Err(CommandTerminated::new_err(TERMINATED_MESSAGE)), - dispatched => dispatched, - }; + let dispatched: PyResult> = match dispatch_command( + &invocation.args, + invocation.handlers.clone(), + &invocation.recall, + ) { + Ok(_) if terminated() => Err(CommandTerminated::new_err(TERMINATED_MESSAGE)), + Ok(_) if interrupted() => Err(PyKeyboardInterrupt::new_err(())), + dispatched => dispatched, + }; match dispatched { Ok(exit_code) => { let payload: Bound<'py, PyDict> = PyDict::new(py); @@ -783,6 +797,33 @@ fn parsed_invocation<'py>( Ok(exit_code) }; } + let namespace = args.getattr("__dict__")?.cast_into::()?; + let command: Option = namespace + .get_item("command")? + .map(|value| value.extract::>()) + .transpose()? + .flatten(); + let location_command: Option = namespace + .get_item("location_command")? + .map(|value| value.extract::>()) + .transpose()? + .flatten(); + if command.as_deref() == Some("location") && location_command.as_deref() == Some("runtime") { + return within(&host::redirect_stdout_to_stderr(py)?, |_| { + execute_parsed_invocation(args, handlers, identity, use_color) + }); + } + execute_parsed_invocation(args, handlers, identity, use_color) +} + +/// Runtime stream routing must cover extension discovery, invocation scopes and their teardown. +fn execute_parsed_invocation<'py>( + args: Bound<'py, PyAny>, + handlers: Option>, + identity: &Bound<'py, PyAny>, + use_color: bool, +) -> PyResult> { + let py = args.py(); let facts: CommandFacts = command_facts(&args)?; let policy: InvocationPolicy = invocation_policy(&facts); let recall: InvocationRecall = if policy.recalled { @@ -858,6 +899,30 @@ fn parsed_invocation<'py>( Ok(_) | Err(_) => None, }; let finished: PyResult<()> = finish_invocation_recall(py, &recall, exit_status); + let worker = recall + .worker + .lock() + .map_err(|_| { + pyo3::exceptions::PyRuntimeError::new_err("worker completion lock is unavailable") + })? + .take(); + if let Some(worker) = worker { + let code = match &finished { + Ok(()) => exit_status.unwrap_or(1), + Err(_) => 1, + }; + let outcome = worker.finish(py, i32::try_from(code).unwrap_or(1))?; + host::write_console( + py, + "stderr", + &format!("Worker execution completed: {}\n", outcome.as_str()), + )?; + if !matches!(outcome, RunOutcome::Success | RunOutcome::Warn) { + dispatched?; + finished?; + return exit(py, code.max(1)); + } + } let exit_code: Bound<'py, PyAny> = dispatched?; finished?; Ok(exit_code) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs index 6a26b63e16..0f48c13014 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs @@ -15,7 +15,7 @@ use sqlbuild_cli::dispatch::models::{ExportSummary, ExporterLoss}; use sqlbuild_core::event_export::constants::DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS; use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; -use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, keywords, within}; use crate::bindings::_helpers::build_progress::progress::NativeProgressProjector; use crate::bindings::_helpers::build_progress::progress::machine_output_mode; use crate::bindings::_helpers::cli::launch_state::withhold_launch_record; @@ -337,10 +337,17 @@ impl Observed { ) -> PyResult<()> { let mut command_output_sinks: bool = false; if native_field(args, "command")?.ne(LINEAGE_COMMAND)? { - let extensions: Bound<'_, PyAny> = call_with( - &host::entry(py, UserCodeHost::DiscoverRuntimeExtensions)?, - &[("project_dir", project_dir.clone())], - )?; + let discover = || { + call_with( + &host::entry(py, UserCodeHost::DiscoverRuntimeExtensions)?, + &[("project_dir", project_dir.clone())], + ) + }; + let extensions: Bound<'_, PyAny> = if machine_output_mode(args)? { + within(&host::redirect_stdout_to_stderr(py)?, |_| discover())? + } else { + discover()? + }; let lifecycle_replacement: Bound<'_, PyAny> = native_field(&extensions, "recall_lifecycle_sink")?; let output_replacement: Bound<'_, PyAny> = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs index 8a824ae6e6..495410b7a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs @@ -189,18 +189,8 @@ fn anchor_attach_paths(config: &Bound<'_, PyDict>, project_dir: &Bound<'_, PyAny anchored.push(entry); continue; }; - let local_file: bool = match mapping.get_item(ATTACH_TYPE_KEY)? { - None => true, - Some(kind) => kind - .extract::() - .is_ok_and(|kind| LOCAL_ATTACH_TYPES.contains(&kind.to_ascii_lowercase().as_str())), - }; let copy: Bound<'_, PyDict> = mapping.copy()?; - if let Some(path_value) = mapping.get_item(ATTACH_PATH_KEY)? - && local_file - && !path_value.extract::().is_ok_and(|text| { - text.contains(URL_SCHEME_SEPARATOR) || text.starts_with(MOTHERDUCK_PATH_PREFIX) - }) + if let Some(path_value) = local_attach_file(mapping)? && let Some(file) = anchored_file_path(&path_value, project_dir)? { copy.set_item(ATTACH_PATH_KEY, file)?; @@ -210,6 +200,59 @@ fn anchor_attach_paths(config: &Bound<'_, PyDict>, project_dir: &Bound<'_, PyAny config.set_item(ATTACH_KEY, pyo3::types::PyList::new(py, anchored)?) } +fn local_attach_file<'py>(mapping: &Bound<'py, PyDict>) -> PyResult>> { + let local = match mapping.get_item(ATTACH_TYPE_KEY)? { + None => true, + Some(kind) => kind + .extract::() + .is_ok_and(|kind| LOCAL_ATTACH_TYPES.contains(&kind.to_ascii_lowercase().as_str())), + }; + Ok(mapping.get_item(ATTACH_PATH_KEY)?.filter(|value| { + local + && !value.extract::().is_ok_and(|text| { + text.contains(URL_SCHEME_SEPARATOR) || text.starts_with(MOTHERDUCK_PATH_PREFIX) + }) + })) +} + +/// Use the resolved adapter connection, including local attachments, not authored TOML guesses. +pub(crate) fn local_database_files( + adapter: &str, + config: &Bound<'_, PyDict>, +) -> PyResult> { + let Some((_, key)) = LOCAL_DATABASE_PATH_KEYS + .iter() + .find(|(name, _)| *name == adapter) + else { + return Ok(Vec::new()); + }; + let mut files: Vec = Vec::new(); + if let Some(value) = config.get_item(*key)? + && let Ok(file) = value.extract::() + && !file.is_empty() + && file != DUCKDB_MEMORY_DATABASE + && !file.contains(URL_SCHEME_SEPARATOR) + && !file.starts_with(MOTHERDUCK_PATH_PREFIX) + { + files.push(file); + } + if adapter == DUCKDB_ADAPTER + && let Some(entries) = config.get_item(ATTACH_KEY)? + && let Ok(entries) = entries.try_iter() + { + for entry in entries { + if let Ok(mapping) = entry?.cast_into::() + && let Some(value) = local_attach_file(&mapping)? + && let Ok(file) = value.extract::() + && file != DUCKDB_MEMORY_DATABASE + { + files.push(file); + } + } + } + Ok(files) +} + /// Resolve dbt profiles, relative DuckDB paths, and the command group's warehouse. #[pyfunction] #[pyo3(signature = (**options))] @@ -323,6 +366,7 @@ pub(crate) fn project_connection_config<'py>( discovered_inputs: &Bound<'py, PyAny>, (project_dir, selected_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), (cli_vars, require_connection): (&Bound<'py, PyAny>, bool), + catalog: bool, ) -> PyResult> { if require_connection { require_configured_connection(discovered_inputs, project_dir, selected_target)?; @@ -331,6 +375,7 @@ pub(crate) fn project_connection_config<'py>( &discovered_inputs.clone(), Some(selected_target.clone()).filter(|value| !value.is_none()), Some(cli_vars.clone()).filter(|value| !value.is_none()), + catalog, )? .cast_into()?; let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( @@ -367,6 +412,7 @@ pub(crate) fn resolve_project_connection_config<'py>( &arguments.value("selected_target")?, ), (&arguments.value("cli_vars")?, require_connection), + false, ) } @@ -390,6 +436,7 @@ pub(crate) fn resolve_target_connection_config<'py>( &arguments.value("target_name")?, ), (&arguments.value("cli_vars")?, true), + false, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs index 382aa2f4bc..e1936e7e25 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs @@ -7,9 +7,9 @@ use pyo3::types::{PyDict, PyString, PyTuple}; use sqlbuild_model_config::templates::models::TemplateOptions; use crate::bindings::_helpers::boundary::python_calls::imported; -use crate::bindings::_helpers::contracts::effective_config::effective_config_connection; use crate::bindings::_helpers::contracts::effective_config::effective_config_settings; use crate::bindings::_helpers::contracts::effective_config::effective_config_vars; +use crate::bindings::_helpers::contracts::effective_config::effective_connection_fields; use crate::bindings::_helpers::contracts::spec_resolution::{ resolve_target_config, resolve_target_name, }; @@ -31,10 +31,25 @@ pub(crate) fn build_effective_connection<'py>( local_config: &Bound<'py, PyAny>, target_config: &Bound<'py, PyAny>, effective_vars: &Bound<'py, PyAny>, +) -> PyResult> { + expanded_connection( + (project_config, local_config), + target_config, + effective_vars, + false, + ) +} + +/// Expand only the connection selected for the caller's writer or catalog role. +fn expanded_connection<'py>( + (project_config, local_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + target_config: &Bound<'py, PyAny>, + effective_vars: &Bound<'py, PyAny>, + catalog: bool, ) -> PyResult> { let py: Python<'py> = project_config.py(); let (connection, error): (Bound<'py, PyAny>, Option) = - effective_config_connection(py, project_config, local_config, target_config) + effective_connection_fields(py, (project_config, local_config), target_config, catalog) .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? .extract()?; if connection.is_none() { @@ -156,20 +171,24 @@ pub(crate) fn build_effective_target_config<'py>( /// Build the effective project connection config without compiling resources. #[pyfunction] -#[pyo3(signature = (*, discovered_inputs, selected_target = None, cli_vars = None))] +#[pyo3(signature = (*, discovered_inputs, selected_target = None, cli_vars = None, catalog = false))] pub(crate) fn build_effective_connection_config<'py>( discovered_inputs: &Bound<'py, PyAny>, selected_target: Option>, cli_vars: Option>, + catalog: bool, ) -> PyResult> { let (_, target_config) = target_selection(discovered_inputs, selected_target)?; let effective_vars: Bound<'py, PyAny> = discovered_vars(discovered_inputs, &target_config, cli_vars)?; - build_effective_connection( - &native_field(discovered_inputs, "project_config")?, - &native_field(discovered_inputs, "local_config")?, + expanded_connection( + ( + &native_field(discovered_inputs, "project_config")?, + &native_field(discovered_inputs, "local_config")?, + ), &target_config, &effective_vars, + catalog, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index f9178da352..9f47662260 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -1,5 +1,7 @@ //! Compiled project assembly: the project's models, resources, checks and semantic diagnostics. +use std::path::PathBuf; + use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyStringMethods, PyTuple}; @@ -29,7 +31,7 @@ use crate::bindings::_helpers::model_config::template_data::early_model_template use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::new_record; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::fields::mapping_get; +use crate::bindings::_helpers::records::fields::{mapping_get, path_posix}; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::scopes::scope_errors::report_scope_index_errors; use crate::bindings::_helpers::scopes::scope_validation::scope_index_with_compile_usages; @@ -287,6 +289,26 @@ fn project_fields<'py>( for name in ["loader_functions", "hook_functions"] { fields.set_item(name, native_field(&discovered, name)?)?; } + let python_inputs: Bound<'py, PyDict> = PyDict::new(py); + for (kind, collection) in [ + ("task", "task_functions"), + ("asset", "asset_functions"), + ("check", "check_functions"), + ] { + for function in native_field(&discovered, collection)?.try_iter()? { + let function = function?; + let name: String = native_field(&function, "name")?.extract()?; + let path: PathBuf = native_field(&function, "file_path")?.extract()?; + python_inputs.set_item( + format!("{kind}:{name}"), + ( + path_posix(&native_field(&function, "relative_path")?)?, + path, + ), + )?; + } + } + fields.set_item("python_execution_inputs", python_inputs)?; fields.set_item( "enforce_explicit_references", native_field(&project_config, "references")?.getattr("enforce_explicit")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index ddcf1ae6dd..689bbfafe6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -75,6 +75,7 @@ pub(crate) fn compiled_phase<'py>( discovered, Some(native_field(&options, "selected_target")?), Some(native_field(&options, "cli_vars")?), + false, )? } else { configured diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index dd9f0a762b..41b587e0f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -274,7 +274,7 @@ fn target_namespaces( let (_, effective_vars, _): (Bound<'_, PyAny>, Bound<'_, PyAny>, Bound<'_, PyAny>) = runtime.extract()?; let connection: Bound<'_, PyAny> = if resolved_connection.is_none() { - build_effective_connection_config(&discovered.clone(), Some(name), None) + build_effective_connection_config(&discovered.clone(), Some(name), None, false) .and_then(|value| value.into_bound_py_any(py))? } else { resolved_connection.clone() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs index 33cf4c8205..15329c6c5b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs @@ -1590,6 +1590,7 @@ const R_COMPILED_PROJECT: RecordSpec = RecordSpec { compared: false, shown: false, }, + FieldSpec::new("python_execution_inputs", FieldDefault::EmptyDict), ], }; const R_COMPILED_SQL_TEST: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs new file mode 100644 index 0000000000..a472d4d696 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs @@ -0,0 +1,5 @@ +//! Frozen execution protocol values and bounded acknowledgement retries. + +pub(crate) const SKIP_ACTION: &str = "skip"; +pub(crate) const NO_CHANGE_REASON: &str = "no_change"; +pub(crate) const REPORT_ATTEMPTS: usize = 3; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index bbd4e1a61a..2101d9ab7d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -1,6 +1,14 @@ pub(crate) mod attachment_inputs; +pub(crate) mod constants; pub(crate) mod project; pub(crate) mod registration; pub(crate) mod rows; pub(crate) mod schema_facts; +pub(crate) mod snapshot; +pub(crate) mod snapshot_boundary; +pub(crate) mod snapshot_declarations; +pub(crate) mod snapshot_execution; +pub(crate) mod snapshot_models; +pub(crate) mod snapshot_resources; +pub(crate) mod snapshot_worker; pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs index fd619ffe5f..2c8daa3464 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs @@ -6,5 +6,7 @@ use crate::bindings::_helpers::{compiled_views, input_loops}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::compiled_project::project::register(module)?; input_loops::registration::register(module)?; - compiled_views::registration::register(module) + compiled_views::registration::register(module)?; + crate::bindings::_helpers::location_command::command::register(module)?; + crate::bindings::_helpers::compiled_project::snapshot::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs new file mode 100644 index 0000000000..ff338a5e1f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -0,0 +1,104 @@ +//! Project metadata stays native until its immutable snapshot is sent over HTTP. + +use std::collections::BTreeMap; + +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyclass, pymethods}; +use serde_json::Value; +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode, SnapshotRecord}; + +use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; + +/// A data-only publication; this object never contains adapters, connections or user callables. +#[pyclass(frozen, module = "sqlbuild._native")] +pub(crate) struct NativeOrchestrationSnapshot { + pub(crate) input: SnapshotInput, +} + +impl NativeOrchestrationSnapshot { + /// Native callers retain transport errors so they can report an uncertain acknowledgement. + pub(crate) fn publication( + &self, + py: Python<'_>, + server_url: &str, + bearer: &str, + ) -> PyResult> { + py.detach(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let result = runtime.block_on(publish_remote_snapshot(server_url, bearer, &self.input)); + runtime.shutdown_timeout(std::time::Duration::from_secs(1)); + Ok(result) + }) + } +} + +#[pymethods] +impl NativeOrchestrationSnapshot { + /// Serialize only at the public boundary; native transports can use the retained input. + fn to_json(&self, py: Python<'_>) -> PyResult { + py.detach(|| serde_json::to_string(&self.input)) + .map_err(|error| PyValueError::new_err(error.to_string())) + } + + /// Publish retained data without projecting it through Python, releasing the GIL during IO. + fn publish(&self, py: Python<'_>, server_url: &str, bearer: &str) -> PyResult { + let record = self + .publication(py, server_url, bearer)? + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + py.detach(|| { + serde_json::to_string(&record) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + /// Retain the verified claim and its independent native heartbeat in the worker interpreter. + fn start_worker( + &self, + py: Python<'_>, + server_url: &str, + bearer: &str, + identity: (String, String), + ) -> PyResult { + NativeOrchestrationWorker::start(self, py, (server_url, bearer), identity) + } +} + +/// Duplicate resource identities must fail, rather than silently replacing an earlier node. +pub(crate) fn insert_node( + mut nodes: BTreeMap, + key: String, + node: SnapshotNode, +) -> PyResult> { + if nodes.insert(key, node).is_some() { + return Err(PyValueError::new_err("duplicate snapshot node identity")); + } + Ok(nodes) +} + +pub(crate) fn dependency_keys(deps: &[(String, String)]) -> Vec { + deps.iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect() +} + +pub(crate) fn destination(value: &sqlbuild_project::models::DestinationFact) -> Value { + serde_json::json!({ + "database": value.database, + "schema": value.schema, + "name": value.name, + "qualified_name": value.qualified_name, + "logical_database": value.logical_database, + "logical_schema": value.logical_schema, + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs new file mode 100644 index 0000000000..e6093663b6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs @@ -0,0 +1,192 @@ +//! Discovery files and authored test payloads remain the Python host's boundary. + +use std::collections::BTreeMap; +use std::fs::File; +use std::io::Read; +use std::path::PathBuf; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use serde_json::{Map, Value, json}; +use sha2::{Digest, Sha256}; +use sqlbuild_orchestration::snapshots::models::SnapshotNode; + +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::snapshot::{destination, insert_node}; +use crate::bindings::_helpers::compiled_project::values::plain_value; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::compiled_views::seed_views::{NativeCompiledSeed, SeedViews}; +use crate::bindings::_helpers::records::fields::{field, path_posix}; + +/// Read only declared metadata fields, never callable attributes or connection settings. +pub(crate) fn metadata(value: &Bound<'_, PyAny>, names: &[&str]) -> PyResult { + let mut result = Map::new(); + for name in names { + result.insert((*name).into(), plain_value(&field(value, name)?)?); + } + Ok(Value::Object(result)) +} + +pub(crate) fn relative_path(value: &Bound<'_, PyAny>) -> PyResult { + path_posix(&field(value, "relative_path")?) +} + +/// Source locations are local compiler diagnostics, not relocatable column metadata. +pub(crate) fn columns(value: &Bound<'_, PyAny>) -> PyResult> { + value + .try_iter()? + .map(|column| { + metadata( + &column?, + &["name", "type", "nullable", "description", "meta"], + ) + }) + .collect() +} + +fn file_digest(value: &Bound<'_, PyAny>) -> PyResult { + let path: PathBuf = field(value, "file_path")?.extract()?; + value.py().detach(|| { + let mut file = File::open(path)?; + let mut hash = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let length = file.read(&mut buffer)?; + if length == 0 { + return Ok(format!("{:x}", hash.finalize())); + } + hash.update(&buffer[..length]); + } + }) +} + +pub(crate) fn boundary_nodes( + py: Python<'_>, + project: &NativeProjectView, + owner: &NativeCompiledProject, + mut nodes: BTreeMap, +) -> PyResult> { + let seed_views = project.seeds(py); + for (seed, view) in owner.seeds().iter().zip(seed_views.bind(py).try_iter()?) { + let effective = view? + .cast::()? + .get() + .current_destination(py)?; + let parts = seed.parts(); + let seed_file = parts.seed_file.bind(py); + nodes = insert_node( + nodes, + format!("seed:{}", seed.name), + SnapshotNode { + kind: "seed".into(), + definition: json!({ + "name": seed.name, + "path": relative_path(seed_file)?, + "content_hash": file_digest(seed_file)?, + "destination": destination(&effective), + "schema": metadata(parts.schema_entry.bind(py), &[ + "tags", "description", "meta", "csv_settings", + ])?, + "columns": columns(&field(parts.schema_entry.bind(py), "columns")?)?, + }), + raw_sql: None, + compiled_sql: None, + python_source: None, + }, + )?; + } + for (kind, objects, names) in [ + ( + "sql_test", + project.sql_tests(py), + ["mode", "payload", "parameter_schema", "parameter_values"].as_slice(), + ), + ( + "sql_scenario", + project.sql_scenarios(py), + ["authored_ctes", "expected_ctes", "assertion_ctes"].as_slice(), + ), + ] { + for item in objects.bind(py).try_iter()? { + let item = item?; + let key = field(&item, "key")?; + let name: String = field(&key, "name")?.extract()?; + let resource_type: String = field(&key, "resource_type")?.str()?.extract()?; + let definition = metadata(&item, names)?; + nodes = insert_node( + nodes, + format!("{resource_type}:{name}"), + SnapshotNode { + kind: kind.into(), + definition: json!({"name": name, "test": definition}), + raw_sql: None, + compiled_sql: Some(field(&item, "sql_body")?.extract()?), + python_source: None, + }, + )?; + } + } + let mut python_sources: Vec<(String, (String, PathBuf))> = + project.python_execution_inputs().into_iter().collect(); + for (kind, objects) in [ + ("loader", project.loader_functions(py)), + ("hook", project.hook_functions(py)), + ("materialization", project.materialization_files(py)), + ] { + for item in objects.bind(py).try_iter()? { + let item = item?; + let name: String = field(&item, "name")?.extract()?; + let path: PathBuf = field(&item, "file_path")?.extract()?; + python_sources.push((format!("{kind}:{name}"), (relative_path(&item)?, path))); + } + } + for (key, (relative, path)) in python_sources { + let (kind, name) = key + .split_once(':') + .ok_or_else(|| PyValueError::new_err("invalid Python execution input identity"))?; + let source = py.detach(|| std::fs::read_to_string(path))?; + nodes = insert_node( + nodes, + key.clone(), + SnapshotNode { + kind: kind.into(), + definition: json!({"name": name, "path": relative}), + raw_sql: None, + compiled_sql: None, + python_source: Some(source), + }, + )?; + } + for item in project.sql_hook_files(py).bind(py).try_iter()? { + let item = item?; + let name: String = field(&item, "name")?.extract()?; + nodes = insert_node( + nodes, + format!("sql_hook:{name}"), + SnapshotNode { + kind: "sql_hook".into(), + definition: json!({"name": name, "path": relative_path(&item)?}), + raw_sql: Some(field(&item, "contents")?.extract()?), + compiled_sql: Some(field(&item, "sql_body")?.extract()?), + python_source: None, + }, + )?; + } + let macros = project.loaded_macros(py); + for (key, item) in macros.bind(py).cast::()?.iter() { + let name: String = key.extract()?; + nodes = insert_node( + nodes, + format!("macro:{name}"), + SnapshotNode { + kind: "macro".into(), + definition: json!({"name": name, "path": relative_path(&item)?}), + raw_sql: None, + compiled_sql: None, + python_source: Some(field(&item, "raw_source")?.extract()?), + }, + )?; + } + Ok(nodes) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs new file mode 100644 index 0000000000..0c2ac4c08b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -0,0 +1,234 @@ +//! Compile authoring records against retained graph data before publication. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use serde_json::Value; +use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; +use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM; +use sqlbuild_discovery::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; +use sqlbuild_discovery::tree::main::rglob::rglob; +use sqlbuild_discovery::tree::models::ProjectTree; +use sqlbuild_orchestration::declarations::main::group_dependencies::group_dependencies; +use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; +use sqlbuild_orchestration::declarations::models::{ + Definitions, GroupDefinition, JobDefinition, TriggerDeclaration, TriggerDefinition, +}; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; + +use crate::bindings::_helpers::native_host::orchestration_declarations::{ + AuthoredDeclaration, AuthoredGroup, AuthoredJob, AuthoredSensor, module_jobs, +}; +use crate::bindings::_helpers::native_host::orchestration_triggers::{ + compile_calendars, compile_cron, compile_sensors, +}; + +/// Dedicated declaration modules are imported only in the code-location interpreter. +pub(crate) fn discover_declarations<'py>( + py: Python<'py>, + project_dir: &Path, +) -> PyResult>> { + let tree = ProjectTree::new(project_dir); + let files = py + .detach(|| { + rglob(&tree, ORCHESTRATION_DIRECTORY, |entry| { + !entry.is_dir + && entry.name.ends_with(PYTHON_FILE_SUFFIX) + && entry.name.strip_suffix(PYTHON_FILE_SUFFIX) != Some(PYTHON_INIT_MODULE_STEM) + }) + }) + .map_err(|error| { + PyValueError::new_err(format!("orchestration discovery failed: {error:?}")) + })?; + let mut declarations: Vec> = Vec::new(); + let mut seen: BTreeSet = BTreeSet::new(); + for file in files { + for job in module_jobs(py, &tree.absolute(&file), project_dir)? { + if seen.insert(job.as_ptr() as usize) { + declarations.push(job); + } + } + } + Ok(declarations) +} + +pub(crate) fn compile_declarations( + py: Python<'_>, + mut input: SnapshotInput, + (project_dir, graph): (&Path, &ProjectGraph), + declarations: Vec, +) -> PyResult { + if declarations.is_empty() { + return Ok(input); + } + if ["jobs", "groups", "triggers"] + .iter() + .any(|key| input.documents.contains_key(*key)) + { + return Err(PyValueError::new_err( + "authored declarations cannot overwrite snapshot job or trigger documents", + )); + } + let mut jobs: BTreeMap = BTreeMap::new(); + let mut authored_groups: Vec = Vec::new(); + let mut authored_jobs: Vec = Vec::new(); + let mut authored_calendars: Vec> = Vec::new(); + let mut authored_sensors: Vec = Vec::new(); + for declaration in declarations { + match declaration { + AuthoredDeclaration::Job(job) => authored_jobs.push(job), + AuthoredDeclaration::Group(group) => authored_groups.push(group), + AuthoredDeclaration::Calendar(calendar) => authored_calendars.push(calendar), + AuthoredDeclaration::Sensor(sensor) => authored_sensors.push(sensor), + } + } + let mut triggers: BTreeMap = BTreeMap::new(); + let mut names: BTreeSet = BTreeSet::new(); + for job in authored_jobs { + if !names.insert(job.name.clone()) { + return Err(PyValueError::new_err(format!("duplicate job {}", job.name))); + } + if !job.targets.is_empty() && !job.targets.contains(&input.target) { + continue; + } + let mut settings: BTreeMap = serde_json::from_value(job.settings) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + if !job.concurrency.is_null() { + if settings.contains_key("concurrency") { + return Err(PyValueError::new_err("job concurrency is declared twice")); + } + settings.insert("concurrency".into(), job.concurrency); + } + job_settings_arguments(&settings).map_err(PyValueError::new_err)?; + let tokens = |value: &str| { + value + .split_whitespace() + .map(str::to_owned) + .collect::>() + }; + if job.select.trim().is_empty() { + return Err(PyValueError::new_err(format!( + "job {} needs a selection", + job.name + ))); + } + let selection: BTreeSet = + resolve_selectors(graph, &tokens(&job.select), &tokens(&job.exclude)) + .map_err(|error| PyValueError::new_err(error.message))? + .into_iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect(); + if let Some(schedule) = job.schedule { + triggers.insert( + job.name.clone(), + compile_cron(schedule.bind(py), &job.name, &job.description)?, + ); + } + jobs.insert( + job.name, + JobDefinition { + description: job.description, + selection, + settings, + concurrency_groups: job.concurrency_groups.into_iter().collect(), + }, + ); + } + let mut groups: BTreeMap = BTreeMap::new(); + for group in authored_groups { + if !names.insert(group.name.clone()) { + return Err(PyValueError::new_err(format!( + "duplicate job or group {}", + group.name + ))); + } + if !group.targets.is_empty() && !group.targets.contains(&input.target) { + continue; + } + let members: BTreeSet = group.jobs.iter().map(|job| job.name.clone()).collect(); + if members.len() != group.jobs.len() { + return Err(PyValueError::new_err("group members must be distinct")); + } + let mut after: BTreeMap> = BTreeMap::new(); + for (job, dependencies) in group.after { + after.insert( + job.name, + dependencies + .into_iter() + .map(|dependency| dependency.name) + .collect(), + ); + } + let dependencies = group_dependencies(&input, &jobs, (&members, &after)) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + let selection: BTreeSet = members + .iter() + .flat_map(|name| jobs[name].selection.iter().cloned()) + .collect(); + if let Some(schedule) = group.schedule { + triggers.insert( + group.name.clone(), + compile_cron(schedule.bind(py), &group.name, &group.description)?, + ); + } + groups.insert( + group.name, + GroupDefinition { + job: JobDefinition { + description: group.description, + selection, + settings: BTreeMap::new(), + concurrency_groups: group.concurrency_groups.into_iter().collect(), + }, + jobs: dependencies, + }, + ); + } + let Definitions { + jobs, + groups, + triggers, + sensors, + .. + } = compile_sensors( + py, + (project_dir, &input, graph), + authored_sensors, + compile_calendars( + py, + (project_dir, &input.target), + authored_calendars, + Definitions { + jobs, + groups, + triggers, + ..Definitions::default() + }, + )?, + )?; + let triggers: BTreeMap<_, _> = triggers + .into_iter() + .map(|(name, trigger)| (name, TriggerDeclaration::Schedule(trigger))) + .chain( + sensors + .into_iter() + .map(|(name, sensor)| (name, TriggerDeclaration::Sensor(sensor))), + ) + .collect(); + for (key, document) in [ + ("jobs", serde_json::to_value(jobs)), + ("groups", serde_json::to_value(groups)), + ("triggers", serde_json::to_value(triggers)), + ] { + input.documents.insert( + key.into(), + document.map_err(|error| PyValueError::new_err(error.to_string()))?, + ); + } + snapshot_definitions(&input).map_err(|error| PyValueError::new_err(error.to_string()))?; + Ok(input) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs new file mode 100644 index 0000000000..ded2b6c2b6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -0,0 +1,342 @@ +//! Frozen selection, executor callbacks and completion belong to one verified worker build. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use pyo3::exceptions::{PyKeyboardInterrupt, PyRuntimeError, PyValueError}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyStringMethods; +use sqlbuild_core::models::ResourceKind; +use sqlbuild_orchestration::runs::models::{NodeOutcome, RunChange, RunExecution, RunOutcome}; + +use crate::bindings::_helpers::compiled_project::constants::{NO_CHANGE_REASON, SKIP_ACTION}; +use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; +use crate::bindings::_helpers::records::fields::field; + +/// This transient owner records only acknowledged outcomes; durable lifecycle remains on the API. +#[derive(Debug)] +pub(crate) struct WorkerBuild { + pub(crate) worker: Py, + execution: RunExecution, + completed: Mutex>, + checks: Mutex>>, +} + +impl WorkerBuild { + pub(crate) fn new(worker: &Bound<'_, NativeOrchestrationWorker>) -> PyResult> { + let state = Arc::new(Self { + execution: worker.get().execution(worker.py())?, + worker: worker.clone().unbind(), + completed: Mutex::default(), + checks: Mutex::default(), + }); + for key in state.execution.skipped_nodes.keys() { + state.complete(worker.py(), key, (NodeOutcome::Skipped, None, None))?; + } + Ok(state) + } + + pub(crate) fn has_work(&self) -> bool { + self.execution + .claim + .request + .selection + .iter() + .any(|key| !self.execution.skipped_nodes.contains_key(key)) + } + + pub(crate) fn check_cancelled(&self, py: Python<'_>) -> PyResult<()> { + if self.worker.bind(py).get().cancel_requested(py) { + return Err(PyKeyboardInterrupt::new_err("run cancellation requested")); + } + Ok(()) + } + + /// Reject an implicitly expanded write set before any executor connection or schema change. + pub(crate) fn enforce_plan_writers( + &self, + pipeline: &Bound<'_, PyAny>, + ) -> Result<(), pyo3::PyErr> { + self.check_cancelled(pipeline.py())?; + let plan = field(pipeline, "plan_output")?; + for collection in [ + "model_entries", + "seed_entries", + "function_entries", + "source_load_entries", + ] { + for entry in field(&plan, collection)?.try_iter()? { + let entry = entry?; + let identity = field(&entry, "key")?; + let kind = field(&identity, "resource_type")? + .str()? + .to_str()? + .to_owned(); + let key = format!("{kind}:{}", field(&identity, "name")?.str()?); + if !self.execution.claim.request.write_set.contains(&key) { + return Err(PyValueError::new_err(format!( + "planned writer {key} is outside the frozen run write set" + ))); + } + if kind == ResourceKind::Model.as_str() + && field(&entry, "action")?.str()?.to_str()? == SKIP_ACTION + { + let outcome = if field(&entry, "reason")?.str()?.to_str()? == NO_CHANGE_REASON { + NodeOutcome::Success + } else { + NodeOutcome::Skipped + }; + self.complete(pipeline.py(), &key, (outcome, None, None))?; + } + } + } + Ok(()) + } + + fn complete( + &self, + py: Python<'_>, + key: &str, + (outcome, rows_affected, error_code): (NodeOutcome, Option, Option), + ) -> PyResult<()> { + if !self.execution.claim.request.selection.contains(key) { + return Ok(()); + } + if self + .completed + .lock() + .map_err(|_| poisoned())? + .contains_key(key) + { + return Ok(()); + } + self.worker.bind(py).get().change( + py, + &format!("node-finished/{key}"), + RunChange::NodeFinished { + node_key: key.into(), + outcome, + rows_affected, + error_code, + skipped_because_of: self.execution.skipped_nodes.get(key).cloned(), + }, + )?; + self.completed + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), outcome); + Ok(()) + } + + pub(crate) fn start( + &self, + py: Python<'_>, + name: &str, + resource_kind: &Bound<'_, PyAny>, + ) -> PyResult<()> { + self.check_cancelled(py)?; + let kind = field(resource_kind, "value")?.extract::()?; + let kind = match kind.as_str() { + "table" | "view" | "snapshot" | "custom" => "model", + kind => kind, + }; + let key = format!("{kind}:{name}"); + if self.execution.claim.request.selection.contains(&key) { + self.worker.bind(py).get().change( + py, + &format!("node-started/{key}"), + RunChange::NodeStarted { node_key: key }, + )?; + } + Ok(()) + } + + /// Existing native progress callbacks publish selected-node evidence without Python relays. + pub(crate) fn completed(&self, result: &Bound<'_, PyAny>, key: &str) -> PyResult<()> { + if self.execution.claim.request.selection.contains(key) { + let status: String = field(result, "status")?.str()?.extract()?; + let outcome = match status.as_str() { + "success" => { + if result.hasattr("warning_messages")? + && field(result, "warning_messages")?.is_truthy()? + { + NodeOutcome::Warn + } else { + NodeOutcome::Success + } + } + "failed" => { + if self + .worker + .bind(result.py()) + .get() + .refresh_cancellation(result.py())? + { + NodeOutcome::Cancelled + } else { + NodeOutcome::Failed + } + } + "skipped" => NodeOutcome::Skipped, + _ => { + return Err(PyValueError::new_err(format!( + "unsupported worker node outcome {status}" + ))); + } + }; + let rows = match result.getattr_opt("rows_affected")? { + Some(value) => value.extract()?, + None => None, + }; + let error = match result.getattr_opt("error_code")? { + Some(value) => value.extract()?, + None => None, + }; + self.complete(result.py(), key, (outcome, rows, error))?; + } + Ok(()) + } + + pub(crate) fn start_check(&self, py: Python<'_>, name: &str) -> PyResult<()> { + self.check_cancelled(py)?; + let key = format!("check:{name}"); + self.worker.bind(py).get().change( + py, + &format!("check-started/{key}"), + RunChange::CheckStarted { + node_key: key.clone(), + }, + )?; + self.checks + .lock() + .map_err(|_| poisoned())? + .insert(key, None); + Ok(()) + } + + /// A completion without our actual-execution start callback is a blocked check, not a failure. + pub(crate) fn completed_check(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { + let name: String = field(result, "node_name")?.extract()?; + let key = format!("check:{name}"); + let started = self + .checks + .lock() + .map_err(|_| poisoned())? + .contains_key(&key); + let outcome = if !started { + NodeOutcome::Skipped + } else if field(result, "passed")?.is_truthy()? { + NodeOutcome::Success + } else if field(result, "warned")?.is_truthy()? { + NodeOutcome::Warn + } else if self + .worker + .bind(result.py()) + .get() + .refresh_cancellation(result.py())? + { + NodeOutcome::Cancelled + } else { + NodeOutcome::Failed + }; + self.finish_check(result.py(), &key, outcome) + } + + fn finish_check(&self, py: Python<'_>, key: &str, outcome: NodeOutcome) -> PyResult<()> { + self.worker.bind(py).get().change( + py, + &format!("check-finished/{key}"), + RunChange::CheckFinished { + node_key: key.into(), + outcome, + }, + )?; + self.checks + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), Some(outcome)); + Ok(()) + } + + /// Never call missing selected nodes successful merely because the executor returned zero. + pub(crate) fn outcome(&self, py: Python<'_>, exit_code: i32) -> PyResult { + let cancelled = self.worker.bind(py).get().refresh_cancellation(py)?; + for key in &self.execution.claim.request.selection { + self.complete( + py, + key, + ( + if cancelled { + NodeOutcome::Cancelled + } else { + NodeOutcome::Skipped + }, + None, + None, + ), + )?; + } + let outcome = { + let completed = self.completed.lock().map_err(|_| poisoned())?; + let good = completed + .values() + .any(|value| matches!(value, NodeOutcome::Success | NodeOutcome::Warn)); + let bad = completed.values().any(|value| { + matches!( + value, + NodeOutcome::Failed | NodeOutcome::Skipped | NodeOutcome::Cancelled + ) + }); + if cancelled { + RunOutcome::Cancelled + } else if bad && good { + RunOutcome::Partial + } else if bad || exit_code != 0 { + RunOutcome::Failed + } else if completed.values().any(|value| *value == NodeOutcome::Warn) { + RunOutcome::Warn + } else { + RunOutcome::Success + } + }; + Ok(outcome) + } + + /// Terminal publication follows executor teardown and the invocation's bounded Recall drain. + pub(crate) fn finish(&self, py: Python<'_>, exit_code: i32) -> PyResult { + let outcome = self.outcome(py, exit_code)?; + let unfinished: Vec = self + .checks + .lock() + .map_err(|_| poisoned())? + .iter() + .filter(|(_, outcome)| outcome.is_none()) + .map(|(key, _)| key.clone()) + .collect(); + for key in unfinished { + self.finish_check( + py, + &key, + if outcome == RunOutcome::Cancelled { + NodeOutcome::Cancelled + } else { + NodeOutcome::Skipped + }, + )?; + } + self.worker.bind(py).get().change( + py, + "worker-finished", + RunChange::Finished { + outcome, + reason: "worker executor completed and closed its connections".into(), + exit_code: Some(i64::from(exit_code)), + }, + )?; + Ok(outcome) + } +} + +fn poisoned() -> pyo3::PyErr { + PyRuntimeError::new_err("worker build outcome lock is unavailable") +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs new file mode 100644 index 0000000000..13fa7aca8c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs @@ -0,0 +1,73 @@ +//! Model catalog data comes directly from the compiler's retained facts and records. + +use std::collections::BTreeMap; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{PyResult, Python}; +use pyo3::types::PyTupleMethods; +use serde_json::{Value, json}; +use sqlbuild_analysis::compiled_project::main::model_facts::model_facts; +use sqlbuild_orchestration::snapshots::models::SnapshotNode; + +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::snapshot::{ + dependency_keys, destination, insert_node, +}; +use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; + +pub(crate) fn model_nodes( + py: Python<'_>, + project: &NativeProjectView, + owner: &NativeCompiledProject, + mut nodes: BTreeMap, +) -> PyResult> { + for model in project.models(py).bind(py).iter() { + let model = model.cast::()?.get(); + let (name, _, _) = model.file_texts(py)?; + let facts = model_facts(owner.facts(), &name) + .ok_or_else(|| PyValueError::new_err("snapshot model has no retained facts"))?; + if facts.config.encoding_error.is_some() { + return Err(PyValueError::new_err( + "model config cannot be represented in snapshot metadata", + )); + } + let mut definition = model.read(py, |input, _| { + let record = input + .compiled + .as_ref() + .ok_or_else(|| PyValueError::new_err("snapshot model has not been assembled"))?; + let columns: Vec = record + .analysis + .inferred + .as_deref() + .unwrap_or_default() + .iter() + .map(|(name, data_type, nullability)| { + json!({"name": name, "type": data_type, "nullability": nullability}) + }) + .collect(); + Ok(json!({ + "name": facts.name, + "path": facts.relative_path, + "config": facts.config.values, + "materialization": facts.config.materialized, + "deps": dependency_keys(&record.deps), + "columns": columns, + })) + })?; + definition["destination"] = model.read_destination(py, |value| Ok(destination(value)))?; + nodes = insert_node( + nodes, + format!("model:{}", facts.name), + SnapshotNode { + kind: "model".into(), + definition, + raw_sql: Some(facts.authored_sql.to_string()), + compiled_sql: Some(facts.query_sql.to_string()), + python_source: None, + }, + )?; + } + Ok(nodes) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_resources.rs new file mode 100644 index 0000000000..213bcc9739 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_resources.rs @@ -0,0 +1,140 @@ +//! Sources, functions and audits are read from their retained native records. + +use std::collections::BTreeMap; + +use pyo3::prelude::{PyAnyMethods, PyResult, Python}; +use serde_json::{Value, json}; +use sqlbuild_orchestration::snapshots::models::SnapshotNode; + +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::snapshot::{ + dependency_keys, destination, insert_node, +}; +use crate::bindings::_helpers::compiled_project::snapshot_boundary::{ + boundary_nodes, columns, metadata, relative_path, +}; +use crate::bindings::_helpers::compiled_project::values::plain_value; +use crate::bindings::_helpers::compiled_views::function_views::NativeCompiledFunction; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; + +pub(crate) fn resource_nodes( + py: Python<'_>, + project: &NativeProjectView, + owner: &NativeCompiledProject, + mut nodes: BTreeMap, +) -> PyResult> { + for source in owner.sources() { + let attached = &source.attached; + nodes = insert_node( + nodes, + format!("source:{}", attached.name), + SnapshotNode { + kind: "source".into(), + definition: json!({ + "name": attached.name, + "path": relative_path(attached.host.source_file.bind(py))?, + "database": attached.database, + "schema": attached.schema, + "table": attached.table, + "expression": attached.expression, + "contract": attached.contract, + "description": attached.description, + "namespace": source.namespace, + "columns": columns(attached.host.columns.bind(py).as_any())?, + "meta": plain_value(attached.host.meta.bind(py))?, + "entry": metadata(attached.host.entry.bind(py), &[ + "loader", "write_strategy", "cursor_column", "unique_key", + "final", "load_batch_size", "freshness", + ])?, + }), + raw_sql: None, + compiled_sql: attached.expression.clone(), + python_source: None, + }, + )?; + } + let function_views = project.functions(py); + for (function, view) in owner + .functions() + .iter() + .zip(function_views.bind(py).try_iter()?) + { + let effective = view? + .cast::()? + .get() + .current_destinations(py)? + .0; + let attached = &function.attached; + let kind = attached.node_type(); + let python = attached.python.as_ref().map(|runtime| { + json!({ + "version": runtime.runtime_version, + "entry_point": runtime.entry_point, + "packages": runtime.packages, + }) + }); + nodes = insert_node( + nodes, + format!("{kind}:{}", attached.name), + SnapshotNode { + kind: kind.into(), + definition: json!({ + "name": attached.name, + "path": attached.relative_path, + "arguments": attached.arguments, + "returns": attached.returns, + "return_columns": attached.return_columns, + "destination": destination(&effective), + "python": python, + "tags": attached.tags, + "description": attached.description, + "deps": dependency_keys(function.deps.as_deref().unwrap_or_default()), + }), + raw_sql: None, + compiled_sql: attached.python.is_none().then(|| attached.body.clone()), + python_source: attached.python.is_some().then(|| attached.body.clone()), + }, + )?; + } + for audit in owner.audits() { + let attached = &audit.attached; + let target: Option = attached.target.as_ref().map( + |target| json!({"kind": target.kind, "name": target.name, "column": target.column}), + ); + nodes = insert_node( + nodes, + format!("audit:{}", attached.compiled_name()), + SnapshotNode { + kind: "audit".into(), + definition: json!({ + "name": attached.compiled_name(), + "definition_name": attached.definition_name, + "measurement": attached.measurement, + "evidence_sql": attached.evidence_sql, + "target": target, + "severity": attached.severity, + "run_scope": attached.run_scope, + "always_run": attached.always_run, + "description": attached.description, + "policy": metadata(attached.host.audit_block.bind(py), &[ + "evaluation_mode", "measurement_contract", + ])?, + "thresholds": plain_value( + attached.host.thresholds.bind(py), + )?, + "minimum_samples": plain_value( + attached.host.minimum_samples.bind(py), + )?, + "evidence_limit": plain_value( + attached.host.evidence_limit.bind(py), + )?, + "deps": dependency_keys(audit.scope_deps.as_deref().unwrap_or_default()), + }), + raw_sql: None, + compiled_sql: Some(attached.sql_body.clone()), + python_source: None, + }, + )?; + } + boundary_nodes(py, project, owner, nodes) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs new file mode 100644 index 0000000000..5c5da15456 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs @@ -0,0 +1,213 @@ +//! A worker interpreter retains the native claim owner; it never opens the control database. + +use std::sync::Mutex; + +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::{PyRef, PyResult, Python}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_orchestration::runs::models::RunChange; +use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; +use sqlbuild_orchestration::worker::main::start_worker::start_worker; +use sqlbuild_orchestration::worker::models::{WorkerClient, WorkerSession}; +use uuid::Uuid; + +use crate::bindings::_helpers::compiled_project::constants::REPORT_ATTEMPTS; +use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; + +struct WorkerOwner { + runtime: tokio::runtime::Runtime, + client: WorkerClient, + session: WorkerSession, +} + +/// Closing this owner stops and joins heartbeat IO but does not claim warehouse work has stopped. +#[pyclass(frozen, module = "sqlbuild._native")] +pub(crate) struct NativeOrchestrationWorker { + owner: Mutex>, +} + +impl NativeOrchestrationWorker { + /// SIGTERM may arrive before the periodic heartbeat observes the durable cancellation fact. + pub(crate) fn refresh_cancellation(&self, py: Python<'_>) -> PyResult { + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + let owner = owner.as_ref().ok_or_else(unavailable)?; + if owner.session.cancel_requested() { + return Ok(true); + } + owner + .runtime + .block_on(owner.client.heartbeat()) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + /// Native execution reads the retained request, never caller-provided selection or settings. + pub(crate) fn execution( + &self, + py: Python<'_>, + ) -> PyResult { + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + Ok(owner + .as_ref() + .ok_or_else(unavailable)? + .session + .execution + .clone()) + }) + } + + /// A bounded retry keeps the exact event identity across an uncertain acknowledgement. + pub(crate) fn change(&self, py: Python<'_>, key: &str, change: RunChange) -> PyResult<()> { + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + let owner = owner.as_ref().ok_or_else(unavailable)?; + let event_id = Uuid::new_v5(&owner.session.execution.claim.claim_id, key.as_bytes()); + owner.runtime.block_on(async { + let mut attempts = 0; + loop { + attempts += 1; + match owner.client.report(event_id, change.clone()).await { + Ok(_) => return Ok(()), + Err(sqlbuild_orchestration::worker::errors::WorkerError::Transport(_)) + if attempts < REPORT_ATTEMPTS => + { + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + } + Err(error) => return Err(PyRuntimeError::new_err(error.to_string())), + } + } + }) + }) + } + + /// Verify this retained baseline before the caller applies run settings or opens a warehouse. + pub(crate) fn start( + snapshot: &NativeOrchestrationSnapshot, + py: Python<'_>, + (server_url, bearer): (&str, &str), + (run_id, claim_id): (String, String), + ) -> PyResult { + let identity = ( + Uuid::parse_str(&run_id).map_err(|error| PyValueError::new_err(error.to_string()))?, + Uuid::parse_str(&claim_id).map_err(|error| PyValueError::new_err(error.to_string()))?, + ); + py.detach(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let connected = runtime.block_on(async { + let client = connect_worker(server_url, bearer, identity)?; + let session = start_worker(&client, &snapshot.input).await?; + Ok::<_, sqlbuild_orchestration::worker::errors::WorkerError>((client, session)) + }); + match connected { + Ok((client, session)) => Ok(NativeOrchestrationWorker { + owner: Mutex::new(Some(WorkerOwner { + runtime, + client, + session, + })), + }), + Err(error) => { + runtime.shutdown_timeout(std::time::Duration::from_secs(1)); + Err(PyRuntimeError::new_err(error.to_string())) + } + } + }) + } +} + +#[pymethods] +impl NativeOrchestrationWorker { + /// The frozen request supplies selection and settings only after baseline verification. + fn execution_json(&self, py: Python<'_>) -> PyResult { + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + let owner = owner.as_ref().ok_or_else(unavailable)?; + serde_json::to_string(&owner.session.execution) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + /// Cancellation remains sticky; a closed worker cannot authorize further work. + pub(crate) fn cancel_requested(&self, py: Python<'_>) -> bool { + py.detach(|| { + self.owner.lock().map_or(true, |owner| { + owner + .as_ref() + .is_none_or(|owner| owner.session.cancel_requested()) + }) + }) + } + + /// Retry an uncertain report with the identical event ID and change, never a new identity. + fn report(&self, py: Python<'_>, event_id: &str, change_json: &str) -> PyResult { + let event_id = + Uuid::parse_str(event_id).map_err(|error| PyValueError::new_err(error.to_string()))?; + let change: RunChange = serde_json::from_str(change_json) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + let owner = owner.as_ref().ok_or_else(unavailable)?; + owner + .runtime + .block_on(owner.client.report(event_id, change)) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + fn __enter__(slf: PyRef<'_, Self>) -> PyResult> { + if slf.owner.lock().map_err(|_| unavailable())?.is_none() { + return Err(unavailable()); + } + Ok(slf) + } + + fn __exit__( + &self, + py: Python<'_>, + _kind: &pyo3::Bound<'_, pyo3::PyAny>, + _value: &pyo3::Bound<'_, pyo3::PyAny>, + _traceback: &pyo3::Bound<'_, pyo3::PyAny>, + ) -> PyResult<()> { + self.close(py) + } + + /// Join the heartbeat without publishing a terminal outcome or releasing physical claims. + fn close(&self, py: Python<'_>) -> PyResult<()> { + py.detach(|| { + let mut owner = self.owner.lock().map_err(|_| unavailable())?; + if let Some(active) = owner.take() { + active.close(); + } + Ok(()) + }) + } +} + +fn unavailable() -> pyo3::PyErr { + PyRuntimeError::new_err("native worker session is closed or unavailable") +} + +impl WorkerOwner { + fn close(self) { + drop(self.session); + drop(self.client); + self.runtime + .shutdown_timeout(std::time::Duration::from_secs(1)); + } +} + +impl Drop for NativeOrchestrationWorker { + /// GC can hold the GIL; heartbeat teardown joins native IO and never needs Python. + fn drop(&mut self) { + if let Ok(owner) = self.owner.get_mut() + && let Some(owner) = owner.take() + { + owner.close(); + } + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs index 90a9f140d2..2a551bbd39 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs @@ -9,6 +9,7 @@ use serde_json::{Map, Number, Value}; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::models::NativeRecord; const DATACLASS_FIELDS: &str = "__dataclass_fields__"; const INSTANCE_DICT: &str = "__dict__"; @@ -49,6 +50,16 @@ pub(crate) fn plain_value(value: &Bound<'_, PyAny>) -> PyResult { if let Ok(mapping) = value.downcast::() { return object_value(mapping); } + if let Ok(record) = value.cast::() { + let record = record.get(); + let mut entries = Vec::with_capacity(record.values.len()); + for (field, item) in record.spec.fields.iter().zip(&record.values) { + if !field.name.starts_with(PRIVATE_PREFIX) { + entries.push((field.name.to_owned(), plain_value(item.bind(value.py()))?)); + } + } + return Ok(sorted_object(entries)); + } object_like_value(value) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs index 441e7d723d..a83881c415 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs @@ -260,6 +260,22 @@ impl NativeCompiledModel { self.read(py, |record, py| read(compiled_of(record)?, py)) } + /// Read the effective destination, including namespace overlays, without a Python projection. + pub(crate) fn read_destination( + &self, + py: Python<'_>, + read: impl FnOnce(&DestinationFact) -> PyResult, + ) -> PyResult { + self.compiled(py, |compiled, _| { + read( + self.overlay + .destination + .as_ref() + .unwrap_or(&compiled.destination), + ) + }) + } + /// This view's typed analysis as the owner's model facts read it, and the model's name. pub(crate) fn analysis_fact(&self, py: Python<'_>) -> PyResult<(String, ModelAnalysisFact)> { self.read(py, |record, _| { @@ -552,16 +568,11 @@ impl NativeCompiledModel { #[getter] pub(crate) fn destination(&self, py: Python<'_>) -> PyResult> { - self.compiled(py, |compiled, py| { + self.read_destination(py, |destination| { self.projections .destination .get_or_try_init(py, || { let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - let destination: &DestinationFact = self - .overlay - .destination - .as_ref() - .unwrap_or(&compiled.destination); relation_location(&classes.compiled, destination) }) .map(|destination| destination.clone_ref(py)) @@ -612,12 +623,7 @@ impl NativeCompiledModel { /// `(database, schema, name)` of the model's destination. #[getter] pub(crate) fn destination_row(&self, py: Python<'_>) -> PyResult { - self.compiled(py, |compiled, _| { - let destination: &DestinationFact = self - .overlay - .destination - .as_ref() - .unwrap_or(&compiled.destination); + self.read_destination(py, |destination| { Ok(( destination.database.clone(), destination.schema.clone(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs index 7be4f6bee8..af49d0e3f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs @@ -436,7 +436,10 @@ impl NativeCompiledFunction { } /// The destination and fingerprint destination, with target defaults once applied. - fn current_destinations(&self, py: Python<'_>) -> PyResult<(DestinationFact, DestinationFact)> { + pub(crate) fn current_destinations( + &self, + py: Python<'_>, + ) -> PyResult<(DestinationFact, DestinationFact)> { if let Some(defaulted) = &self.defaulted { return Ok(defaulted.clone()); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/getters.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/getters.rs index dda648c730..c28a376f1c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/getters.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/getters.rs @@ -635,6 +635,7 @@ fn read_native_project_view<'py>( "sql_lexical_syntax" => this.sql_lexical_syntax(py).into_bound_py_any(py), "compile_cache_dir" => this.compile_cache_dir(py).into_bound_py_any(py), "loader_functions" => this.loader_functions(py).into_bound_py_any(py), + "python_execution_inputs" => this.python_execution_inputs().into_bound_py_any(py), "hook_functions" => this.hook_functions(py).into_bound_py_any(py), "sql_hook_files" => this.sql_hook_files(py).into_bound_py_any(py), "materialization_files" => this.materialization_files(py).into_bound_py_any(py), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs index 90cdf45297..f28724aab8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs @@ -1,5 +1,8 @@ //! A compiled project, read from the native compiled project that owns its models. +use std::collections::BTreeMap; +use std::path::PathBuf; + use pyo3::exceptions::PyTypeError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; @@ -46,6 +49,8 @@ struct BoundaryInputs { compile_cache_dir: Py, /// Discovered loader functions, user callables. loader_functions: Py, + /// Task/asset/check source locations, without retaining additional Python callables. + python_execution_inputs: BTreeMap, /// Discovered Python hooks, user callables. hook_functions: Py, /// Discovered SQL hook files. @@ -123,6 +128,7 @@ impl ProjectFields { sql_lexical_syntax: required(fields, "sql_lexical_syntax")?.unbind(), compile_cache_dir: required(fields, "compile_cache_dir")?.unbind(), loader_functions: required(fields, "loader_functions")?.unbind(), + python_execution_inputs: required(fields, "python_execution_inputs")?.extract()?, hook_functions: required(fields, "hook_functions")?.unbind(), sql_hook_files: required(fields, "sql_hook_files")?.unbind(), materialization_files: required(fields, "materialization_files")?.unbind(), @@ -195,6 +201,10 @@ impl ProjectFields { self.boundary.compile_cache_dir.bind(py), )?; fields.set_item("loader_functions", self.boundary.loader_functions.bind(py))?; + fields.set_item( + "python_execution_inputs", + &self.boundary.python_execution_inputs, + )?; fields.set_item("hook_functions", self.boundary.hook_functions.bind(py))?; fields.set_item("sql_hook_files", self.boundary.sql_hook_files.bind(py))?; fields.set_item( @@ -391,6 +401,11 @@ impl NativeProjectView { self.fields.boundary.loader_functions.clone_ref(py) } + #[getter] + pub(crate) fn python_execution_inputs(&self) -> BTreeMap { + self.fields.boundary.python_execution_inputs.clone() + } + #[getter] pub(crate) fn hook_functions(&self, py: Python<'_>) -> Py { self.fields.boundary.hook_functions.clone_ref(py) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs index b6f857e5f5..2d070f9535 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs @@ -237,7 +237,7 @@ impl NativeCompiledSeed { }) } - fn current_destination(&self, py: Python<'_>) -> PyResult { + pub(crate) fn current_destination(&self, py: Python<'_>) -> PyResult { if let Some(defaulted) = &self.defaulted { return Ok(defaulted.clone()); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs index 34186a05d0..8a268be998 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs @@ -147,6 +147,7 @@ const R_TARGET_CONFIG: RecordSpec = RecordSpec { FieldDefault::NativeCall(RecordId::TargetWarehousesConfig), ), FieldSpec::new("concurrency", FieldDefault::None), + FieldSpec::new("catalog_connection", FieldDefault::None), ], }; const R_LOCAL_TARGET_CONFIG: RecordSpec = RecordSpec { @@ -186,6 +187,7 @@ const R_LOCAL_TARGET_CONFIG: RecordSpec = RecordSpec { FieldDefault::NativeCall(RecordId::TargetWarehousesConfig), ), FieldSpec::new("concurrency", FieldDefault::None), + FieldSpec::new("catalog_connection", FieldDefault::None), ], }; const R_SETTINGS_CONFIG: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs index 896dcdd112..6a4176ae83 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs @@ -287,7 +287,25 @@ pub(crate) fn effective_config_connection<'py>( local_config: &Bound<'py, PyAny>, target_config: &Bound<'py, PyAny>, ) -> PyResult>> { - let target: Option<(Option>, Pairs<'py>)> = if target_config.is_none() { + effective_connection_fields(py, (project_config, local_config), target_config, false) +} + +/// Catalog selection uses the same named-layer resolver, with no writer layers. +pub(crate) fn effective_connection_fields<'py>( + py: Python<'py>, + (project_config, local_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + target_config: &Bound<'py, PyAny>, + catalog: bool, +) -> PyResult>> { + let catalog_name = if catalog && !target_config.is_none() { + optional_key(&native_field(target_config, "catalog_connection")?)? + } else { + None + }; + let independent = catalog_name.is_some(); + let target: Option<(Option>, Pairs<'py>)> = if independent { + Some((catalog_name, Vec::new())) + } else if target_config.is_none() { None } else { Some(( @@ -296,15 +314,33 @@ pub(crate) fn effective_config_connection<'py>( )) }; let layers: ConnectionLayers, Value<'py>> = ConnectionLayers { - project: pairs(&native_field(project_config, "connection")?)?, + project: if independent { + Vec::new() + } else { + pairs(&native_field(project_config, "connection")?)? + }, target, project_named: named(&native_field(project_config, "connections")?)?, local_named: named(&native_field(local_config, "connections")?)?, - local: pairs(&native_field(local_config, "connection")?)?, + local: if independent { + Vec::new() + } else { + pairs(&native_field(local_config, "connection")?)? + }, }; Ok(match effective_connection(layers) { Ok(connection) => (Some(dictionary(py, connection)?), None), - Err(error) => (None, Some(message(py, &error)?)), + Err(error) => { + let message = message(py, &error)?; + ( + None, + Some(if independent { + PyString::new(py, &format!("catalog_connection: {message}")) + } else { + message + }), + ) + } }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs index 4b3fdea078..d140a8017d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs @@ -42,7 +42,7 @@ pub(crate) fn connection_config<'py>( ) -> PyResult> { let py = environment.py(); let discovered = native_field(environment, "discovered_inputs")?; - let raw = build_effective_connection_config(&discovered.clone(), None, None)?; + let raw = build_effective_connection_config(&discovered.clone(), None, None, false)?; let raw: Bound<'_, PyDict> = raw.cast_into()?; let project_dir = native_field(environment, "project_dir")?; let adapter_name = native_field(environment, "adapter_name")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index ae20cf9408..0d8524bc71 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -28,7 +28,9 @@ use crate::bindings::_helpers::boundary::python_calls::{ }; use crate::bindings::_helpers::cli::failure_cause::{FailureCause, report_failure_cause}; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; -use crate::bindings::_helpers::command_runtime::connection::project_connection_config; +use crate::bindings::_helpers::command_runtime::connection::{ + local_database_files, project_connection_config, +}; use crate::bindings::_helpers::command_runtime::runtime_errors::{ expected_resolution_error, resolution_message, }; @@ -39,6 +41,7 @@ use crate::bindings::_helpers::command_runtime::warehouses::{ use crate::bindings::_helpers::compile_attachment::target::build_effective_target_namespace; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::native_host::host::redirect_stdout_to_stderr; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::construct::record_class; use crate::bindings::_helpers::records::fields::field as native_field; @@ -290,6 +293,7 @@ fn build_report( &context.discovered_inputs, (&context.project_dir, &context.selected_target), (&py.None().into_bound(py), true), + false, ) })?; let connection_config: Bound<'_, PyDict> = @@ -357,6 +361,7 @@ fn build_report( ConnectionCheck::Skipped }; Ok(DebugReport { + local_files: local_database_files(&facts.adapter_name, &connection_config)?, runtime: debug_runtime_lines(&runtime(py)?), configuration: debug_configuration_lines(&facts, &warehouses), providers: debug_provider_lines(&providers(&context.discovered_inputs)?), @@ -381,15 +386,23 @@ pub(crate) fn run_debug_command( } else { project_dir.extract()? }; - let report: DebugReport = build_report( - py, - &project_dir, - ( - arguments.value("selected_target")?, - !arguments.flag("no_connection")?, - ), - )?; - let text: String = if arguments.flag("json_output")? { + let json_output = arguments.flag("json_output")?; + let build = || { + build_report( + py, + &project_dir, + ( + arguments.value("selected_target")?, + !arguments.flag("no_connection")?, + ), + ) + }; + let report: DebugReport = if json_output { + within(&redirect_stdout_to_stderr(py)?, |_| build())? + } else { + build()? + }; + let text: String = if json_output { format_debug_json(&report) + "\n" } else { let use_color: bool = !arguments.flag("no_color")? && stdout_supports_color(py)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index 7e4ed5ab92..511a468e3d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -3,13 +3,10 @@ use fensu_policy::lifecycle::constants::CUSTOM_HOST_PROTOCOL_VERSION; use sqlbuild_analysis::contracts::constants::CONTRACT_ENFORCED; use sqlbuild_analysis::fingerprint_storage::constants::{ - COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_METADATA_JSON_B64, COLUMN_NODE_NAME, - COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_SCHEMA_FINGERPRINT, COLUMN_TARGET_DATABASE, - COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, COLUMN_VERSION_HASH, - NODE_TYPE_MODEL, NODE_TYPE_SEED, -}; -use sqlbuild_analysis::planner_changes::constants::{ - AUDIT_GATE_METADATA_KEY, MIGRATION_FINGERPRINT_METADATA_KEY, + COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_EVENT_ID, COLUMN_METADATA_JSON_B64, + COLUMN_NODE_NAME, COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_SCHEMA_FINGERPRINT, + COLUMN_TARGET_DATABASE, COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, + COLUMN_VERSION_HASH, NODE_TYPE_MODEL, NODE_TYPE_SEED, }; use sqlbuild_analysis::snowflake_cost::constants::QUERY_TAG_APP; use sqlbuild_cli::compute_logs::constants::{ @@ -29,6 +26,9 @@ use sqlbuild_core::constants::{ use sqlbuild_core::diagnostic_routing::constants::{ SQL_DIGEST_FIELD, SQL_PRIVATE_RECORD_FIELD, SQL_TEXT_FIELD, }; +use sqlbuild_core::fingerprint_identity::constants::{ + AUDIT_GATE_METADATA_KEY, MIGRATION_FINGERPRINT_METADATA_KEY, +}; use sqlbuild_host::entries::constants::CUSTOM_HOST_LAUNCH_MODULE; use sqlbuild_rules::constants::MIN_CUSTOM_RULE_TEST_CASES_KEY; use sqlbuild_rules::custom_host_pool::constants::CUSTOM_RULE_PROJECT_SUBJECT; @@ -99,6 +99,7 @@ pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ ("COLUMN_DEFINITION_B64", COLUMN_DEFINITION_B64), ("COLUMN_METADATA_JSON_B64", COLUMN_METADATA_JSON_B64), ("COLUMN_TIMESTAMP", COLUMN_TIMESTAMP), + ("COLUMN_EVENT_ID", COLUMN_EVENT_ID), ("NODE_TYPE_MODEL", NODE_TYPE_MODEL), ("NODE_TYPE_SEED", NODE_TYPE_SEED), ("AUDIT_GATE_METADATA_KEY", AUDIT_GATE_METADATA_KEY), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs index e977d6a155..3347752271 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs @@ -130,7 +130,7 @@ fn borrowed(identities: &[(String, String)]) -> Vec> { .collect() } -fn row_fingerprint<'py>( +pub(crate) fn row_fingerprint<'py>( fingerprint_class: &Bound<'py, PyAny>, from_iso: &Bound<'py, PyAny>, row: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs index 3a92672f90..e78d3b228f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs @@ -24,6 +24,7 @@ use crate::bindings::_helpers::fingerprints::storage_sql::{ use crate::bindings::_helpers::records::construct::new_record; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::state_column_evolution::ensure_event_id_column; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; @@ -97,8 +98,12 @@ fn create_sql( )) } -/// The table DDL, adapter-rendered when the adapter overrides it, then any index DDL. -fn setup_sqls(location: &Location<'_, '_>, writer: &Writer<'_, '_>) -> PyResult> { +/// Create the table/indexes without an extra catalog read for a current or new table. +fn setup<'py>( + location: &Location<'_, 'py>, + writer: &Writer<'_, 'py>, + describe: &dyn Fn(&PyErr) -> PyResult>, +) -> PyResult<()> { let py: Python<'_> = writer.execute.py(); let mut sqls: Vec = vec![match writer.render_create_table_sql { Some(render) => render @@ -118,7 +123,16 @@ fn setup_sqls(location: &Location<'_, '_>, writer: &Writer<'_, '_>) -> PyResult< sqls.push(sql?.extract()?); } } - Ok(sqls) + retry_write( + py, + || { + for sql in &sqls { + execute_sql(writer, sql)?; + } + Ok(()) + }, + describe, + ) } fn execute_sql<'py>(writer: &Writer<'_, 'py>, sql: &str) -> PyResult> { @@ -128,6 +142,37 @@ fn execute_sql<'py>(writer: &Writer<'_, 'py>, sql: &str) -> PyResult( + location: &Location<'_, 'py>, + writer: &Writer<'_, 'py>, + request: &Request<'_, 'py>, + sql: &str, +) -> PyResult<()> { + let error = match execute_sql(writer, sql) { + Ok(_) => return Ok(()), + Err(error) => error, + }; + let py = writer.execute.py(); + if !error.is_instance_of::(py) || present(request, "read_columns")?.is_none() { + return Err(error); + } + ensure_event_id_column( + (writer.connection, writer.execute), + ( + &location.database.into_pyobject(py)?.into_any(), + &location.schema.into_pyobject(py)?.into_any(), + ), + (FINGERPRINT_TABLE_NAME, &qualified_table_name(location)?), + ( + &request.get("read_columns")?, + &request.get("render_add_columns")?, + writer.render_framework_type, + ), + )?; + execute_sql(writer, sql).map(|_| ()) +} + /// Log one failed attempt with `details` after the attempt counts, then back off. fn retry_pause( py: Python<'_>, @@ -156,22 +201,14 @@ fn error_details<'py>(py: Python<'py>, error: &PyErr) -> PyResult( - writer: &Writer<'_, 'py>, - sqls: &[String], +/// Retry a complete warehouse operation after transient exceptions within the write budget. +fn retry_write<'py>( + py: Python<'py>, + write: impl Fn() -> PyResult<()>, describe: &dyn Fn(&PyErr) -> PyResult>, ) -> PyResult<()> { - let py: Python<'py> = writer.execute.py(); for attempt in 0..WRITE_ATTEMPTS { - let mut failure: Option = None; - for sql in sqls { - if let Err(error) = execute_sql(writer, sql) { - failure = Some(error); - break; - } - } - let Some(error) = failure else { + let Err(error) = write() else { return Ok(()); }; if attempt + 1 == WRITE_ATTEMPTS || !error.is_instance_of::(py) { @@ -376,16 +413,20 @@ fn write_fingerprint<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult return Err(error), } } - let mut sqls: Vec = setup_sqls(&location, &writer)?; - sqls.push(insert); - execute_with_retry(&writer, &sqls, &|error| { + let describe = |error: &PyErr| { let mut details: Vec> = vec![ native_field(&fingerprint, "node_type")?, native_field(&fingerprint, "node_name")?, ]; details.extend(error_details(py, error)?); Ok((SINGLE_RETRY_MESSAGE, details)) - })?; + }; + setup(&location, &writer, &describe)?; + retry_write( + py, + || execute_insert(&location, &writer, &request, &insert), + &describe, + )?; Ok(true) } @@ -420,8 +461,7 @@ fn write_fingerprints<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult<()> let on_progress: Option> = present(&request, "on_progress")?; let parts: WriteParts<'py> = WriteParts::read(&request)?; let (location, writer) = (parts.location(), parts.writer()); - let setup: Vec = setup_sqls(&location, &writer)?; - execute_with_retry(&writer, &setup, &|error| { + setup(&location, &writer, &|error| { batch_warning(&fingerprints, error) })?; let batches = fingerprint_insert_batches( @@ -434,9 +474,11 @@ fn write_fingerprints<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult<()> let mut completed: usize = 0; for (batch, insert_sql) in batches { let members: Vec> = batch.iter().collect(); - execute_with_retry(&writer, &[insert_sql], &|error| { - batch_warning(&members, error) - })?; + retry_write( + py, + || execute_insert(&location, &writer, &request, &insert_sql), + &|error| batch_warning(&members, error), + )?; completed += members.len(); if let Some(progress) = &on_progress { let keywords: Bound<'py, PyDict> = PyDict::new(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs index ab91f568ae..598be15e62 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs @@ -177,6 +177,11 @@ fn renderers<'py>(adapter: &Bound<'py, PyAny>) -> PyResult> { "render_create_index_sqls", native_field(adapter, "render_create_source_freshness_index_sqls")?, ), + ("read_columns", native_field(adapter, "get_columns")?), + ( + "render_add_columns", + native_field(adapter, "render_add_columns")?, + ), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs index 292bada80f..4a004cbbda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs @@ -4,7 +4,10 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; -use sqlbuild_state::freshness::constants::{REQUIRED_STATE_COLUMNS, SOURCE_FRESHNESS_COLUMN_TYPES}; +use sqlbuild_state::freshness::constants::{ + REQUIRED_STATE_COLUMNS, SOURCE_FRESHNESS_COLUMN_TYPES, SOURCE_FRESHNESS_EVENT_ID, + STRING_STATE_VALUE_TYPE, +}; use sqlbuild_state::freshness::main::read_latest_sql::read_latest_sql; use sqlbuild_state::freshness::main::relevant_sources_sql::relevant_sources_sql; @@ -13,6 +16,7 @@ use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; use crate::bindings::_helpers::freshness::support::{ INPUT_ERROR, exception_text, freshness_error, optional_str, presumed_utc_datetime, text, }; +use crate::bindings::_helpers::native_host::state_column_evolution::ensure_event_id_column; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; @@ -82,12 +86,20 @@ fn create_table_sql<'py>( ) -> PyResult> { let py: Python<'py> = database.py(); let qualified_name: String = qualified_table_name(render_qualified_name, (database, schema))?; - let columns: Bound<'py, PyAny> = - PyTuple::new(py, SOURCE_FRESHNESS_COLUMN_TYPES.map(|(column, _)| column))?.into_any(); + let columns: Bound<'py, PyAny> = PyTuple::new( + py, + SOURCE_FRESHNESS_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .chain([SOURCE_FRESHNESS_EVENT_ID]) + .collect::>(), + )? + .into_any(); let column_types: Bound<'py, PyDict> = PyDict::new(py); for (column, declared) in SOURCE_FRESHNESS_COLUMN_TYPES { column_types.set_item(column, declared)?; } + column_types.set_item(SOURCE_FRESHNESS_EVENT_ID, STRING_STATE_VALUE_TYPE)?; let column_types: Bound<'py, PyAny> = column_types.into_any(); let required: Bound<'py, PyAny> = PyFrozenSet::new(py, REQUIRED_STATE_COLUMNS)?.into_any(); let sql: String = state_table_create_sql( @@ -180,7 +192,7 @@ freshness table to regenerate source freshness state." ) } -fn row_record<'py>(row: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn row_record<'py>(row: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = row.py(); let raw_observed_at: Bound<'py, PyAny> = row.get_item(9)?; let observed_at: Bound<'py, PyAny> = if date_time_typed(&raw_observed_at)? { @@ -306,6 +318,20 @@ pub(crate) fn write_source_freshness_records<'py>( call_with(&render_create, &location())? }; execute_sql((execute, connection), create_sql)?; + let qualified = qualified_table_name( + &native_field(renderers, "render_qualified_name")?, + (database, schema), + )?; + ensure_event_id_column( + (connection, execute), + (database, schema), + (SOURCE_FRESHNESS_TABLE_NAME, &qualified), + ( + &native_field(renderers, "read_columns")?, + &native_field(renderers, "render_add_columns")?, + &native_field(renderers, "render_framework_type")?, + ), + )?; let render_indexes: Bound<'py, PyAny> = native_field(renderers, "render_create_index_sqls")?; if !render_indexes.is_none() { for index_sql in call_with(&render_indexes, &location())?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 9ee1fcb3d2..b20cc9d2a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -260,6 +260,7 @@ pub(crate) fn loader_defaults<'py>( &discovered.clone(), Some(selected_target.clone()).filter(|value| !value.is_none()), Some(cli_vars.clone()).filter(|value| !value.is_none()), + false, )? .downcast_into()?; let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs new file mode 100644 index 0000000000..dfd9e0eb1c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -0,0 +1,321 @@ +//! Compile in the location interpreter and publish data through its scoped HTTP credential. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; +use std::process::{Command, Stdio}; + +use chrono::{DateTime, Utc}; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::git_revisions::main::head_commit::head_commit; +use sqlbuild_orchestration::declarations::models::{ + Schedule, TriggerDeclaration, TriggerDefinition, +}; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; +use sqlbuild_orchestration::warehouse_metadata::models::WarehouseStateLocation; + +use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::pipeline::compiled_phase; +use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::location_command::constants::{EXECUTE_ACTION, PUBLISH_ACTION}; +use crate::bindings::_helpers::location_command::runtime::run_runtime; +use crate::bindings::_helpers::location_command::snapshot::build_orchestration_snapshot; +use crate::bindings::_helpers::location_command::worker::execute_worker; +use crate::bindings::_helpers::native_host::host::write_console; +use crate::bindings::_helpers::query_command::project::{ + ProjectAdapter, ProjectInputs, project_adapter, +}; +use crate::bindings::_helpers::recall::session::InvocationRecall; +use crate::bindings::_helpers::records::construct::new_record_with; +use crate::bindings::_helpers::records::fields::field; +use crate::bindings::_helpers::records::ids::RecordId; + +fn status(py: Python<'_>, message: &str) -> PyResult<()> { + write_console(py, "stderr", &format!("{message}\n")) +} + +/// Git is optional in deployed images; absent metadata is not fabricated from the last publication. +fn metadata( + py: Python<'_>, + arguments: &KeywordArguments<'_>, + directory: &Path, +) -> PyResult { + let location: String = arguments.value("location_id")?.extract()?; + let location_id = location + .parse() + .map_err(|_| PyValueError::new_err("location ID must be a UUID"))?; + let target: String = arguments.value("selected_target")?.extract()?; + let action = arguments.value("action")?.extract::>()?; + let generation = arguments + .value("generation")? + .extract::>()? + .or_else(|| (action.as_deref() == Some(EXECUTE_ACTION)).then_some(1)) + .ok_or_else(|| PyValueError::new_err("publish requires --generation"))?; + if target.trim().is_empty() || generation < 1 { + return Err(PyValueError::new_err( + "target must not be blank and generation must be positive", + )); + } + let (git_revision, git_dirty, git_author) = py.detach(|| -> PyResult<_> { + let revision = head_commit(directory); + let mut dirty = false; + let mut author = None; + if revision.is_some() { + for (is_dirty, options) in [ + (false, vec!["log", "-1", "--format=%an", "HEAD"]), + ( + true, + vec!["status", "--porcelain", "--untracked-files=normal"], + ), + ] { + let output = Command::new("git") + .current_dir(directory) + .args(options) + .stdin(Stdio::null()) + .output()?; + if !output.status.success() { + return Err(PyRuntimeError::new_err( + "could not read location Git metadata", + )); + } + let text = String::from_utf8_lossy(&output.stdout); + if is_dirty { + dirty = !text.trim().is_empty(); + } else { + author = Some(text.trim().to_owned()); + } + } + } + Ok((revision, dirty, author)) + })?; + Ok(SnapshotInput { + location_id, + target, + generation, + git_revision, + git_dirty, + git_author, + image_id: arguments.value("image_id")?.extract()?, + environment_hash: String::new(), + sqb_version: String::new(), + nodes: Default::default(), + documents: Default::default(), + }) +} + +fn compile_location<'py>( + py: Python<'py>, + arguments: &KeywordArguments<'py>, + input: &SnapshotInput, +) -> PyResult<(NativeOrchestrationSnapshot, Bound<'py, PyAny>)> { + status(py, "Discovering location project...")?; + let ProjectAdapter { + project_dir, + discovered, + adapter, + } = project_adapter( + py, + Some(arguments.value("project_dir")?), + ProjectInputs::Discovered, + )?; + status(py, "Location project discovered. Compiling snapshot...")?; + let options = new_record_with( + py, + RecordId::CompilePipelineOptions, + &[("selected_target", arguments.value("selected_target")?)], + )?; + let compiled = compiled_phase( + (&discovered, &adapter), + &options, + &py.None().into_bound(py), + None, + )?; + let project = field(&compiled, "project")?; + let view = project.cast::()?.borrow(); + let snapshot = build_orchestration_snapshot( + view, + &serde_json::to_string(input).map_err(|error| PyValueError::new_err(error.to_string()))?, + project_dir.extract()?, + arguments + .value("action")? + .extract::>()? + .is_some_and(|action| action == EXECUTE_ACTION) + .then(Vec::new), + )?; + status(py, "Location snapshot compiled.")?; + Ok((snapshot, discovered)) +} + +pub(super) fn location_arguments<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + KeywordArguments::read( + py, + "run_location_command", + options, + ( + &[ + "project_dir", + "location_id", + "selected_target", + "generation", + "server_url", + ], + &["image_id", "action", "run_id", "claim_id"], + ), + ) +} + +pub(super) fn publish_location( + py: Python<'_>, + arguments: &KeywordArguments<'_>, +) -> PyResult<( + SnapshotRecord, + Option>, + Vec, +)> { + if arguments + .value("action")? + .extract::>()? + .is_some_and(|action| action != PUBLISH_ACTION) + { + return Err(PyValueError::new_err( + "location serve must run through the native sqb executable", + )); + } + let bearer = std::env::var("SQLBUILD_LOCATION_TOKEN").map_err(|_| { + PyValueError::new_err("SQLBUILD_LOCATION_TOKEN must contain a UTF-8 location credential") + })?; + if bearer.trim().is_empty() { + return Err(PyValueError::new_err( + "SQLBUILD_LOCATION_TOKEN must not be blank", + )); + } + let directory = arguments + .value("project_dir")? + .extract::()?; + let input = metadata(py, arguments, &directory)?; + let (snapshot, _) = compile_location(py, arguments, &input)?; + let locations: BTreeSet<_> = snapshot + .input + .nodes + .values() + .filter_map(|node| { + let destination = &node.definition["destination"]; + Some(WarehouseStateLocation { + database: destination["database"].as_str().map(str::to_owned), + schema: destination["schema"].as_str()?.to_owned(), + }) + }) + .collect(); + let triggers: BTreeMap = serde_json::from_value( + snapshot + .input + .documents + .get("triggers") + .cloned() + .unwrap_or_else(|| serde_json::json!({})), + ) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + let refresh_at = triggers + .values() + .filter_map(|trigger| match trigger { + TriggerDeclaration::Schedule(TriggerDefinition { + schedule: Schedule::Calendar { refresh_at, .. }, + .. + }) => Some(*refresh_at), + TriggerDeclaration::Schedule(_) | TriggerDeclaration::Sensor(_) => None, + }) + .min(); + status(py, "Publishing location snapshot...")?; + let record = snapshot.publication( + py, + &arguments.value("server_url")?.extract::()?, + &bearer, + )?.map_err(|error| { + let message = if matches!(error, sqlbuild_orchestration::api::errors::ClientError::Transport(_)) { + format!("Publication outcome is unknown; generation {} may already be published. Check the current snapshot before retrying: {error}", input.generation) + } else { + error.to_string() + }; + PyValueError::new_err(message) + })?; + status(py, "Location snapshot published.")?; + Ok((record, refresh_at, locations.into_iter().collect())) +} + +/// One-shot publication and the supervisor's private stdio runtime share the same compiler path. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn run_location_command<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + run_location(py, options, None) +} + +/// CLI dispatch retains the verified worker through invocation history delivery. +pub(crate) fn run_location<'py>( + py: Python<'py>, + options: Option>, + recall: Option<&InvocationRecall>, +) -> PyResult { + let arguments = location_arguments(py, options.clone())?; + if arguments + .value("action")? + .extract::>()? + .as_deref() + == Some("execute") + { + let directory = arguments + .value("project_dir")? + .extract::()?; + let input = metadata(py, &arguments, &directory)?; + let bearer = std::env::var("SQLBUILD_RUN_TOKEN") + .map_err(|_| PyValueError::new_err("execute requires SQLBUILD_RUN_TOKEN"))?; + let identity = ( + arguments.value("run_id")?.extract::()?, + arguments.value("claim_id")?.extract::()?, + ); + let (snapshot, discovered) = compile_location(py, &arguments, &input)?; + return execute_worker( + py, + (&snapshot, discovered), + &arguments, + (&bearer, identity, recall), + ); + } + if arguments + .value("action")? + .extract::>()? + .as_deref() + == Some("runtime") + { + return run_runtime(py, options.unwrap_or_else(|| PyDict::new(py))); + } + let publication = (|| { + let (record, _, _) = publish_location(py, &arguments)?; + let output = serde_json::to_string(&record) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + write_console(py, "stdout", &format!("{output}\n")).map_err(|error| { + PyValueError::new_err(format!( + "Snapshot {} was published, but result delivery failed: {error}", + record.snapshot_id + )) + })?; + Ok(0) + })(); + if publication.is_err() { + status(py, "Location command failed.")?; + } + publication +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_orchestration_snapshot, module)?)?; + module.add_function(wrap_pyfunction!(run_location_command, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs new file mode 100644 index 0000000000..ffee78c649 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -0,0 +1,5 @@ +/// Location actions accepted by the interpreter-side one-shot publisher. +pub(crate) const PUBLISH_ACTION: &str = "publish"; + +/// Workers verify the pinned nodes and environment without reevaluating declarations. +pub(crate) const EXECUTE_ACTION: &str = "execute"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs new file mode 100644 index 0000000000..79131a3bb0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod command; +mod constants; +mod runtime; +mod snapshot; +mod worker; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs new file mode 100644 index 0000000000..c1dd2726d4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs @@ -0,0 +1,111 @@ +//! A location keeps its interpreter warm; only the supervisor writes generation requests. + +use std::io::{self, Write}; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_orchestration::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; +use sqlbuild_orchestration::warehouse_metadata::models::{ + WarehouseHistoryKind, WarehouseHistoryRead, +}; + +use crate::bindings::_helpers::boundary::python_calls::within; +use crate::bindings::_helpers::location_command::command::{location_arguments, publish_location}; +use crate::bindings::_helpers::native_host::host::{redirect_stdout_to_stderr, write_console}; +use crate::bindings::_helpers::native_host::orchestration_sensors::evaluate_sensor; +use crate::bindings::_helpers::native_host::warehouse_metadata::read_history; + +pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResult { + options.set_item("action", "publish")?; + let redirect = redirect_stdout_to_stderr(py)?; + within(&redirect, |_| { + loop { + let line = py.detach(|| -> io::Result> { + let mut line = String::new(); + Ok((io::stdin().read_line(&mut line)? != 0).then_some(line)) + })?; + let Some(line) = line else { + break; + }; + let request: serde_json::Value = serde_json::from_str(&line).map_err(|_| { + PyValueError::new_err( + "runtime expects a JSON generation or sensor evaluation per line", + ) + })?; + let output = if let Some(generation) = request.as_i64() { + options.set_item("generation", generation)?; + let arguments = location_arguments(py, Some(options.clone()))?; + let record = match publish_location(py, &arguments) { + Ok(record) => Some(record), + Err(error) => { + write_console( + py, + "stderr", + &format!("Location compilation/publication failed: {error}\n"), + )?; + None + } + }; + serde_json::to_string(&record) + } else if WarehouseHistoryKind::ALL + .iter() + .any(|kind| request.get(kind.as_str()).is_some()) + { + let metadata: WarehouseHistoryRead = + serde_json::from_value(request).map_err(|_| { + PyValueError::new_err("runtime received an invalid metadata read") + })?; + let arguments = location_arguments(py, Some(options.clone()))?; + let page = match read_history(py, &arguments, &metadata) { + Ok(page) => Some(page), + Err(_) => { + write_console( + py, + "stderr", + "Warehouse metadata read failed. No page delivered.\n", + )?; + None + } + }; + serde_json::to_string(&page) + } else { + let evaluation: SensorEvaluationRequest = + serde_json::from_value(request).map_err(|_| { + PyValueError::new_err("runtime received an invalid sensor evaluation") + })?; + let arguments = location_arguments(py, Some(options.clone()))?; + if arguments.value("location_id")?.extract::()? + != evaluation.location_id.to_string() + || arguments.value("selected_target")?.extract::()? != evaluation.target + { + return Err(PyValueError::new_err( + "sensor work belongs to another location or target", + )); + } + let directory = arguments + .value("project_dir")? + .extract::()?; + let outcome = match evaluate_sensor(py, &directory, &evaluation) { + Ok(outcome) => outcome, + Err(_) => { + write_console( + py, + "stderr", + "Custom sensor evaluation failed. No cursor or run requests committed.\n", + )?; + CustomSensorOutcome::Failed + } + }; + serde_json::to_string(&outcome) + }; + let output = output.map_err(|error| PyValueError::new_err(error.to_string()))?; + py.detach(|| -> io::Result<()> { + let mut stdout = io::stdout().lock(); + writeln!(stdout, "{output}")?; + stdout.flush() + })?; + } + Ok(0) + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs new file mode 100644 index 0000000000..f975aa24ed --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -0,0 +1,113 @@ +//! Location publication assembles catalog data and planner-owned local identities. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{IntoPyObject, pyfunction}; +use serde_json::json; +use sqlbuild_core::models::DiagnosticSeverity; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; + +use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; +use crate::bindings::_helpers::compiled_project::snapshot_declarations::{ + compile_declarations, discover_declarations, +}; +use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; +use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::native_host::orchestration_declarations::collect_declarations; +use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; +use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; +use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; +use crate::bindings::_helpers::records::fields::field; + +/// Compile catalog nodes and identities in the location, never in the server. +#[pyfunction] +#[pyo3(signature = (project, metadata_json, project_dir, *, declarations=None))] +pub(crate) fn build_orchestration_snapshot( + project: PyRef<'_, NativeProjectView>, + metadata_json: &str, + project_dir: PathBuf, + declarations: Option>>, +) -> PyResult { + let py = project.py(); + let mut input: SnapshotInput = serde_json::from_str(metadata_json) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + if !input.nodes.is_empty() { + return Err(PyValueError::new_err( + "snapshot nodes must come from the compiled project", + )); + } + if project + .effective_target_name() + .is_some_and(|target| target != input.target) + { + return Err(PyValueError::new_err( + "snapshot target differs from the compiled project", + )); + } + for diagnostic in project.diagnostics(py).bind(py).try_iter()? { + let severity: String = field(&field(&diagnostic?, "severity")?, "value")?.extract()?; + if severity == DiagnosticSeverity::Error.as_str() { + return Err(PyValueError::new_err( + "cannot publish a project with compiler errors", + )); + } + } + input.sqb_version = env!("CARGO_PKG_VERSION").into(); + input.environment_hash = environment_hash(py, &project_dir)?; + let owner = project.owner().bind(py).borrow(); + input.nodes = model_nodes(py, &project, &owner, input.nodes)?; + input.nodes = resource_nodes(py, &project, &owner, input.nodes)?; + drop(owner); + let declarations = match declarations { + Some(declarations) => declarations, + None => discover_declarations(py, &project_dir)?, + }; + if !declarations.is_empty() { + let authored = collect_declarations(py, declarations)?; + let graph = project.lineage_graph(py)?; + input = compile_declarations( + py, + input, + (&project_dir, graph.bind(py).get().graph()), + authored, + )?; + } + input.nodes = identity_nodes(project.into_pyobject(py)?.as_any(), input.nodes)?; + Ok(NativeOrchestrationSnapshot { input }) +} + +fn identity_nodes( + project: &Bound<'_, PyAny>, + mut nodes: BTreeMap, +) -> PyResult> { + let py = project.py(); + let options = PyDict::new(py); + options.set_item("project", project)?; + let scope = build_planner_scope(py, Some(options))?; + let identities = direct_model_version_identities( + (&field(project, "functions")?, &scope), + field(project, "sql_analysis_dialect")?.extract()?, + ( + &field(project, "seeds")?, + &py.None().into_bound(py), + &field(project, "hook_functions")?, + ), + )?; + for node in nodes.values_mut() { + let hashes = match node.kind.as_str() { + "model" => "model_local_hashes", + "udf" | "table_fn" => "function_local_hashes", + "seed" => "seed_version_hashes", + _ => continue, + }; + let name = node.definition["name"].as_str().unwrap_or_default(); + let hash: String = field(&identities, hashes)?.get_item(name)?.extract()?; + node.definition["planner_identity"] = json!({"local_hash": hash}); + } + Ok(nodes) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs new file mode 100644 index 0000000000..b7876f07f4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -0,0 +1,177 @@ +//! Execute only the API's frozen request after compiling and verifying the unmodified baseline. + +use std::sync::{Arc, mpsc}; +use std::time::Duration; + +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; +use sqlbuild_cli::dispatch::main::command_dispatch::command_dispatch; +use sqlbuild_cli::dispatch::models::Dispatch; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_orchestration::runs::models::{RunOutcome, RunWindow}; + +use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with, within}; +use crate::bindings::_helpers::build_command::command::run_build; +use crate::bindings::_helpers::cli::argv::parse_cli_invocation; +use crate::bindings::_helpers::cli::dispatch::{Invocation, command_facts, warehouse_scope}; +use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; +use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; +use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; +use crate::bindings::_helpers::native_host::host::{entry, write_console}; +use crate::bindings::_helpers::recall::session::InvocationRecall; +use crate::bindings::_helpers::records::fields::{field, replaced}; + +fn build_request<'py>( + py: Python<'py>, + discovered: Bound<'py, PyAny>, + arguments: &KeywordArguments<'py>, + worker: &Bound<'py, NativeOrchestrationWorker>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let execution = worker.get().execution(py)?; + let frozen = execution.claim.request; + let mut settings = frozen.settings; + if let Some(window) = frozen.window { + for key in [ + "start_cursor", + "end_cursor", + "start_cursor_ts", + "end_cursor_ts", + "start_cursor_int", + "end_cursor_int", + ] { + settings.remove(key); + } + let (kind, start, end) = match window { + RunWindow::Timestamp { start, end } => ("ts", start.to_rfc3339(), end.to_rfc3339()), + RunWindow::Integer { start, end } => ("int", start.to_string(), end.to_string()), + }; + settings.insert(format!("start_cursor_{kind}"), serde_json::json!(start)); + settings.insert(format!("end_cursor_{kind}"), serde_json::json!(end)); + } + let mut argv: Vec> = std::iter::once("build".to_owned()) + .chain(job_settings_arguments(&settings).map_err(PyValueError::new_err)?) + .map(|argument| PyString::new(py, &argument)) + .collect(); + argv.extend([ + PyString::new(py, "--project-dir"), + arguments.value("project_dir")?.str()?, + PyString::new(py, "--target"), + PyString::new(py, &frozen.target), + PyString::new(py, "--no-color"), + PyString::new(py, "--json"), + ]); + for key in &frozen.selection { + if execution.skipped_nodes.contains_key(key) { + continue; + } + let (_, name) = key + .split_once(':') + .ok_or_else(|| PyValueError::new_err("frozen run contains an invalid node key"))?; + argv.extend([PyString::new(py, "--select"), PyString::new(py, name)]); + } + let parsed = parse_cli_invocation(py, argv, false)?; + let args = field(&parsed, "args")?; + if args.is_none() { + return Err(PyValueError::new_err("invalid frozen build settings")); + } + let Dispatch::Run(route, None) = command_dispatch(&command_facts(&args)?) else { + return Err(PyValueError::new_err( + "frozen settings must dispatch a build", + )); + }; + let request = Invocation::new(&args)?.value(route.arguments[0].source)?; + Ok(( + replaced(&request, &[("discovered_inputs", discovered)])?, + warehouse_scope(&args)?, + )) +} + +/// The heartbeat is native; only driver cancellation crosses the executor host boundary. +fn run_cancellable( + py: Python<'_>, + worker: &Arc, + request: &Bound<'_, PyAny>, +) -> PyResult { + let state = Arc::clone(worker); + let (stop, stopping) = mpsc::channel::<()>(); + let thread = std::thread::Builder::new() + .name("sqb-worker-cancel".into()) + .spawn(move || { + while matches!( + stopping.recv_timeout(Duration::from_millis(100)), + Err(mpsc::RecvTimeoutError::Timeout) + ) { + let _ = Python::attach(|py| -> PyResult<()> { + if state.worker.bind(py).get().cancel_requested(py) { + call_with(&entry(py, ExecutorHost::CancelRunningStatements)?, &[])?; + } + Ok(()) + }); + } + }) + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let result = run_build(py, request, Some(Arc::clone(worker))); + drop(stop); + py.detach(|| thread.join()) + .map_err(|_| PyRuntimeError::new_err("worker cancellation thread failed"))?; + result +} + +pub(super) fn execute_worker<'py>( + py: Python<'py>, + (snapshot, discovered): (&NativeOrchestrationSnapshot, Bound<'py, PyAny>), + arguments: &KeywordArguments<'py>, + (bearer, identity, recall): (&str, (String, String), Option<&InvocationRecall>), +) -> PyResult { + write_console(py, "stderr", "Verifying frozen run inputs...\n")?; + let owner = Bound::new( + py, + NativeOrchestrationWorker::start( + snapshot, + py, + (&arguments.value("server_url")?.extract::()?, bearer), + identity, + ) + .map_err(|error| PyValueError::new_err(error.to_string()))?, + )?; + let worker = WorkerBuild::new(&owner)?; + if let Some(recall) = recall { + *recall + .worker + .lock() + .map_err(|_| PyRuntimeError::new_err("worker completion lock is unavailable"))? = + Some(Arc::clone(&worker)); + } + let result = (|| { + if !worker.has_work() { + write_console( + py, + "stderr", + "Run inputs verified. All selected nodes are blocked by upstream failures.\n", + )?; + return Ok(0); + } + let (request, scope) = build_request(py, discovered, arguments, &owner)?; + write_console( + py, + "stderr", + "Run inputs verified. Executing frozen selection...\n", + )?; + within(&scope, |_| run_cancellable(py, &worker, &request)) + })(); + let code = result.as_ref().copied().unwrap_or(1); + let outcome = if recall.is_some() { + worker.outcome(py, code)? + } else { + worker.finish(py, code)? + }; + result.map(|code| { + if matches!(outcome, RunOutcome::Success | RunOutcome::Warn) { + code + } else { + code.max(1) + } + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 457ca748d9..732a57220e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -59,6 +59,7 @@ pub(crate) mod janitor; pub(crate) mod lifecycle_events; pub(crate) mod lineage; pub(crate) mod load; +pub(crate) mod location_command; pub(crate) mod macro_bridge; pub(crate) mod manifest; pub(crate) mod microbatches; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs new file mode 100644 index 0000000000..db99dfcfe9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -0,0 +1,13 @@ +//! Authoring parameters reserved by the code-location host. + +/// The calendar's evaluation window is supplied by the host, not a provider. +pub(crate) const CALENDAR_WINDOW_PARAMETER: &str = "window"; + +pub(crate) const RUN_EVENT_CONDITION: &str = "run_event"; +pub(crate) const NODE_EVENT_CONDITION: &str = "node_event"; +pub(crate) const CHECK_KIND: &str = "check"; +pub(crate) const SOURCE_NEW_DATA_CONDITION: &str = "source_new_data"; +pub(crate) const SOURCE_STALE_CONDITION: &str = "source_stale"; + +/// The sensor context is supplied by the host, not a provider. +pub(crate) const SENSOR_CONTEXT_PARAMETER: &str = "ctx"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs index c2904c48dc..babcc38c91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs @@ -65,6 +65,21 @@ pub(crate) fn entry<'py>(py: Python<'py>, id: impl Into) -> PyResult, stream: &str, text: &str) -> PyResult<()> { + let stream = py.import("sys")?.getattr(stream)?; + stream.call_method1("write", (text,))?; + stream.call_method0("flush")?; + Ok(()) +} + +/// Keep user prints off machine-readable stdout while running project code. +pub(crate) fn redirect_stdout_to_stderr(py: Python<'_>) -> PyResult> { + py.import("contextlib")? + .getattr("redirect_stdout")? + .call1((py.import("sys")?.getattr("stderr")?,)) +} + /// `log_debug_event(logger=logging.getLogger(logger), message=message)`. pub(crate) fn log_debug_message(py: Python<'_>, logger: &str, message: &str) -> PyResult<()> { let kwargs: Bound<'_, PyDict> = PyDict::new(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/interrupt.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/interrupt.rs new file mode 100644 index 0000000000..d10774e0fc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/interrupt.rs @@ -0,0 +1,84 @@ +//! Python's default compilation Ctrl+C handler, retained across unraisable callbacks. + +use std::cell::Cell; + +use pyo3::exceptions::PyKeyboardInterrupt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyCFunction, PyDict, PyTuple}; + +thread_local! { + static INTERRUPTED: Cell = const { Cell::new(false) }; +} +const SIGINT: i64 = 2; + +/// The previous Python handler and the command's replacement, restored on every exit path. +pub(crate) struct InterruptHandler { + previous: Py, + installed: Py, +} + +impl InterruptHandler { + /// Capture only Python's default main-thread handler; leave embedding applications' handlers alone. + pub(crate) fn install(py: Python<'_>) -> PyResult> { + let threading = py.import("threading")?; + if !threading + .call_method0("current_thread")? + .is(threading.call_method0("main_thread")?) + { + return Ok(None); + } + let signal = py.import("signal")?; + let previous = signal.call_method1("getsignal", (SIGINT,))?; + if !previous.is(signal.getattr("default_int_handler")?) { + return Ok(None); + } + let installed = PyCFunction::new_closure( + py, + None, + None, + |_: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| -> PyResult<()> { + INTERRUPTED.set(true); + Err(PyKeyboardInterrupt::new_err(())) + }, + )?; + INTERRUPTED.set(false); + signal.call_method1("signal", (SIGINT, &installed))?; + Ok(Some(Self { + previous: previous.unbind(), + installed: installed.into_any().unbind(), + })) + } +} + +impl Drop for InterruptHandler { + fn drop(&mut self) { + Python::attach(|py| { + let restored: PyResult<()> = (|| { + let signal = py.import("signal")?; + if signal + .call_method1("getsignal", (SIGINT,))? + .is(self.installed.bind(py)) + { + signal.call_method1("signal", (SIGINT, self.previous.bind(py)))?; + } + Ok(()) + })(); + drop(restored); + }); + INTERRUPTED.set(false); + } +} + +/// Whether the default handler observed Ctrl+C during the current command. +pub(crate) fn interrupted() -> bool { + INTERRUPTED.get() +} + +/// Check both pending Python signals and a Ctrl+C whose exception a Python callback swallowed. +pub(crate) fn check_signals(py: Python<'_>) -> PyResult<()> { + py.check_signals()?; + if interrupted() { + return Err(PyKeyboardInterrupt::new_err(())); + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 8f9ded74e2..849f8bf165 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -1,10 +1,20 @@ //! Permanent Python host calls and compatibility views of native-owned execution data. pub(crate) mod connections; +pub(crate) mod constants; pub(crate) mod header_projection; pub(crate) mod host; +pub(crate) mod interrupt; pub(crate) mod model_lookup; +pub(crate) mod orchestration_declarations; +pub(crate) mod orchestration_environment; +pub(crate) mod orchestration_sensors; +pub(crate) mod orchestration_triggers; pub(crate) mod registration; pub(crate) mod results; pub(crate) mod sqlite_connection; pub(crate) mod sqlite_cursor; +pub(crate) mod state_column_evolution; +pub(crate) mod warehouse_catalog; +pub(crate) mod warehouse_metadata; +pub(crate) mod warehouse_statistics; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs new file mode 100644 index 0000000000..4bfe0de411 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -0,0 +1,233 @@ +//! The public authoring records enter native compilation here, never scheduler execution. + +use std::collections::{BTreeMap, HashSet}; +use std::path::Path; + +use pyo3::FromPyObject; +use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use serde_json::Value; +use sqlbuild_host::entries::models::UserCodeHost; + +use crate::bindings::_helpers::native_host::constants::RUN_EVENT_CONDITION; +use crate::bindings::_helpers::native_host::host::entry; + +#[derive(FromPyObject)] +pub(crate) struct AuthoredJob { + pub(crate) name: String, + pub(crate) description: String, + pub(crate) select: String, + pub(crate) exclude: String, + #[pyo3(from_py_with = authored_setting)] + pub(crate) settings: Value, + #[pyo3(from_py_with = authored_setting)] + pub(crate) concurrency: Value, + pub(crate) concurrency_groups: Vec, + pub(crate) schedule: Option>, + pub(crate) targets: Vec, +} + +#[derive(FromPyObject, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct AuthoredJobReference { + pub(crate) name: String, +} + +#[derive(FromPyObject)] +pub(crate) struct AuthoredGroup { + pub(crate) name: String, + pub(crate) description: String, + pub(crate) jobs: Vec, + pub(crate) after: BTreeMap>, + pub(crate) concurrency_groups: Vec, + pub(crate) schedule: Option>, + pub(crate) targets: Vec, +} + +#[derive(FromPyObject)] +pub(crate) struct AuthoredSensorCondition { + pub(crate) kind: String, + #[pyo3(default)] + pub(crate) kinds: Vec, + #[pyo3(default)] + pub(crate) job: Option, + #[pyo3(default)] + pub(crate) executed_successfully: Option>, + #[pyo3(default)] + pub(crate) select: Option, + #[pyo3(default, from_py_with = authored_setting)] + pub(crate) after_seconds: Value, +} + +#[derive(FromPyObject)] +pub(crate) struct AuthoredSensor { + pub(crate) name: String, + pub(crate) description: String, + pub(crate) when: Option, + pub(crate) function: Option>, + pub(crate) run: AuthoredJobReference, + #[pyo3(from_py_with = authored_setting)] + pub(crate) minimum_interval_seconds: Value, + pub(crate) default_status: String, + pub(crate) targets: Vec, +} + +pub(crate) enum AuthoredDeclaration { + Job(AuthoredJob), + Group(AuthoredGroup), + Calendar(Py), + Sensor(AuthoredSensor), +} + +/// JSON authoring values cross once; the interpreter rejects circular user containers safely. +pub(crate) fn authored_setting(value: &Bound<'_, PyAny>) -> PyResult { + let text: String = value + .py() + .import("json")? + .call_method1("dumps", (value,))? + .extract()?; + serde_json::from_str(&text).map_err(|error| PyValueError::new_err(error.to_string())) +} + +/// Only module execution crosses into Python; discovery and collection stay native. +pub(crate) fn module_jobs<'py>( + py: Python<'py>, + file_path: &Path, + project_dir: &Path, +) -> PyResult>> { + let options = PyDict::new(py); + options.set_item("file_path", file_path)?; + options.set_item("project_dir", project_dir)?; + let module = entry(py, UserCodeHost::LoadOrchestrationModule)?.call((), Some(&options))?; + let job_class = py + .import("sqlbuild.orchestration_declarations.models.authoring")? + .getattr("Job")?; + let group_class = py + .import("sqlbuild.orchestration_declarations.models.authoring")? + .getattr("JobGroup")?; + let calendar_class = py + .import("sqlbuild.orchestration_declarations.models.authoring")? + .getattr("CalendarSchedule")?; + let sensor_class = py + .import("sqlbuild.orchestration_declarations.models.authoring")? + .getattr("Sensor")?; + let attributes = module.getattr("__dict__")?.cast_into::()?; + let mut pending: Vec> = attributes.values().iter().collect(); + let mut seen: HashSet = HashSet::new(); + let mut jobs: Vec> = Vec::new(); + while let Some(value) = pending.pop() { + if !seen.insert(value.as_ptr() as usize) { + continue; + } + if value.is_instance(&job_class)? + || value.is_instance(&group_class)? + || value.is_instance(&calendar_class)? + || value.is_instance(&sensor_class)? + { + jobs.push(value); + } else if value.is_instance_of::() || value.is_instance_of::() { + pending.extend(value.try_iter()?.collect::>>()?); + } + } + Ok(jobs) +} + +pub(crate) fn collect_declarations<'py>( + py: Python<'py>, + declarations: Vec>, +) -> PyResult> { + let job = py + .import("sqlbuild.orchestration_declarations.models.authoring")? + .getattr("Job")?; + let group = py + .import("sqlbuild.orchestration_declarations.models.authoring")? + .getattr("JobGroup")?; + let calendar = py + .import("sqlbuild.orchestration_declarations.models.authoring")? + .getattr("CalendarSchedule")?; + let sensor = py + .import("sqlbuild.orchestration_declarations.models.authoring")? + .getattr("Sensor")?; + let mut roots: HashSet = HashSet::new(); + for declaration in &declarations { + if !roots.insert(declaration.as_ptr() as usize) { + if declaration.is_instance(&calendar)? { + return Err(PyValueError::new_err("duplicate calendar declaration")); + } + return Err(PyValueError::new_err(format!( + "duplicate job {}", + declaration.getattr("name")?.extract::()? + ))); + } + } + let mut pending = declarations; + let mut seen: HashSet = HashSet::new(); + let mut collected: Vec = Vec::new(); + while let Some(declaration) = pending.pop() { + if !seen.insert(declaration.as_ptr() as usize) { + continue; + } + let authored = if declaration.is_instance(&sensor)? { + let target = declaration.getattr("run")?; + if !target.is_instance(&job)? && !target.is_instance(&group)? { + return Err(PyTypeError::new_err( + "sensor run must be a Job or JobGroup object", + )); + } + pending.push(target); + let condition = declaration.getattr("when")?; + if !condition.is_none() + && condition.getattr("kind")?.extract::()? == RUN_EVENT_CONDITION + { + let source = condition.getattr("job")?; + if !source.is_instance(&job)? && !source.is_instance(&group)? { + return Err(PyTypeError::new_err( + "run_event job must be a Job or JobGroup object", + )); + } + pending.push(source); + } + AuthoredDeclaration::Sensor(declaration.extract()?) + } else if declaration.is_instance(&calendar)? { + let target = declaration.getattr("job")?; + if !target.is_instance(&job)? && !target.is_instance(&group)? { + return Err(PyTypeError::new_err( + "calendar job must be a Job or JobGroup object", + )); + } + pending.push(target); + AuthoredDeclaration::Calendar(declaration.unbind()) + } else if declaration.is_instance(&group)? { + for member in declaration.getattr("jobs")?.try_iter()? { + let member = member?; + if !member.is_instance(&job)? { + return Err(PyTypeError::new_err("group members must be Job objects")); + } + pending.push(member); + } + for (member, dependencies) in declaration.getattr("after")?.cast::()?.iter() { + if !member.is_instance(&job)? { + return Err(PyTypeError::new_err( + "group dependencies must be Job objects", + )); + } + for dependency in dependencies.try_iter()? { + if !dependency?.is_instance(&job)? { + return Err(PyTypeError::new_err( + "group dependencies must be Job objects", + )); + } + } + } + AuthoredDeclaration::Group(declaration.extract()?) + } else if !declaration.is_instance(&job)? { + return Err(PyTypeError::new_err( + "snapshot declarations must be Job, JobGroup, CalendarSchedule or Sensor objects", + )); + } else { + AuthoredDeclaration::Job(declaration.extract()?) + }; + collected.push(authored); + } + Ok(collected) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs new file mode 100644 index 0000000000..26cfa2bf67 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs @@ -0,0 +1,108 @@ +//! Read the location interpreter's installed dependencies, never its credentials or paths. + +use std::path::Path; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sha2::{Digest, Sha256}; +use sqlbuild_discovery::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; +use sqlbuild_discovery::tree::main::rglob::rglob; +use sqlbuild_discovery::tree::models::ProjectTree; + +/// A lockfile binds declared dependencies; without one, bind installed distribution versions. +pub(crate) fn environment_hash(py: Python<'_>, project_dir: &Path) -> PyResult { + let mut hash = Sha256::new(); + let version = py.version_info(); + hash.update(format!( + "sqlbuild:{}\npython:{}.{}.{}{}\n", + env!("CARGO_PKG_VERSION"), + version.major, + version.minor, + version.patch, + version.suffix.unwrap_or_default(), + )); + match py.detach(|| std::fs::read(project_dir.join("uv.lock"))) { + Ok(lockfile) => { + hash.update(b"uv.lock\n"); + hash.update(lockfile); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let mut packages: Vec<(String, String)> = Vec::new(); + for distribution in py + .import("importlib.metadata")? + .call_method0("distributions")? + .try_iter()? + { + let distribution = distribution?; + let name: String = distribution + .getattr("metadata")? + .get_item("Name")? + .extract()?; + let version: String = distribution.getattr("version")?.extract()?; + packages.push((name.replace(['_', '.'], "-").to_ascii_lowercase(), version)); + } + packages.sort(); + hash.update(b"installed-packages\n"); + hash.update( + serde_json::to_vec(&packages) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?, + ); + } + Err(error) => return Err(error.into()), + } + Ok(format!("{:x}", hash.finalize())) +} + +/// Hash declaration/helper sources once per compilation, never once per sensor. +pub(crate) fn orchestration_code_hash(project_dir: &Path) -> PyResult { + let tree = ProjectTree::new(project_dir); + let files = rglob(&tree, ORCHESTRATION_DIRECTORY, |entry| { + !entry.is_dir && entry.name.ends_with(PYTHON_FILE_SUFFIX) + }) + .map_err(|error| { + pyo3::exceptions::PyValueError::new_err(format!( + "sensor source discovery failed: {error:?}" + )) + })?; + let mut hash = Sha256::new(); + for file in files { + let bytes = std::fs::read(tree.absolute(&file))?; + hash.update(file.len().to_le_bytes()); + hash.update(file.as_bytes()); + hash.update(bytes.len().to_le_bytes()); + hash.update(bytes); + } + Ok(format!("{:x}", hash.finalize())) +} + +/// Include the callable's code even for public in-memory declaration compilation. +pub(crate) fn callback_code_hash(function: &Bound<'_, PyAny>, sources: &str) -> PyResult { + let bytes: Vec = function + .py() + .import("marshal")? + .call_method1("dumps", (portable_code(function.getattr("__code__")?)?,))? + .extract()?; + let mut hash = Sha256::new(); + hash.update(sources.as_bytes()); + hash.update(bytes); + Ok(format!("{:x}", hash.finalize())) +} + +/// Code objects retain absolute import paths, including inside nested functions. +fn portable_code(code: Bound<'_, PyAny>) -> PyResult> { + let py = code.py(); + let code_type = py.import("types")?.getattr("CodeType")?; + let mut constants: Vec> = Vec::new(); + for value in code.getattr("co_consts")?.try_iter()? { + let value = value?; + constants.push(if value.is_instance(&code_type)? { + portable_code(value)? + } else { + value + }); + } + let changes = PyDict::new(py); + changes.set_item("co_filename", "")?; + changes.set_item("co_consts", PyTuple::new(py, constants)?)?; + code.call_method("replace", (), Some(&changes)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs new file mode 100644 index 0000000000..434c71c2fc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs @@ -0,0 +1,135 @@ +//! User callbacks and provider lifetimes belong only to the isolated location interpreter. + +use std::path::Path; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_orchestration::sensors::models::{ + CustomSensorOutcome, SensorCondition, SensorEvaluationRequest, +}; + +use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::compiled_project::snapshot_declarations::discover_declarations; +use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; +use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; +use crate::bindings::_helpers::native_host::constants::SENSOR_CONTEXT_PARAMETER; +use crate::bindings::_helpers::native_host::host::entry; +use crate::bindings::_helpers::native_host::orchestration_environment::{ + callback_code_hash, orchestration_code_hash, +}; + +pub(crate) fn evaluate_sensor( + py: Python<'_>, + project_dir: &Path, + evaluation: &SensorEvaluationRequest, +) -> PyResult { + let SensorCondition::Custom { + function: name, + code_hash, + } = &evaluation.definition.when + else { + return Err(PyValueError::new_err("custom sensor work required")); + }; + let authoring = py.import("sqlbuild.orchestration_declarations.models.authoring")?; + let sensor_class = authoring.getattr("Sensor")?; + let mut matches: Vec> = Vec::new(); + for declaration in discover_declarations(py, project_dir)? { + if declaration.is_instance(&sensor_class)? + && declaration.getattr("name")?.extract::()? == *name + { + matches.push(declaration); + } + } + let [sensor] = matches.as_slice() else { + return Err(PyValueError::new_err( + "custom sensor declaration must resolve uniquely", + )); + }; + let targets: Vec = sensor.getattr("targets")?.extract()?; + if !sensor.getattr("when")?.is_none() + || (!targets.is_empty() && !targets.contains(&evaluation.target)) + || sensor + .getattr("run")? + .getattr("name")? + .extract::()? + != evaluation.definition.job_key + { + return Err(PyValueError::new_err("custom sensor definition changed")); + } + let function = sensor.getattr("function")?; + let sources = py.detach(|| orchestration_code_hash(project_dir))?; + if callback_code_hash(&function, &sources)? != *code_hash { + return Err(PyValueError::new_err("custom sensor code changed")); + } + let fields = PyDict::new(py); + fields.set_item("cursor", &evaluation.cursor.custom)?; + fields.set_item("target", &evaluation.target)?; + fields.set_item("evaluation_id", evaluation.evaluation_id.to_string())?; + let context = py + .import("sqlbuild.orchestration")? + .getattr("SensorContext")? + .call((), Some(&fields))?; + let providers = + discover_provider_classes(imported(py, "pathlib", "Path")?.call1((project_dir,))?)?; + let by_name = provider_by_name(&providers)?; + let session = call_with( + &entry(py, UserCodeHost::BuildProviderSession)?, + &[ + ("discovered_providers", providers), + ("allow_shared", PyBool::new(py, false).to_owned().into_any()), + ], + )?; + within(&session, |_| { + let arguments = PyDict::new(py); + arguments.set_item("ctx", &context)?; + for usage in provider_usages(&function, &by_name)?.try_iter()? { + let usage = usage?; + if usage.getattr("parameter_name")?.extract::()? != SENSOR_CONTEXT_PARAMETER { + arguments.set_item( + usage.getattr("parameter_name")?, + session.call_method1("get", (usage.getattr("provider_name")?,))?, + )?; + } + } + let result = function.call((), Some(&arguments))?; + if result.is_instance(&authoring.getattr("SkipReason")?)? { + let reason: String = result.getattr("reason")?.extract()?; + if reason.trim().is_empty() { + return Err(PyValueError::new_err("SkipReason needs a nonempty reason")); + } + return Ok(CustomSensorOutcome::Skipped { + cursor: context.getattr("cursor")?.extract()?, + reason, + }); + } + let request_class = authoring.getattr("RunRequest")?; + let mut run_keys: Vec = Vec::new(); + for request in result + .try_iter()? + .take(sqlbuild_orchestration::sensors::constants::MAX_SENSOR_RUN_REQUESTS + 1) + { + let request = request?; + if !request.is_instance(&request_class)? { + return Err(PyValueError::new_err( + "custom sensors return RunRequest records or SkipReason", + )); + } + let key: String = request.getattr("run_key")?.extract()?; + if key.trim().is_empty() { + return Err(PyValueError::new_err("RunRequest needs a nonempty run_key")); + } + run_keys.push(key); + } + if run_keys.len() > sqlbuild_orchestration::sensors::constants::MAX_SENSOR_RUN_REQUESTS { + return Err(PyValueError::new_err( + "custom sensor exceeds the bounded run-request page", + )); + } + Ok(CustomSensorOutcome::Completed { + cursor: context.getattr("cursor")?.extract()?, + run_keys, + }) + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs new file mode 100644 index 0000000000..532b5e0723 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -0,0 +1,364 @@ +//! Authored trigger controls and bounded calendars evaluated only in the location interpreter. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use chrono::{DateTime, TimeDelta, Timelike, Utc}; +use chrono_tz::Tz; +use pyo3::FromPyObject; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use serde_json::{Value, json}; +use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_orchestration::declarations::models::{Definitions, Schedule, TriggerDefinition}; +use sqlbuild_orchestration::scheduler::constants::MAX_TICKS_PER_BATCH; +use sqlbuild_orchestration::sensors::models::SensorDefinition; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; + +use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; +use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; +use crate::bindings::_helpers::native_host::constants::{ + CALENDAR_WINDOW_PARAMETER, CHECK_KIND, NODE_EVENT_CONDITION, SOURCE_NEW_DATA_CONDITION, + SOURCE_STALE_CONDITION, +}; +use crate::bindings::_helpers::native_host::host::entry; +use crate::bindings::_helpers::native_host::orchestration_declarations::{ + AuthoredSensor, authored_setting, +}; +use crate::bindings::_helpers::native_host::orchestration_environment::{ + callback_code_hash, orchestration_code_hash, +}; + +#[derive(FromPyObject)] +struct AuthoredControls { + #[pyo3(from_py_with = authored_setting)] + starts_at: Value, + default_status: String, +} + +fn compile_trigger( + authored: &Bound<'_, PyAny>, + job_key: &str, + description: &str, + schedule: Schedule, +) -> PyResult { + let controls = authored.extract::()?; + Ok(TriggerDefinition { + description: description.into(), + job_key: job_key.into(), + schedule, + starts_at: serde_json::from_value(controls.starts_at) + .map_err(|error| PyValueError::new_err(format!("starts_at: {error}")))?, + default_status: serde_json::from_value(Value::String(controls.default_status)) + .map_err(|error| PyValueError::new_err(format!("default_status: {error}")))?, + }) +} + +pub(crate) fn compile_cron( + authored: &Bound<'_, PyAny>, + name: &str, + description: &str, +) -> PyResult { + compile_trigger( + authored, + name, + description, + Schedule::Cron { + expression: authored.getattr("cron")?.extract()?, + timezone: authored.getattr("timezone")?.extract()?, + }, + ) +} + +fn horizon(value: &str) -> PyResult { + let invalid = || { + PyValueError::new_err( + "calendar horizon must be a positive fixed duration using d, h, m or s, within datetime range", + ) + }; + let digits = parse_duration(value).ok_or_else(invalid)?; + if [&digits.years, &digits.months] + .iter() + .any(|digits| !digits.trim_start_matches('0').is_empty()) + { + return Err(invalid()); + } + let seconds = [ + (digits.days, 86_400_i64), + (digits.hours, 3_600), + (digits.minutes, 60), + (digits.seconds, 1), + ] + .into_iter() + .try_fold(0_i64, |total, (digits, factor)| { + let digits = digits.parse::().map_err(|_| invalid())?; + let seconds = digits.checked_mul(factor).ok_or_else(invalid)?; + total.checked_add(seconds).ok_or_else(invalid) + })?; + TimeDelta::try_seconds(seconds).ok_or_else(invalid) +} + +fn calendar_times( + authored: &Bound<'_, PyAny>, + session: &Bound<'_, PyAny>, + providers: &Bound<'_, PyDict>, + start: DateTime, +) -> PyResult { + let py = authored.py(); + let timezone: String = authored.getattr("timezone")?.extract()?; + timezone + .parse::() + .map_err(|_| PyValueError::new_err("calendar timezone must be an IANA timezone"))?; + let duration = horizon(&authored.getattr("horizon")?.extract::()?)?; + let end = start + .checked_add_signed(duration) + .ok_or_else(|| PyValueError::new_err("calendar horizon is out of datetime range"))?; + let datetime = imported(py, "datetime", "datetime")?; + let zone = imported(py, "zoneinfo", "ZoneInfo")?.call1((timezone,))?; + let window = PyDict::new(py); + for (name, instant) in [("start", start), ("end", end)] { + window.set_item( + name, + datetime + .call_method1("fromisoformat", (instant.to_rfc3339(),))? + .call_method1("astimezone", (&zone,))?, + )?; + } + let window = py + .import("sqlbuild.orchestration")? + .getattr("CalendarWindow")? + .call((), Some(&window))?; + let function = authored.getattr("function")?; + let arguments = PyDict::new(py); + arguments.set_item(CALENDAR_WINDOW_PARAMETER, window)?; + for usage in provider_usages(&function, providers)?.try_iter()? { + let usage = usage?; + if usage.getattr("parameter_name")?.extract::()? == CALENDAR_WINDOW_PARAMETER { + continue; + } + arguments.set_item( + usage.getattr("parameter_name")?, + session.call_method1("get", (usage.getattr("provider_name")?,))?, + )?; + } + let mut times: BTreeSet> = BTreeSet::new(); + for (index, time) in function + .call((), Some(&arguments))? + .try_iter()? + .take(MAX_TICKS_PER_BATCH + 1) + .enumerate() + { + if index == MAX_TICKS_PER_BATCH { + return Err(PyValueError::new_err(format!( + "calendar exceeds {MAX_TICKS_PER_BATCH} upcoming times" + ))); + } + let time = time?; + if !time.is_instance(&datetime)? { + return Err(PyValueError::new_err( + "calendar times must be timezone-aware datetimes", + )); + } + let time = + DateTime::parse_from_rfc3339(&time.call_method0("isoformat")?.extract::()?) + .map_err(|_| { + PyValueError::new_err("calendar times must be timezone-aware datetimes") + })? + .with_timezone(&Utc); + if time < start || time >= end { + return Err(PyValueError::new_err( + "calendar time lies outside its half-open evaluation window", + )); + } + times.insert(time); + } + Ok(Schedule::Calendar { + times, + refresh_at: start + duration / 2, + }) +} + +pub(crate) fn compile_calendars( + py: Python<'_>, + (project_dir, target): (&Path, &str), + authored: Vec>, + mut definitions: Definitions, +) -> PyResult { + let mut calendars: Vec> = Vec::new(); + for calendar in authored { + let targets = calendar + .bind(py) + .getattr("targets")? + .extract::>()?; + if targets.is_empty() || targets.iter().any(|value| value == target) { + calendars.push(calendar); + } + } + if calendars.is_empty() { + return Ok(definitions); + } + let providers = + discover_provider_classes(imported(py, "pathlib", "Path")?.call1((project_dir,))?)?; + let by_name = provider_by_name(&providers)?; + let session = call_with( + &entry(py, UserCodeHost::BuildProviderSession)?, + &[ + ("discovered_providers", providers), + ("allow_shared", PyBool::new(py, false).to_owned().into_any()), + ], + )?; + let now = Utc::now(); + let start = now + .with_nanosecond(now.timestamp_subsec_micros() * 1_000) + .ok_or_else(|| PyValueError::new_err("calendar clock is out of range"))?; + within(&session, |_| { + for calendar in calendars { + let calendar = calendar.bind(py); + let name: String = calendar + .getattr("function")? + .getattr("__name__")? + .extract()?; + let job_key: String = calendar.getattr("job")?.getattr("name")?.extract()?; + let description: String = calendar.getattr("description")?.extract()?; + if description.trim().is_empty() + || (!definitions.jobs.contains_key(&job_key) + && !definitions.groups.contains_key(&job_key)) + { + return Err(PyValueError::new_err(format!( + "calendar {name} needs a description and a published job" + ))); + } + if definitions.triggers.contains_key(&name) { + return Err(PyValueError::new_err(format!("duplicate trigger {name}"))); + } + let schedule = calendar_times(calendar, &session, &by_name, start)?; + let mut trigger = compile_trigger(calendar, &job_key, &description, schedule)?; + trigger.starts_at.get_or_insert(start); + definitions.triggers.insert(name, trigger); + } + Ok(definitions) + }) +} + +pub(crate) fn compile_sensors( + py: Python<'_>, + (project_dir, input, graph): (&Path, &SnapshotInput, &ProjectGraph), + authored: Vec, + mut definitions: Definitions, +) -> PyResult { + let sources = if authored.iter().any(|sensor| sensor.function.is_some()) { + orchestration_code_hash(project_dir)? + } else { + String::new() + }; + let mut names: BTreeSet = BTreeSet::new(); + for sensor in authored { + if !names.insert(sensor.name.clone()) || definitions.triggers.contains_key(&sensor.name) { + return Err(PyValueError::new_err(format!( + "duplicate trigger {}", + sensor.name + ))); + } + if !sensor.targets.is_empty() && !sensor.targets.iter().any(|value| value == &input.target) + { + continue; + } + let condition = match (sensor.when, sensor.function) { + (Some(when), None) => { + let mut condition = json!({"kind": when.kind}); + if when.kind == SOURCE_STALE_CONDITION { + condition["after_seconds"] = when.after_seconds; + } else if when.kind != SOURCE_NEW_DATA_CONDITION { + condition["kinds"] = json!(when.kinds); + } + if let Some(job) = when.job { + condition["job_key"] = json!(job.name); + } + if let Some(select) = when.select { + let selectors: Vec = + select.split_whitespace().map(str::to_owned).collect(); + condition["selection"] = + sensor_nodes(graph, &input.nodes, &selectors, &when.kind)?; + } + if let Some(selectors) = when.executed_successfully { + condition["executed_successfully"] = + sensor_nodes(graph, &input.nodes, &selectors, "executed_successfully")?; + } + condition + } + (None, Some(function)) => json!({"kind": "custom", "function": sensor.name, + "code_hash": callback_code_hash(function.bind(py), &sources)?}), + _ => { + return Err(PyValueError::new_err( + "sensor needs either a built-in condition or a decorated polling callback; callbacks with when are not supported yet", + )); + } + }; + let definition: SensorDefinition = serde_json::from_value(json!({ + "description": sensor.description, + "job_key": sensor.run.name, + "when": condition, + "minimum_interval_seconds": sensor.minimum_interval_seconds, + "default_status": sensor.default_status, + })) + .map_err(|error| PyValueError::new_err(format!("sensor {}: {error}", sensor.name)))?; + definitions.sensors.insert(sensor.name, definition); + } + Ok(definitions) +} + +fn sensor_nodes( + graph: &ProjectGraph, + available: &BTreeMap, + selectors: &[String], + field: &str, +) -> PyResult { + if selectors.is_empty() || selectors.iter().any(|value| value.trim().is_empty()) { + return Err(PyValueError::new_err(format!( + "{field} needs nonempty selectors" + ))); + } + let mut nodes: BTreeSet = BTreeSet::new(); + let mut sql_selectors: Vec = Vec::new(); + for selector in selectors { + if field == NODE_EVENT_CONDITION && selector.starts_with("check:") { + if !available + .get(selector) + .is_some_and(|node| node.kind == CHECK_KIND) + { + return Err(PyValueError::new_err(format!( + "unknown check node {selector}" + ))); + } + nodes.insert(selector.clone()); + } else { + sql_selectors.push(selector.clone()); + } + } + if !sql_selectors.is_empty() { + nodes.extend( + resolve_selectors(graph, &sql_selectors, &[]) + .map_err(|error| PyValueError::new_err(error.message))? + .into_iter() + .map(|(kind, name)| format!("{kind}:{name}")), + ); + } + if nodes.is_empty() { + return Err(PyValueError::new_err(format!( + "{field} needs nonempty selectors" + ))); + } + if matches!(field, SOURCE_NEW_DATA_CONDITION | SOURCE_STALE_CONDITION) + && nodes.iter().any(|key| !key.starts_with("source:")) + { + return Err(PyValueError::new_err(format!( + "{field} must select only source nodes" + ))); + } + Ok(json!(nodes)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/state_column_evolution.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/state_column_evolution.rs new file mode 100644 index 0000000000..bfb47dcba2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/state_column_evolution.rs @@ -0,0 +1,85 @@ +//! Warehouse catalog and nullable-column DDL adapter edge for persisted event identity. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyString, PyTuple}; +use sqlbuild_host::entries::models::AdapterHost; + +use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::records::fields::field; + +/// Missing callbacks retain the existing renderer-only API for already-current/new schemas. +pub(crate) fn ensure_event_id_column<'py>( + (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (database, schema): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (table, qualified): (&str, &str), + (read_columns, render_add, render_type): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult<()> { + if read_columns.is_none() && render_add.is_none() { + return Ok(()); + } + if read_columns.is_none() || render_add.is_none() { + return Err(PyValueError::new_err( + "state column evolution requires both read_columns and render_add_columns", + )); + } + let py = connection.py(); + let scope = [ + ("connection", connection.clone()), + ("database", database.clone()), + ("schema", schema.clone()), + ("name", PyString::new(py, table).into_any()), + ]; + if event_id_present(read_columns, &scope)? { + return Ok(()); + } + let string_type = render_type.call1((field( + &host::entry(py, AdapterHost::FrameworkType)?, + "STRING", + )?,))?; + let column = call_with( + &host::entry(py, AdapterHost::ColumnInfo)?, + &[ + ("name", PyString::new(py, "event_id").into_any()), + ("type", string_type), + ], + )?; + let statements = call_with( + render_add, + &[ + ("destination", PyString::new(py, qualified).into_any()), + ("columns", PyTuple::new(py, [column])?.into_any()), + ], + )?; + for statement in statements.try_iter()? { + if let Err(error) = call_with( + execute, + &[("connection", connection.clone()), ("sql", statement?)], + ) && !event_id_present(read_columns, &scope).unwrap_or(false) + { + return Err(error); + } + } + Ok(()) +} + +/// Concurrent additions are harmless only once the authoritative catalog confirms the column. +fn event_id_present( + read_columns: &Bound<'_, PyAny>, + scope: &[(&str, Bound<'_, PyAny>)], +) -> PyResult { + for column in call_with(read_columns, scope)?.try_iter()? { + if field(&column?, "name")? + .extract::()? + .eq_ignore_ascii_case("event_id") + { + return Ok(true); + } + } + Ok(false) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_catalog.rs new file mode 100644 index 0000000000..112caea0a2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_catalog.rs @@ -0,0 +1,132 @@ +//! Bounded catalog pages use adapter inspection, never model declarations or warehouse writes. + +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use sqlbuild_orchestration::warehouse_metadata::models::{ + CatalogRead, WarehouseCatalogColumn, WarehouseCatalogRecord, WarehouseHistoryPage, + WarehouseHistoryRecord, +}; + +use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::records::fields::field; + +pub(crate) fn read_catalog( + py: Python<'_>, + (adapter, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + request: &CatalogRead, +) -> PyResult { + let limit = request.limit.unwrap_or(MAX_HISTORY_RECORDS); + if request.observed_since.is_some() { + return Err(PyValueError::new_err( + "catalog observations require full scans, not timestamp watermarks", + )); + } + if limit == 0 || limit > MAX_HISTORY_RECORDS { + return Err(PyValueError::new_err( + "metadata page limit is outside the supported bounds", + )); + } + let namespace = [ + ( + "database", + request.location.database.clone().into_bound_py_any(py)?, + ), + ( + "schema", + request.location.schema.clone().into_bound_py_any(py)?, + ), + ]; + if !call_with( + &adapter.getattr("lists_relations_authoritatively")?, + &namespace, + )? + .extract::()? + { + return Err(PyValueError::new_err( + "catalog listing is not authoritative for this namespace; scan incomplete", + )); + } + let relations = call_with( + &adapter.getattr("list_relations")?, + &[ + ("connection", connection.clone()), + ( + "database", + request.location.database.clone().into_bound_py_any(py)?, + ), + ( + "schemas", + (request.location.schema.clone(),).into_bound_py_any(py)?, + ), + ], + )?; + let mut selected: Vec<(String, Bound<'_, PyAny>)> = Vec::new(); + for relation in relations.try_iter()? { + let relation = relation?; + let name: String = field(&relation, "name")?.extract()?; + if request + .after + .as_ref() + .and_then(|cursor| cursor[0].as_deref()) + .is_none_or(|after| name.as_str() > after) + { + selected.push((name, relation)); + } + } + selected.sort_by(|left, right| left.0.cmp(&right.0)); + let more = selected.len() > limit; + selected.truncate(limit); + if selected.is_empty() { + return Ok(WarehouseHistoryPage { + records: Vec::new(), + next_after: None, + }); + } + let names = PyTuple::new(py, selected.iter().map(|(name, _)| name))?; + let columns = call_with( + &adapter.getattr("get_all_columns")?, + &[ + ("connection", connection.clone()), + ("database", namespace[0].1.clone()), + ( + "schemas", + (request.location.schema.clone(),).into_bound_py_any(py)?, + ), + ("names", names.into_any()), + ], + )?; + let columns = columns.cast::()?; + let next_after = if more { + selected.last().map(|(name, _)| vec![Some(name.clone())]) + } else { + None + }; + let mut records: Vec = Vec::new(); + for (name, relation) in selected { + let relation_columns = columns.get_item(name.as_str())?.ok_or_else(|| { + PyValueError::new_err("catalog relation columns are unavailable; scan incomplete") + })?; + let mut decoded: Vec = Vec::new(); + for column in relation_columns.try_iter()? { + let column = column?; + decoded.push(WarehouseCatalogColumn { + name: field(&column, "name")?.extract()?, + data_type: field(&column, "type")?.extract()?, + }); + } + records.push(WarehouseHistoryRecord::Catalog(WarehouseCatalogRecord { + state_database: request.location.database.clone(), + state_schema: request.location.schema.clone(), + name, + relation_type: field(&relation, "relation_type")?.extract()?, + columns: decoded, + })); + } + Ok(WarehouseHistoryPage { + records, + next_after, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs new file mode 100644 index 0000000000..a5c265e81e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -0,0 +1,519 @@ +//! Read-only warehouse adapter edge; never compile declarations or write warehouse state. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use serde_json::{Value, json}; +use sqlbuild_analysis::fingerprint_storage::constants::{COLUMN_TIMESTAMP, FINGERPRINT_COLUMNS}; +use sqlbuild_config::command_connection::constants::{ + DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, LOCAL_DATABASE_PATH_KEYS, +}; +use sqlbuild_core::constants::{FINGERPRINT_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME}; +use sqlbuild_core::models::SqlValue; +use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use sqlbuild_orchestration::warehouse_metadata::models::{ + WarehouseHistoryPage, WarehouseHistoryRead, WarehouseHistoryRecord, WarehouseStateLocation, +}; +use sqlbuild_state::events::constants::{MIGRATION_COLUMNS, OLD_NAME_VIEW_COLUMNS}; +use sqlbuild_state::freshness::constants::{ + OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES, SQLITE_ADAPTER, +}; +use sqlbuild_state::state_tables::main::read_history_sql::read_history_sql; +use sqlbuild_state::state_tables::models::HistoryRead; + +use crate::bindings::_helpers::authored_values::sql_values::{SqlValueClasses, sql_value}; +use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::command_runtime::connection::project_connection_config; +use crate::bindings::_helpers::compiled_project::values::plain_value; +use crate::bindings::_helpers::fingerprints::latest_rows::row_fingerprint; +use crate::bindings::_helpers::freshness::state_io::row_record; +use crate::bindings::_helpers::freshness::support::presumed_utc_datetime; +use crate::bindings::_helpers::native_host::warehouse_catalog::read_catalog; +use crate::bindings::_helpers::native_host::warehouse_statistics::read_statistics; +use crate::bindings::_helpers::query_command::project::{ + ProjectAdapter, ProjectInputs, project_adapter, +}; +use crate::bindings::_helpers::records::fields::field; +use crate::bindings::_helpers::records::{construct::record_class, ids::RecordId}; + +pub(crate) fn read_history( + py: Python<'_>, + arguments: &KeywordArguments<'_>, + request: &WarehouseHistoryRead, +) -> PyResult { + let location = match request { + WarehouseHistoryRead::Statistics(read) => &read.location, + WarehouseHistoryRead::Catalog(read) => &read.location, + WarehouseHistoryRead::SourceFreshness(read) => &read.location, + WarehouseHistoryRead::Fingerprints(read) => &read.location, + WarehouseHistoryRead::AuditResults(read) + | WarehouseHistoryRead::JanitorEvents(read) + | WarehouseHistoryRead::Migrations(read) + | WarehouseHistoryRead::OldNameViews(read) + | WarehouseHistoryRead::Microbatches(read) => &read.location, + }; + if location.schema.trim().is_empty() { + return Err(PyValueError::new_err( + "metadata state schema must not be blank", + )); + } + let ProjectAdapter { + project_dir, + discovered, + adapter, + } = project_adapter( + py, + Some(arguments.value("project_dir")?), + ProjectInputs::Configuration, + )?; + let connection_config = project_connection_config( + &discovered, + (&project_dir, &arguments.value("selected_target")?), + (&py.None().into_bound(py), true), + true, + )?; + let adapter_name: String = adapter.getattr("adapter_name")?.extract()?; + if let Some((_, path_key)) = LOCAL_DATABASE_PATH_KEYS + .iter() + .find(|(name, _)| *name == adapter_name) + { + let config = connection_config.cast::()?; + let path: Option = config + .get_item(path_key)? + .map(|path| path.extract()) + .transpose()?; + if path + .as_deref() + .is_some_and(|path| path != DUCKDB_MEMORY_DATABASE) + { + config.set_item("read_only", true)?; + } + if adapter_name == DUCKDB_ADAPTER + && let Some(attachments) = config.get_item("attach")? + { + for attachment in attachments.try_iter()? { + attachment?.set_item("read_only", true)?; + } + } + } + let connection = adapter.call_method1("connect", (connection_config,))?; + let read = read_page(py, (&adapter, &connection), request); + let closed = adapter.call_method1("close", (&connection,)); + let page = read?; + closed?; + Ok(page) +} + +fn read_page( + py: Python<'_>, + (adapter, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + request: &WarehouseHistoryRead, +) -> PyResult { + let freshness_columns = SOURCE_FRESHNESS_COLUMN_TYPES.map(|(name, _)| name); + let event_columns: Vec = match request { + WarehouseHistoryRead::AuditResults(_) => py + .import("sqlbuild.executor.audit_results.constants")? + .getattr("AUDIT_RESULT_COLUMNS")? + .extract()?, + WarehouseHistoryRead::JanitorEvents(_) => py + .import("sqlbuild.executor.janitor_events.constants")? + .getattr("JANITOR_EVENT_COLUMNS")? + .extract()?, + WarehouseHistoryRead::Microbatches(_) => py + .import("sqlbuild.microbatches.constants")? + .getattr("MICROBATCH_COLUMNS")? + .extract()?, + _ => Vec::new(), + }; + let event_columns: Vec<&str> = event_columns.iter().map(String::as_str).collect(); + let (location, after, since, limit, table, columns, timestamp) = match request { + WarehouseHistoryRead::Statistics(read) => { + return read_statistics(py, (adapter, connection), read); + } + WarehouseHistoryRead::Catalog(read) => { + return read_catalog(py, (adapter, connection), read); + } + WarehouseHistoryRead::SourceFreshness(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + read.limit, + SOURCE_FRESHNESS_TABLE_NAME, + freshness_columns.as_slice(), + OBSERVED_AT_COLUMN, + ), + WarehouseHistoryRead::Fingerprints(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + read.limit, + FINGERPRINT_TABLE_NAME, + FINGERPRINT_COLUMNS.as_slice(), + COLUMN_TIMESTAMP, + ), + WarehouseHistoryRead::AuditResults(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + read.limit, + "_sqlbuild_audit_results", + event_columns.as_slice(), + "occurred_at", + ), + WarehouseHistoryRead::JanitorEvents(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + read.limit, + "_sqlbuild_janitor_events", + event_columns.as_slice(), + "occurred_at", + ), + WarehouseHistoryRead::Migrations(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + read.limit, + "_sqlbuild_migrations", + MIGRATION_COLUMNS.as_slice(), + "created_at", + ), + WarehouseHistoryRead::OldNameViews(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + read.limit, + "_sqlbuild_old_name_views", + OLD_NAME_VIEW_COLUMNS.as_slice(), + "created_at", + ), + WarehouseHistoryRead::Microbatches(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + read.limit, + "_sqlbuild_microbatches", + event_columns.as_slice(), + "created_at", + ), + }; + let limit = limit.unwrap_or(MAX_HISTORY_RECORDS); + if limit == 0 || limit > MAX_HISTORY_RECORDS { + return Err(PyValueError::new_err( + "metadata page limit is outside the supported bounds", + )); + } + let database = location.database.as_ref().map_or_else( + || py.None().into_bound(py), + |value| PyString::new(py, value).into_any(), + ); + let schema = PyString::new(py, &location.schema).into_any(); + let name = PyString::new(py, table).into_any(); + let exists = call_with( + &adapter.getattr("relation_exists")?, + &[ + ("connection", connection.clone()), + ("database", database.clone()), + ("schema", schema.clone()), + ("name", name.clone()), + ], + )? + .is_truthy()?; + if !exists { + return Ok(WarehouseHistoryPage { + records: Vec::new(), + next_after: None, + }); + } + let catalog_columns = call_with( + &adapter.getattr("get_columns")?, + &[ + ("connection", connection.clone()), + ("database", database.clone()), + ("schema", schema.clone()), + ("name", name.clone()), + ], + )?; + let mut actual_columns: std::collections::BTreeSet = std::collections::BTreeSet::new(); + for column in catalog_columns.try_iter()? { + actual_columns.insert(field(&column?, "name")?.extract::()?.to_lowercase()); + } + let has_event_id = actual_columns.contains("event_id"); + let optional_columns: &[&str] = match request { + WarehouseHistoryRead::AuditResults(_) => &["audit_description", "audit_meta_json"], + WarehouseHistoryRead::Migrations(_) => &[ + "target_name", + "origin_model", + "origin_database", + "origin_schema", + "destination_database", + ], + WarehouseHistoryRead::OldNameViews(_) => &[ + "target_name", + "old_database", + "old_schema", + "new_database", + "view_retention", + "archive_name", + "column_aliases", + "grants_copied", + "view_sql", + "expires_at", + "drop_reason", + ], + _ => &[], + }; + let key_columns: Option<&[&str]> = match request { + WarehouseHistoryRead::AuditResults(_) => Some(&["result_id", "occurred_at"]), + WarehouseHistoryRead::JanitorEvents(_) => Some(&["event_id", "occurred_at"]), + WarehouseHistoryRead::Migrations(_) + | WarehouseHistoryRead::OldNameViews(_) + | WarehouseHistoryRead::Microbatches(_) => Some(&["event_id", "created_at"]), + _ => None, + }; + let mut cursor_indexes: Vec = columns + .iter() + .enumerate() + .filter_map(|(index, name)| { + key_columns + .is_none_or(|keys| keys.contains(name)) + .then_some(index) + }) + .collect(); + cursor_indexes.push(columns.len()); + let mut literals = vec![None; key_columns.unwrap_or(columns).len() + 1]; + if let Some(after) = after { + let classes = SqlValueClasses::load(py)?; + for (output, value) in literals.iter_mut().zip(after) { + if let Some(value) = value { + let value = sql_value(&SqlValue::String(value.clone()), &classes)?; + *output = Some( + call_with( + &adapter.getattr("render_typed_scalar")?, + &[("value", value)], + )? + .extract()?, + ); + } + } + } + let qualified: String = call_with( + &adapter.getattr("render_qualified_name")?, + &[("database", database), ("schema", schema), ("name", name)], + )? + .extract()?; + let adapter_name = adapter.getattr("adapter_name")?.extract::()?; + let observed_since = since + .map(|since| { + let text = if adapter_name == SQLITE_ADAPTER { + since + .date_naive() + .pred_opt() + .unwrap_or(since.date_naive()) + .format("%Y-%m-%d") + .to_string() + } else { + since.format("%Y-%m-%dT%H:%M:%S+00:00").to_string() + }; + let classes = SqlValueClasses::load(py)?; + let value = sql_value(&SqlValue::String(text), &classes)?; + call_with( + &adapter.getattr("render_typed_scalar")?, + &[("value", value)], + )? + .extract::() + }) + .transpose()?; + let projection: Vec = columns + .iter() + .map(|column| { + if optional_columns.contains(column) && !actual_columns.contains(*column) { + format!("NULL AS {column}") + } else { + (*column).to_owned() + } + }) + .collect(); + let projection: Vec<&str> = projection.iter().map(String::as_str).collect(); + let sql = read_history_sql( + &qualified, + after.map(|_| literals.as_slice()), + HistoryRead { + columns: &projection, + key_columns: key_columns.or(Some(columns)), + timestamp_column: timestamp, + observed_since: observed_since.as_deref(), + adapter: &adapter_name, + has_event_id, + limit, + }, + ) + .map_err(PyValueError::new_err)?; + let result = call_with( + &adapter.getattr("query")?, + &[ + ("connection", connection.clone()), + ("sql", PyString::new(py, &sql).into_any()), + ( + "limit", + pyo3::IntoPyObjectExt::into_bound_py_any(limit, py)?, + ), + ], + )?; + let mut records: Vec = Vec::new(); + let mut next_after = None; + let mut read_count = 0; + for row in field(&result, "rows")?.try_iter()? { + read_count += 1; + let row = row?; + let record = decode_record(py, (request, &row, &qualified), (columns, timestamp))?; + let mut cursor = vec![None; cursor_indexes.len()]; + for (index, value) in cursor.iter_mut().enumerate() { + let raw = row.get_item(cursor_indexes[index])?; + *value = if raw.is_none() { + None + } else { + Some(raw.str()?.extract()?) + }; + } + let observed = match &record { + WarehouseHistoryRecord::Catalog(_) | WarehouseHistoryRecord::Statistics(_) => { + return Err(PyValueError::new_err( + "catalog does not use timestamp history filtering", + )); + } + WarehouseHistoryRecord::SourceFreshness(record) => record.observed_at, + WarehouseHistoryRecord::Fingerprint(record) => record.ts, + WarehouseHistoryRecord::AuditResult(record) => record.occurred_at, + WarehouseHistoryRecord::JanitorEvent(record) => record.occurred_at, + WarehouseHistoryRecord::Migration(record) => record.created_at, + WarehouseHistoryRecord::OldNameView(record) => record.created_at, + WarehouseHistoryRecord::Microbatch(record) => record.created_at, + }; + if since.is_none_or(|since| observed >= since) { + records.push(record); + } + next_after = Some(cursor); + } + if read_count < limit { + next_after = None; + } + Ok(WarehouseHistoryPage { + records, + next_after, + }) +} + +fn decode_record( + py: Python<'_>, + (request, row, qualified): (&WarehouseHistoryRead, &Bound<'_, PyAny>, &str), + (columns, timestamp): (&[&str], &str), +) -> PyResult { + let (location, record) = match request { + WarehouseHistoryRead::Catalog(_) | WarehouseHistoryRead::Statistics(_) => { + return Err(PyValueError::new_err( + "catalog does not use state-table row decoding", + )); + } + WarehouseHistoryRead::SourceFreshness(read) => (&read.location, row_record(row)?), + WarehouseHistoryRead::Fingerprints(read) => ( + &read.location, + row_fingerprint( + &record_class(py, RecordId::Fingerprint)?, + &py.import("datetime")? + .getattr("datetime")? + .getattr("fromisoformat")?, + row, + qualified, + )?, + ), + WarehouseHistoryRead::AuditResults(read) | WarehouseHistoryRead::Migrations(read) => { + return decode_event_record(&read.location, row, (columns, &[timestamp])); + } + WarehouseHistoryRead::JanitorEvents(read) => { + return decode_event_record( + &read.location, + row, + (columns, &[timestamp, "archived_at"]), + ); + } + WarehouseHistoryRead::OldNameViews(read) => { + return decode_event_record(&read.location, row, (columns, &[timestamp, "expires_at"])); + } + WarehouseHistoryRead::Microbatches(read) => { + return decode_event_record( + &read.location, + row, + ( + columns, + &[ + timestamp, + "origin_run_started_at", + "execution_run_started_at", + "completed_at", + "observed_at", + ], + ), + ); + } + }; + let mut value = json!({ + "state_database": location.database, + "state_schema": location.schema, + "event_id": row.get_item(columns.len())?.extract::>()?, + }); + for name in columns { + let name = match *name { + "definition_b64" => "definition", + "metadata_json_b64" => "metadata_json", + name => name, + }; + let item = field(&record, name)?; + value[name] = if item.is_none() { + Value::Null + } else if name == timestamp { + json!( + presumed_utc_datetime(&item)? + .call_method0("isoformat")? + .extract::()? + ) + } else { + json!(item.extract::()?) + }; + } + if matches!(request, WarehouseHistoryRead::SourceFreshness(_)) { + serde_json::from_value(value).map(WarehouseHistoryRecord::SourceFreshness) + } else { + serde_json::from_value(value).map(WarehouseHistoryRecord::Fingerprint) + } + .map_err(|error| PyValueError::new_err(error.to_string())) +} + +fn decode_event_record( + location: &WarehouseStateLocation, + row: &Bound<'_, PyAny>, + (columns, timestamps): (&[&str], &[&str]), +) -> PyResult { + let mut value = json!({"state_database": location.database, "state_schema": location.schema}); + for (index, name) in columns.iter().enumerate() { + let item = row.get_item(index)?; + value[*name] = if timestamps.contains(name) && !item.is_none() { + let item = if item.is_instance_of::() { + item.py() + .import("datetime")? + .getattr("datetime")? + .call_method1("fromisoformat", (&item,))? + } else { + item + }; + json!( + presumed_utc_datetime(&item)? + .call_method0("isoformat")? + .extract::()? + ) + } else { + plain_value(&item)? + }; + } + serde_json::from_value(value).map_err(|error| PyValueError::new_err(error.to_string())) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_statistics.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_statistics.rs new file mode 100644 index 0000000000..3114d4e852 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_statistics.rs @@ -0,0 +1,61 @@ +//! Metadata estimates never execute exact counts or imply source-data currency. + +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use sqlbuild_orchestration::warehouse_metadata::models::{ + StatisticsRead, WarehouseHistoryPage, WarehouseHistoryRecord, WarehouseStatisticsRecord, +}; + +use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::records::fields::field; + +pub(crate) fn read_statistics( + py: Python<'_>, + (adapter, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + request: &StatisticsRead, +) -> PyResult { + if request.names.len() > MAX_HISTORY_RECORDS + || request.names.iter().any(|name| name.trim().is_empty()) + { + return Err(PyValueError::new_err( + "statistics sampling needs at most 100 exact relation names", + )); + } + let mut records: Vec = Vec::new(); + for name in &request.names { + let estimate = call_with( + &adapter.getattr("estimate_relation_row_count")?, + &[ + ("connection", connection.clone()), + ( + "database", + request.location.database.clone().into_bound_py_any(py)?, + ), + ( + "schema", + request.location.schema.clone().into_bound_py_any(py)?, + ), + ("name", name.into_bound_py_any(py)?), + ("metadata_only", true.into_bound_py_any(py)?), + ], + )?; + records.push(WarehouseHistoryRecord::Statistics( + WarehouseStatisticsRecord { + state_database: request.location.database.clone(), + state_schema: request.location.schema.clone(), + name: name.clone(), + observed_at: chrono::Utc::now(), + row_count: field(&estimate, "row_count")?.extract()?, + detail: field(&estimate, "detail")?.extract()?, + byte_count: field(&estimate, "byte_count")?.extract()?, + byte_count_basis: field(&estimate, "byte_count_basis")?.extract()?, + }, + )); + } + Ok(WarehouseHistoryPage { + records, + next_after: request.next_after.map(|id| vec![Some(id.to_string())]), + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/interruptible_join.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_threads/interruptible_join.rs index 2c03b0b48e..29aecc94b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/interruptible_join.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_threads/interruptible_join.rs @@ -5,6 +5,8 @@ use std::time::Duration; use pyo3::prelude::{PyErr, Python}; +use crate::bindings::_helpers::native_host::interrupt::check_signals; + /// How often a join checks for pending signals such as Ctrl+C. const SIGNAL_POLL_INTERVAL: Duration = Duration::from_millis(5); @@ -21,7 +23,7 @@ pub(crate) fn join_interruptibly( interrupt: impl FnOnce(), ) -> Joined { while !thread.is_finished() { - if let Err(signal) = py.check_signals() { + if let Err(signal) = check_signals(py) { interrupt(); let _ = py.detach(|| thread.join()); return Joined::Interrupted(signal); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs index bf4b2868d7..cc372a504b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs @@ -11,8 +11,8 @@ use sqlbuild_analysis::planner_identity::constants::{ }; use sqlbuild_analysis::planner_identity::main::expected_graph_identity_hashes::expected_graph_identity_hashes; use sqlbuild_analysis::planner_identity::main::graph_write_identity_hashes::graph_write_identity_hashes; -use sqlbuild_analysis::planner_identity::main::model_local_identity_hash::model_local_identity_hash; use sqlbuild_analysis::planner_identity::models::{GraphKey, IdentityGraph, IdentityNode}; +use sqlbuild_core::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs index f4d132463e..bc63fdbe52 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs @@ -11,8 +11,8 @@ use sqlbuild_analysis::planner_identity::constants::{ VERSION_IDENTITY_CONFIG_KEYS, }; use sqlbuild_analysis::planner_identity::main::function_fingerprint_sql::function_fingerprint_sql; -use sqlbuild_analysis::planner_identity::main::stable_identity_hash::stable_identity_hash; use sqlbuild_analysis::planner_identity::models::FunctionFingerprintFields; +use sqlbuild_core::fingerprint_identity::main::stable_identity_hash::stable_identity_hash; use crate::bindings::_helpers::boundary::python_calls::{imported, keywords, within}; use crate::bindings::_helpers::compiled_views::function_reads; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index 85714984a5..43b2fafcfa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -36,6 +36,7 @@ use sqlbuild_recall::writer::models::{ DeliverySummary, RecallSession, SessionConfig, SessionStore, SetupOutcome, }; +use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::output_capture::records::invocation_context; use crate::bindings::_helpers::recall::clickhouse_store::{ @@ -56,6 +57,8 @@ use crate::bindings::_helpers::records::fields::field as native_field; /// What the invocation records: its session, when enabled, and the resolved target and adapter. #[derive(Clone, Debug, Default)] pub(crate) struct InvocationRecall { + /// The verified worker remains owned until Recall has drained and its claim can be released. + pub(crate) worker: Arc>>>, pub(crate) session: Option>, pub(crate) target: Option, pub(crate) adapter: Option, @@ -409,35 +412,38 @@ fn quarantined_notice(path: &Path) -> Option { fn started_session( py: Python<'_>, (args, identity, facts): (&Bound<'_, PyAny>, &Bound<'_, PyAny>, &CommandFacts), - (settings, project_dir): (&RecallSettings, &Path), + (settings, project_dir, hosted): (&RecallSettings, &Path, Option), ) -> PyResult>> { - let store: SessionStore = match &settings.store { - StoreLocation::Sqlite { path } => { - notify(py, quarantined_notice(path)); - if let Some(reason) = store_location_problem(path) { - notify(py, permission_notice((settings, project_dir), &reason)); + let store: SessionStore = match hosted { + Some(store) => store, + None => match &settings.store { + StoreLocation::Sqlite { path } => { + notify(py, quarantined_notice(path)); + if let Some(reason) = store_location_problem(path) { + notify(py, permission_notice((settings, project_dir), &reason)); + return Ok(None); + } + SessionStore::Sqlite(path.clone()) + } + StoreLocation::ClickHouse { + config, database, .. + } => SessionStore::Hosted(Arc::new(ClickHouseStoreOpener { + location: ClickHouseLocation::new(config, database, settings.retention_days), + retention_notice: notice_due, + project_dir: project_dir.to_path_buf(), + information: Mutex::default(), + })), + StoreLocation::Unsupported { + connection, + adapter, + } => { + notify( + py, + unsupported_notice((settings, project_dir), (connection, adapter)), + ); return Ok(None); } - SessionStore::Sqlite(path.clone()) - } - StoreLocation::ClickHouse { - config, database, .. - } => SessionStore::Hosted(Arc::new(ClickHouseStoreOpener { - location: ClickHouseLocation::new(config, database, settings.retention_days), - retention_notice: notice_due, - project_dir: project_dir.to_path_buf(), - information: Mutex::default(), - })), - StoreLocation::Unsupported { - connection, - adapter, - } => { - notify( - py, - unsupported_notice((settings, project_dir), (connection, adapter)), - ); - return Ok(None); - } + }, }; let Some(invocation_id) = text_value(&native_field(identity, "invocation_id")?)? else { return Ok(None); @@ -462,13 +468,44 @@ fn started_session( max_output_bytes: settings.max_output_bytes, heartbeat_interval: heartbeat_interval(), setup_budget: Duration::ZERO, - maintenance: true, + maintenance: !worker_invocation(args, facts)?, defer_context: true, audit_evidence: settings.audit_evidence, }); Ok(Some(Arc::new(session))) } +fn worker_invocation(args: &Bound<'_, PyAny>, facts: &CommandFacts) -> PyResult { + Ok(facts.command.as_deref() == Some("location") + && attribute(args, "location_command")? + .extract::>()? + .as_deref() + == Some("execute")) +} + +fn worker_store(args: &Bound<'_, PyAny>) -> PyResult { + let identity = ( + attribute(args, "location_run_id")? + .extract::()? + .parse() + .map_err(|_| PyValueError::new_err("run ID must be a UUID"))?, + attribute(args, "location_claim_id")? + .extract::()? + .parse() + .map_err(|_| PyValueError::new_err("claim ID must be a UUID"))?, + ); + let bearer = std::env::var("SQLBUILD_RUN_TOKEN") + .map_err(|_| PyValueError::new_err("execute requires SQLBUILD_RUN_TOKEN"))?; + let url = attribute(args, "location_server_url")?.extract::()?; + args.py().detach(|| { + sqlbuild_orchestration::worker::main::connect_worker::connect_worker( + &url, &bearer, identity, + ) + .map(|client| client.recall_store()) + .map_err(|error| PyValueError::new_err(error.to_string())) + }) +} + /// Begin recording when Recall is enabled, without waiting on the store. Only the write connection is resolved; a write connection without a read connection is kept for the command to fail with, and other unresolvable settings record nothing. pub(crate) fn start_invocation_recall( py: Python<'_>, @@ -482,7 +519,7 @@ pub(crate) fn start_invocation_recall( selected_target: text_value(&args.getattr("target")?)?, purpose: StorePurpose::Record, }; - let settings: RecallSettings = match resolve_recall_settings(&request) { + let mut settings: RecallSettings = match resolve_recall_settings(&request) { Ok(settings) => settings, Err(error) if error.invalid_configuration => { return Ok(InvocationRecall { @@ -492,14 +529,25 @@ pub(crate) fn start_invocation_recall( } Err(_unreachable_store) => return Ok(InvocationRecall::default()), }; + let hosted = if worker_invocation(args, facts)? { + settings.enabled = true; + Some(worker_store(args)?) + } else { + None + }; let session: Option> = if settings.enabled && settings.project_name.is_some() { - started_session(py, (args, identity, facts), (&settings, &project_dir))? + started_session( + py, + (args, identity, facts), + (&settings, &project_dir, hosted), + )? } else { None }; let audit_evidence: AuditEvidence = settings.audit_evidence; Ok(InvocationRecall { + worker: Arc::default(), session, target: settings.target.clone(), adapter: settings.project_adapter.clone(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index d8e3df098d..3fdd402cc8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -982,6 +982,7 @@ const CLI_ENTRYPOINT_HANDLERS: RecordSpec = RecordSpec { FieldSpec::new("run_refactor", FieldDefault::None), FieldSpec::new("run_recall", FieldDefault::None), FieldSpec::new("run_review", FieldDefault::None), + FieldSpec::new("run_location", FieldDefault::None), ], }; const COMMAND_WAREHOUSE_SCOPE: RecordSpec = RecordSpec { @@ -1366,6 +1367,8 @@ const SOURCE_FRESHNESS_RENDERERS: RecordSpec = RecordSpec { FieldSpec::required("render_insert_records_sql"), FieldSpec::new("render_create_table_sql", FieldDefault::None), FieldSpec::new("render_create_index_sqls", FieldDefault::None), + FieldSpec::new("read_columns", FieldDefault::None), + FieldSpec::new("render_add_columns", FieldDefault::None), ], }; const SOURCE_FRESHNESS_OBSERVATION: RecordSpec = RecordSpec { @@ -1560,6 +1563,7 @@ const CLI_COMMAND: EnumSpec = EnumSpec { ("DIFF", "diff"), ("REVIEW", "review"), ("DEBUG", "debug"), + ("LOCATION", "location"), ("LINEAGE", "lineage"), ("QUERY", "query"), ("COST", "cost"), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs index b3f87cb241..67a12b28ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs @@ -12,6 +12,7 @@ use sqlbuild_rules::configuration::errors::RulesConfigLoadError; use sqlbuild_rules::configuration::main::load_rules_config::load_rules_config as load_config; use sqlbuild_rules::models::RulesConfig; +use crate::bindings::_helpers::boundary::config_values::json_value_to_python; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::new_record; @@ -64,31 +65,6 @@ fn option_scalar<'py>(py: Python<'py>, value: &Value) -> PyResult(py: Python<'py>, value: &Value) -> PyResult> { - match value { - Value::Array(items) => { - let values: Vec> = items - .iter() - .map(|item| option_item(py, item)) - .collect::>()?; - Ok(pyo3::types::PyList::new(py, values)?.into_any()) - } - Value::Object(entries) => { - let table: Bound<'py, PyDict> = PyDict::new(py); - for (key, item) in entries { - table.set_item(key, option_item(py, item)?)?; - } - Ok(table.into_any()) - } - Value::Null => Ok(py.None().into_bound(py)), - Value::Number(number) if number.as_i64().is_none() && number.as_u64().is_none() => { - number.as_f64().unwrap_or(f64::NAN).into_bound_py_any(py) - } - other => Ok(option_scalar(py, other)?.unwrap_or_else(|| py.None().into_bound(py))), - } -} - fn rule_options<'py>(py: Python<'py>, config: &RulesConfig) -> PyResult> { let result: Bound<'py, PyDict> = PyDict::new(py); for (code, options) in &config.rule_options { @@ -97,7 +73,7 @@ fn rule_options<'py>(py: Python<'py>, config: &RulesConfig) -> PyResult> = items .iter() - .map(|item| option_item(py, item)) + .map(|item| json_value_to_python(py, item)) .collect::>()?; values.set_item(name, PyTuple::new(py, tuple)?)?; } else if let Some(scalar) = option_scalar(py, value)? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs index 2a93fd2855..b02904a131 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs @@ -14,7 +14,9 @@ use sqlbuild_state::freshness::main::age_status::age_status; use sqlbuild_state::freshness::main::data_version_hash::data_version_hash; use sqlbuild_state::freshness::main::normalized_data_version::normalized_data_version; use sqlbuild_state::freshness::main::records_equivalent::records_equivalent; -use sqlbuild_state::freshness::models::{AgeStatus, ComparableRecord, DataVersion}; +use sqlbuild_state::freshness::models::{ + AgeStatus, ComparableRecord, DataVersion, FreshnessEventIdentity, +}; use sqlbuild_state::freshness::types::TimestampParser; use crate::bindings::_helpers::boundary::python_calls::imported; @@ -31,6 +33,40 @@ const TIMESTAMP_KIND: &str = "timestamp"; const INTEGER_KIND: &str = "integer"; const STRING_KIND: &str = "string"; +/// Normalize timestamp offsets and precision before computing a persisted observation identity. +pub(crate) fn record_event_id(record: &Bound<'_, PyAny>) -> PyResult { + let observed_at = presumed_utc(&native_field(record, "observed_at")?)?; + let observed_at = format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:06}+00:00", + observed_at.date.year, + observed_at.date.month, + observed_at.date.day, + observed_at.hour, + observed_at.minute, + observed_at.second, + observed_at.microsecond, + ); + Ok(sqlbuild_state::freshness::main::event_id::event_id( + &FreshnessEventIdentity { + source_name: &native_field(record, "source_name")?.extract::()?, + target_database: native_field(record, "target_database")? + .extract::>()? + .as_deref(), + target_schema: native_field(record, "target_schema")? + .extract::>()? + .as_deref(), + target_name: native_field(record, "target_name")? + .extract::>()? + .as_deref(), + run_id: &native_field(record, "run_id")?.extract::()?, + strategy: &native_field(record, "strategy")?.extract::()?, + value_kind: &native_field(record, "value_kind")?.extract::()?, + data_version_hash: &native_field(record, "data_version_hash")?.extract::()?, + observed_at: &observed_at, + }, + )) +} + fn raised(py: Python<'_>, class: NativeAttr, message: String) -> PyErr { match registry::native(py, class).and_then(|class| class.call1((message,))) { Ok(error) => PyErr::from_value(error), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs index 40a1d39bf6..9f7aca6b06 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs @@ -9,7 +9,8 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_state::freshness::constants::{ - SOURCE_FRESHNESS_COLUMN_TYPES, STRING_STATE_VALUE_TYPE, TEXT_TIMESTAMP_STATE_VALUE_TYPE, + SOURCE_FRESHNESS_COLUMN_TYPES, SOURCE_FRESHNESS_EVENT_ID, STRING_STATE_VALUE_TYPE, + TEXT_TIMESTAMP_STATE_VALUE_TYPE, }; use sqlbuild_state::state_tables::main::create_table_sql::create_table_sql; use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; @@ -20,6 +21,7 @@ use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::contracts::state_literals::render_state_sql_literal; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::state_rows::freshness::record_event_id; const MISSING_SCHEMA_MESSAGE: &str = "source freshness table requires a target schema"; /// Each state value type a state table may declare and the framework type that renders it. @@ -121,18 +123,20 @@ fn render_insert_source_freshness_records_sql<'py>( let mut rows: Vec> = Vec::new(); for record in records.try_iter()? { let record: Bound<'py, PyAny> = record?; - let row: Vec = SOURCE_FRESHNESS_COLUMN_TYPES + let mut row: Vec = SOURCE_FRESHNESS_COLUMN_TYPES .iter() .map(|(column, declared)| { render_state_sql_literal(py, &native_field(&record, column)?, declared, None) }) .collect::>()?; + row.push(format!("'{}'", record_event_id(&record)?)); rows.push(row); } - let names: Vec<&str> = SOURCE_FRESHNESS_COLUMN_TYPES + let mut names: Vec<&str> = SOURCE_FRESHNESS_COLUMN_TYPES .iter() .map(|(column, _)| *column) .collect(); + names.push(SOURCE_FRESHNESS_EVENT_ID); Ok(values_insert_sql( &table.extract::()?, &names, diff --git a/crates/sqlbuild-recall/Cargo.toml b/crates/sqlbuild-recall/Cargo.toml index 5738b05067..8b54b4542e 100644 --- a/crates/sqlbuild-recall/Cargo.toml +++ b/crates/sqlbuild-recall/Cargo.toml @@ -9,11 +9,14 @@ rust-version.workspace = true [dependencies] blake3.workspace = true +chrono.workspace = true rusqlite.workspace = true +serde.workspace = true serde_json.workspace = true sha2.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true +utoipa.workspace = true [target.'cfg(unix)'.dependencies] rustix.workspace = true diff --git a/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs b/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs index 9863904c39..a6934e5168 100644 --- a/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs +++ b/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs @@ -62,10 +62,6 @@ pub(crate) fn key_text(value: Option<&RecallValue>, column_type: ColumnType) -> Some(RecallValue::Text(text)) if column_type == ColumnType::Timestamp => { parse_timestamp(text).map_or_else(|| text.clone(), format_timestamp) } - Some(RecallValue::Text(text)) => text.clone(), - Some(RecallValue::Integer(number)) => number.to_string(), - Some(RecallValue::Real(number)) => number.to_string(), - Some(RecallValue::Blob(_)) => "".to_owned(), - Some(RecallValue::Null) | None => String::new(), + _ => value.map_or_else(String::new, RecallValue::key_text), } } diff --git a/crates/sqlbuild-recall/src/clickhouse/constants.rs b/crates/sqlbuild-recall/src/clickhouse/constants.rs index d11b50dc61..2c8f2719df 100644 --- a/crates/sqlbuild-recall/src/clickhouse/constants.rs +++ b/crates/sqlbuild-recall/src/clickhouse/constants.rs @@ -61,8 +61,6 @@ pub(crate) const MAX_DATETIME64_PRECISION: u64 = 9; pub(crate) const MICROS_DIGITS: u64 = 6; /// One decimal digit of timestamp scale. pub(crate) const DECIMAL_SCALE: i64 = 10; -/// The column the store sets when a row is written. -pub(crate) const INGESTED_AT_COLUMN: &str = "ingested_at"; pub(crate) const VARINT_PAYLOAD_BITS: u32 = 7; pub(crate) const VARINT_CONTINUE: u8 = 0x80; pub(crate) const VARINT_MASK: u64 = 0x7f; diff --git a/crates/sqlbuild-recall/src/clickhouse/main/encode_rows.rs b/crates/sqlbuild-recall/src/clickhouse/main/encode_rows.rs index a91f7a0e35..7bd512fa2a 100644 --- a/crates/sqlbuild-recall/src/clickhouse/main/encode_rows.rs +++ b/crates/sqlbuild-recall/src/clickhouse/main/encode_rows.rs @@ -1,9 +1,10 @@ //! `encode_rows`: one table's rows as a `RowBinary` insert block. use crate::clickhouse::_helpers::row_binary::{encoded, scaled}; -use crate::clickhouse::constants::{INGESTED_AT_COLUMN, MICROS_DIGITS}; +use crate::clickhouse::constants::MICROS_DIGITS; use crate::clickhouse::models::{ColumnEncoding, StoredColumn}; use crate::records::models::{RecallRow, RecallValue}; +use crate::schema::constants::INGESTED_AT_COLUMN; /// The column names and `RowBinary` block of `rows`, all of one table, written with `ingested_at`. Values for columns the table lacks are left out; columns the rows lack take their defaults. pub fn encode_rows( diff --git a/crates/sqlbuild-recall/src/records/_helpers/mod.rs b/crates/sqlbuild-recall/src/records/_helpers/mod.rs index eaf86fcc66..e04c007fa0 100644 --- a/crates/sqlbuild-recall/src/records/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/records/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod events; pub(crate) mod ids; +pub(crate) mod record_row; pub(crate) mod rows; pub(crate) mod time; diff --git a/crates/sqlbuild-recall/src/records/_helpers/record_row.rs b/crates/sqlbuild-recall/src/records/_helpers/record_row.rs new file mode 100644 index 0000000000..3a99f89e08 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/_helpers/record_row.rs @@ -0,0 +1,61 @@ +//! Decode transported record values against the physical schema, then compute their digest. + +use std::collections::BTreeMap; + +use crate::records::_helpers::rows::finish_row; +use crate::records::errors::RecordError; +use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::{ColumnType, Table}; + +/// Envelope and digest columns belong to the receiver, never the transported values. +pub(crate) fn record_row( + envelope: &RowEnvelope, + table: Table, + values: &BTreeMap, +) -> Result { + let columns = table_spec(table).columns; + let mut decoded = Vec::with_capacity(values.len()); + for (name, value) in values { + let column = columns + .iter() + .find(|column| column.name == name) + .ok_or_else(|| RecordError::new(format!("unknown Recall record column {name}")))?; + let valid = match (value, column.column_type) { + (RecallValue::Null, _) => column.nullable, + (RecallValue::Integer(0 | 1), ColumnType::Boolean) => true, + (RecallValue::Integer(_), ColumnType::Integer) => true, + (RecallValue::Real(value), ColumnType::Real) => value.is_finite(), + (RecallValue::Blob(_), ColumnType::Blob) => true, + (RecallValue::Text(_), ColumnType::Text) => true, + (RecallValue::Text(value), ColumnType::Json) => { + serde_json::from_str::(value).map_err(|error| { + RecordError::new(format!("invalid Recall JSON column {name}: {error}")) + })?; + true + } + (RecallValue::Text(value), ColumnType::Timestamp) => { + chrono::DateTime::parse_from_rfc3339(value).map_err(|error| { + RecordError::new(format!("invalid Recall timestamp column {name}: {error}")) + })?; + true + } + _ => false, + }; + if !valid { + return Err(RecordError::new(format!( + "invalid Recall record column {name}" + ))); + } + decoded.push((column.name, value.clone())); + } + for column in columns.iter().filter(|column| !column.nullable) { + if !values.contains_key(column.name) { + return Err(RecordError::new(format!( + "missing Recall record column {}", + column.name + ))); + } + } + Ok(finish_row(table, envelope, decoded)) +} diff --git a/crates/sqlbuild-recall/src/records/models.rs b/crates/sqlbuild-recall/src/records/models.rs index e9da820c87..eaab7e2c60 100644 --- a/crates/sqlbuild-recall/src/records/models.rs +++ b/crates/sqlbuild-recall/src/records/models.rs @@ -3,7 +3,8 @@ use crate::schema::models::Table; /// One stored value. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", content = "value", rename_all = "snake_case")] pub enum RecallValue { Null, Integer(i64), @@ -12,6 +13,20 @@ pub enum RecallValue { Blob(Vec), } +impl RecallValue { + /// Human-readable conflict-key text, before dialect-specific timestamp normalization. + #[must_use] + pub fn key_text(&self) -> String { + match self { + Self::Text(text) => text.clone(), + Self::Integer(number) => number.to_string(), + Self::Real(number) => number.to_string(), + Self::Blob(_) => "".to_owned(), + Self::Null => String::new(), + } + } +} + /// One row bound for a Recall table: named values, without `ingested_at`, which the store sets. #[derive(Clone, Debug, PartialEq)] pub struct RecallRow { @@ -20,6 +35,15 @@ pub struct RecallRow { } impl RecallRow { + /// Validate schema-typed values and rebuild the receiver-owned envelope and content digest. + pub fn from_values( + envelope: &RowEnvelope, + table: Table, + values: &std::collections::BTreeMap, + ) -> Result { + crate::records::_helpers::record_row::record_row(envelope, table, values) + } + /// The value of `column`, if the row has it. #[must_use] pub fn value(&self, column: &str) -> Option<&RecallValue> { @@ -48,7 +72,10 @@ pub struct RowEnvelope { } /// A process output stream. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] pub enum OutputStream { Stdout, Stderr, @@ -75,7 +102,8 @@ impl OutputStream { } /// One exact-byte output chunk. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] pub struct OutputChunk { /// Per invocation, from 0, shared by both streams. pub sequence: u64, @@ -91,7 +119,18 @@ pub struct OutputChunk { } /// Output of one stream that was not recorded because it exceeded `max_output_bytes`. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[derive( + Clone, + Copy, + Debug, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, +)] +#[serde(deny_unknown_fields)] pub struct OmittedOutput { pub bytes: u64, /// Sequences this gap accounts for, its own included. diff --git a/crates/sqlbuild-recall/src/schema/_helpers/mod.rs b/crates/sqlbuild-recall/src/schema/_helpers/mod.rs index 275383fa1c..6282151cb0 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod clickhouse_views; +pub(crate) mod relational_tables; +pub(crate) mod relational_views; pub(crate) mod rendering; -pub(crate) mod sqlite_views; pub(crate) mod version_row; diff --git a/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs b/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs new file mode 100644 index 0000000000..f0f25c8267 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs @@ -0,0 +1,48 @@ +//! One logical schema and index list for SQLite and PostgreSQL. + +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + +use crate::schema::_helpers::rendering::{all_columns, sqlite_type}; +use crate::schema::constants::EVENTS_TABLE; +use crate::schema::models::{ColumnType, TableSpec}; + +pub(crate) fn relational_table(spec: &TableSpec, postgres: bool, control: bool) -> Vec { + let mut definitions: Vec = all_columns(spec) + .iter() + .map(|column| { + let not_null = if column.nullable { "" } else { " NOT NULL" }; + let kind = if postgres { + column.column_type.postgres_type() + } else if control && column.column_type == ColumnType::Json { + "JSON" + } else { + sqlite_type(column.column_type) + }; + format!(" {} {kind}{not_null}", sqlite_identifier(column.name)) + }) + .collect(); + if control && spec.name == EVENTS_TABLE.name { + definitions.push(if postgres { + " storage_id BIGINT GENERATED ALWAYS AS IDENTITY".into() + } else { + " storage_id INTEGER PRIMARY KEY AUTOINCREMENT".into() + }); + } + let key: Vec = spec.key.iter().copied().map(sqlite_identifier).collect(); + definitions.push(format!(" UNIQUE ({})", key.join(", "))); + let mut statements = vec![format!( + "CREATE TABLE IF NOT EXISTS {} (\n{}\n)", + sqlite_identifier(spec.name), + definitions.join(",\n") + )]; + for (suffix, columns) in spec.sqlite_indexes { + let columns: Vec = columns.iter().map(|name| sqlite_identifier(name)).collect(); + statements.push(format!( + "CREATE INDEX IF NOT EXISTS {} ON {} ({})", + sqlite_identifier(&format!("{}_{suffix}", spec.name)), + sqlite_identifier(spec.name), + columns.join(", ") + )); + } + statements +} diff --git a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs similarity index 60% rename from crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs rename to crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs index 8d4e1c2f81..321bdb28a8 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs @@ -1,11 +1,15 @@ -//! The bodies of the supported views in SQLite. +//! Shared relational views emit dialect differences at each expression's definition. +use crate::schema::_helpers::rendering::{json_field, relational_cast}; use crate::schema::constants::OUTPUT_TAIL_RECORDS; -/// The current time as stored timestamps spell it. -const NOW: &str = "strftime('%Y-%m-%dT%H:%M:%fZ', 'now')"; +const EVENTS: &str = "SELECT * FROM recall_events"; +const DIAGNOSTICS: &str = "SELECT * FROM recall_diagnostics"; +const CONFLICTS: &str = "SELECT * FROM recall_conflicts"; -const STATEMENTS: &str = "WITH facts AS ( +fn statements(postgres: bool) -> String { + let available = relational_cast(postgres, ("artifact.digest IS NOT NULL", "INTEGER")); + format!("WITH facts AS ( SELECT * FROM recall_events WHERE statement_id IS NOT NULL AND event_type LIKE 'statement!_%' ESCAPE '!' @@ -41,7 +45,7 @@ terminals AS ( ), captures AS ( SELECT firsts.statement_id, ( - SELECT MAX(artifact.digest IS NOT NULL) + SELECT MAX({available}) FROM recall_artifact_refs AS ref LEFT JOIN recall_artifacts AS artifact ON artifact.digest = ref.digest WHERE ref.invocation_id = firsts.invocation_id @@ -102,9 +106,13 @@ LEFT JOIN captures ON captures.statement_id = firsts.statement_id LEFT JOIN resources ON resources.invocation_id = firsts.invocation_id AND resources.resource_id = firsts.resource_id -WHERE firsts.position = 1"; +WHERE firsts.position = 1") +} -const AUDIT_RESULTS: &str = "SELECT +fn audit_results(postgres: bool) -> String { + let field = |path, cast| json_field(postgres, ("payload", path), cast); + format!( + "SELECT invocation_id, invocation_started_at, event_id, @@ -112,40 +120,63 @@ const AUDIT_RESULTS: &str = "SELECT run_id, resource_id, resource_attempt_id, - json_extract(payload, '$.audit_name') AS audit_name, - json_extract(payload, '$.audit_definition_name') AS audit_definition_name, - CASE WHEN json_extract(payload, '$.target_name') IS NULL THEN NULL ELSE - COALESCE(json_extract(payload, '$.target_database') || '.', '') - || COALESCE(json_extract(payload, '$.target_schema') || '.', '') - || json_extract(payload, '$.target_name') + {audit_name} AS audit_name, + {audit_definition_name} AS audit_definition_name, + CASE WHEN {target_name} IS NULL THEN NULL ELSE + COALESCE({target_database} || '.', '') + || COALESCE({target_schema} || '.', '') + || {target_name} END AS target_relation, - json_extract(payload, '$.attached_target_kind') AS attached_target_kind, - json_extract(payload, '$.attached_target_name') AS attached_target_name, - json_extract(payload, '$.attached_column_name') AS \"column\", + {attached_target_kind} AS attached_target_kind, + {attached_target_name} AS attached_target_name, + {attached_column_name} AS \"column\", outcome, - json_extract(payload, '$.severity') AS severity, - json_extract(payload, '$.violation_count') AS violation_count, - json_extract(payload, '$.measured_value') AS measured_value, - json_extract(payload, '$.evaluation_mode') AS evaluation_mode, - json_extract(payload, '$.execution_error') AS execution_error, + {severity} AS severity, + {violation_count} AS violation_count, + {measured_value} AS measured_value, + {evaluation_mode} AS evaluation_mode, + {execution_error} AS execution_error, sql_digest, - json_extract(payload, '$.evidence_count') AS evidence_count, - json_extract(payload, '$.evidence_truncated') AS evidence_truncated, - json_extract(payload, '$.sample_digest') AS sample_digest + {evidence_count} AS evidence_count, + {evidence_truncated} AS evidence_truncated, + {sample_digest} AS sample_digest FROM recall_events -WHERE event_type = 'audit_completed'"; +WHERE event_type = 'audit_completed'", + audit_name = field("audit_name", ""), + audit_definition_name = field("audit_definition_name", ""), + target_name = field("target_name", ""), + target_database = field("target_database", ""), + target_schema = field("target_schema", ""), + attached_target_kind = field("attached_target_kind", ""), + attached_target_name = field("attached_target_name", ""), + attached_column_name = field("attached_column_name", ""), + severity = field("severity", ""), + violation_count = field("violation_count", "::BIGINT"), + measured_value = field("measured_value", "::DOUBLE PRECISION"), + evaluation_mode = field("evaluation_mode", ""), + execution_error = field("execution_error", ""), + evidence_count = field("evidence_count", "::BIGINT"), + evidence_truncated = field("evidence_truncated", "::BOOLEAN::INTEGER"), + sample_digest = field("sample_digest", "") + ) +} -const OUTPUT_INVOCATIONS: &str = "WITH received AS ( +fn output_invocations(postgres: bool) -> String { + let sum = |expression| relational_cast(postgres, (&format!("SUM({expression})"), "BIGINT")); + let maximum = if postgres { "GREATEST" } else { "MAX" }; + let filter = relational_cast(postgres, ("context.recall_filter_active", "INTEGER")); + let flush = relational_cast(postgres, ("delivery.flush_complete", "INTEGER")); + format!("WITH received AS ( SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at, COUNT(*) AS chunks, - SUM(byte_count) AS bytes, - SUM(CASE WHEN stream = 'stdout' THEN byte_count ELSE 0 END) AS stdout_bytes, - SUM(CASE WHEN stream = 'stderr' THEN byte_count ELSE 0 END) AS stderr_bytes, + {bytes} AS bytes, + {stdout_bytes} AS stdout_bytes, + {stderr_bytes} AS stderr_bytes, MAX(sequence) AS max_sequence, - SUM(dropped_before) AS queue_drops, - SUM(COALESCE(omitted_sequences, 1)) AS spans, - SUM(COALESCE(omitted_bytes, 0)) AS truncated_bytes + {drops} AS queue_drops, + {spans} AS spans, + {truncated} AS truncated_bytes FROM recall_output GROUP BY invocation_id ), @@ -166,15 +197,15 @@ counted AS ( COALESCE(received.stderr_bytes, 0) AS stderr_bytes, COALESCE(received.truncated_bytes, 0) AS truncated_bytes, received.max_sequence, - MAX( + {maximum}( COALESCE(received.max_sequence + 1 - received.spans, 0), COALESCE(delivery.accepted, 0) - COALESCE(received.chunks, 0) ) AS missing_sequences, - MAX(COALESCE(received.queue_drops, 0), COALESCE(delivery.dropped, 0)) AS reported_drops, + {maximum}(COALESCE(received.queue_drops, 0), COALESCE(delivery.dropped, 0)) AS reported_drops, COALESCE(delivery.filtered, 0) AS filtered, COALESCE(delivery.failed, 0) AS failed, - COALESCE(context.recall_filter_active, 0) AS filter_active, - COALESCE(delivery.flush_complete, 0) AS flush_complete, + COALESCE({filter}, 0) AS filter_active, + COALESCE({flush}, 0) AS flush_complete, delivery.invocation_id IS NOT NULL AS delivery_reported, COALESCE(delivery.delivered, 0) AS delivered FROM ids @@ -210,26 +241,21 @@ SELECT AND chunks >= delivered THEN 1 ELSE 0 END AS complete -FROM counted"; - -const OUTPUT_TRANSCRIPTS: &str = "SELECT - invocation_id, - MIN(invocation_started_at) AS invocation_started_at, - CAST(COALESCE( - group_concat(CASE WHEN stream = 'stdout' THEN data END, x'' ORDER BY sequence), x'' - ) AS BLOB) AS stdout, - CAST(COALESCE( - group_concat(CASE WHEN stream = 'stderr' THEN data END, x'' ORDER BY sequence), x'' - ) AS BLOB) AS stderr, - CAST(COALESCE(group_concat(data, x'' ORDER BY sequence), x'') AS BLOB) AS combined -FROM recall_output -GROUP BY invocation_id"; - -const EVENTS: &str = "SELECT * FROM recall_events"; - -const DIAGNOSTICS: &str = "SELECT * FROM recall_diagnostics"; +FROM counted", + bytes = sum("byte_count"), stdout_bytes = sum("CASE WHEN stream = 'stdout' THEN byte_count ELSE 0 END"), + stderr_bytes = sum("CASE WHEN stream = 'stderr' THEN byte_count ELSE 0 END"), drops = sum("dropped_before"), + spans = sum("COALESCE(omitted_sequences, 1)"), truncated = sum("COALESCE(omitted_bytes, 0)")) +} -const ARTIFACTS: &str = "SELECT +fn artifacts(postgres: bool) -> String { + let available = relational_cast(postgres, ("artifact.digest IS NOT NULL", "INTEGER")); + let text = if postgres { + "artifact.content" + } else { + "CAST(artifact.content AS TEXT)" + }; + format!( + "SELECT refs.invocation_id, refs.invocation_started_at, refs.digest, @@ -240,17 +266,61 @@ const ARTIFACTS: &str = "SELECT artifact.kind, artifact.media_type, artifact.size, - artifact.digest IS NOT NULL AS available, + {available} AS available, CASE WHEN artifact.digest IS NULL THEN 'expired_or_not_captured' END AS unavailable_reason, CASE WHEN artifact.media_type LIKE 'text/%' OR artifact.media_type = 'application/json' - THEN CAST(artifact.content AS TEXT) END AS text, + THEN {text} END AS text, artifact.content FROM recall_artifact_refs AS refs -LEFT JOIN recall_artifacts AS artifact ON artifact.digest = refs.digest"; +LEFT JOIN recall_artifacts AS artifact ON artifact.digest = refs.digest" + ) +} -const CONFLICTS: &str = "SELECT * FROM recall_conflicts"; +fn output_transcripts(postgres: bool) -> String { + let columns: Vec = [ + ("CASE WHEN stream = 'stdout' THEN data END", "stdout"), + ("CASE WHEN stream = 'stderr' THEN data END", "stderr"), + ("data", "combined"), + ] + .into_iter() + .map(|(expression, alias)| { + let value = if postgres { + format!("COALESCE(string_agg({expression}, ''::BYTEA ORDER BY sequence), ''::BYTEA)") + } else { + format!( + "CAST(COALESCE(group_concat({expression}, x'' ORDER BY sequence), x'') AS BLOB)" + ) + }; + format!(" {value} AS {alias}") + }) + .collect(); + format!( + "SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at,\n{}\nFROM recall_output GROUP BY invocation_id", + columns.join(",\n") + ) +} -fn invocations() -> String { +fn invocations(postgres: bool) -> String { + let now = if postgres { + "CURRENT_TIMESTAMP" + } else { + "strftime('%Y-%m-%dT%H:%M:%fZ', 'now')" + }; + let selection_count = if postgres { + "jsonb_array_length(context.selection)" + } else { + "json_array_length(context.selection)" + }; + let valid_context = if postgres { + "context.external_context IS NOT NULL" + } else { + "json_valid(context.external_context)" + }; + let elapsed = if postgres { + "CAST(ROUND(CAST(EXTRACT(EPOCH FROM (last_heartbeat_at - started_at)) * 1000 AS NUMERIC), 3) AS DOUBLE PRECISION)" + } else { + "ROUND((unixepoch(last_heartbeat_at, 'subsec') - unixepoch(started_at, 'subsec')) * 1000.0, 3)" + }; format!( "WITH ids AS ( SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at @@ -259,7 +329,7 @@ fn invocations() -> String { UNION ALL SELECT invocation_id, invocation_started_at FROM recall_heartbeats UNION ALL SELECT invocation_id, invocation_started_at FROM recall_events WHERE event_type IN ('invocation_started', 'invocation_completed', 'invocation_failed') - ) + ) AS source_rows GROUP BY invocation_id ), started AS ( @@ -270,9 +340,9 @@ started AS ( ), terminals AS ( SELECT invocation_id, event_type, occurred_at, exit_code, duration_ms, error_type, error_code, - json_extract(payload, '$.error_message') AS error_message, - json_extract(payload, '$.signal_number') AS signal_number, - json_extract(payload, '$.cost_status') AS cost_status, + {error_message} AS error_message, + {signal_number} AS signal_number, + {cost_status} AS cost_status, ROW_NUMBER() OVER ( PARTITION BY invocation_id ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC @@ -297,7 +367,7 @@ projected AS ( AND terminals.signal_number IS NOT NULL THEN 'cancelled' WHEN terminals.event_type = 'invocation_failed' THEN 'failed' WHEN beats.stale_at IS NULL THEN 'unknown' - WHEN beats.stale_at >= {NOW} THEN 'running' + WHEN beats.stale_at >= {now} THEN 'running' ELSE 'abandoned' END AS status, terminals.event_type AS terminal_event, @@ -317,7 +387,7 @@ projected AS ( context.adapter, context.project_name, context.selection, - COALESCE(context.selection_count, json_array_length(context.selection)) + COALESCE(context.selection_count, {selection_count}) AS selection_count, context.selection_truncated, context.selector_files, @@ -328,17 +398,17 @@ projected AS ( context.host_token, context.pid, context.external_context, - CASE WHEN json_valid(context.external_context) - THEN json_extract(context.external_context, '$.integration.name') END + CASE WHEN {valid_context} + THEN {integration} END AS integration, - CASE WHEN json_valid(context.external_context) - THEN json_extract(context.external_context, '$.integration.job_name') END + CASE WHEN {valid_context} + THEN {integration_job} END AS integration_job, - CASE WHEN json_valid(context.external_context) - THEN json_extract(context.external_context, '$.integration.run_id') END + CASE WHEN {valid_context} + THEN {integration_run_id} END AS integration_run_id, - CASE WHEN json_valid(context.external_context) - THEN json_extract(context.external_context, '$.integration.retry_number') END + CASE WHEN {valid_context} + THEN {integration_retry} END AS integration_retry, context.recall_filter_active, beats.last_heartbeat_at @@ -360,9 +430,7 @@ SELECT END AS ended_at, CASE WHEN terminal_event IS NOT NULL THEN terminal_duration_ms - WHEN status = 'abandoned' THEN ROUND( - (unixepoch(last_heartbeat_at, 'subsec') - unixepoch(started_at, 'subsec')) * 1000.0, 3 - ) + WHEN status = 'abandoned' THEN {elapsed} END AS duration_ms, CASE WHEN terminal_event IS NOT NULL THEN 0 @@ -398,34 +466,62 @@ SELECT integration_retry, recall_filter_active, last_heartbeat_at -FROM projected" +FROM projected", + error_message = json_field(postgres, ("payload", "error_message"), ""), + signal_number = json_field(postgres, ("payload", "signal_number"), "::BIGINT"), + cost_status = json_field(postgres, ("payload", "cost_status"), ""), + integration = json_field( + postgres, + ("context.external_context", "integration.name"), + "" + ), + integration_job = json_field( + postgres, + ("context.external_context", "integration.job_name"), + "" + ), + integration_run_id = json_field( + postgres, + ("context.external_context", "integration.run_id"), + "" + ), + integration_retry = json_field( + postgres, + ("context.external_context", "integration.retry_number"), + "::BIGINT" + ) ) } -fn output_tail() -> String { +fn output_tail(postgres: bool) -> String { + let text = if postgres { + "data" + } else { + "CAST(data AS TEXT)" + }; format!( "SELECT invocation_id, invocation_started_at, sequence, stream, occurred_at, byte_count, - CAST(data AS TEXT) AS text + {text} AS text FROM ( SELECT *, ROW_NUMBER() OVER (PARTITION BY invocation_id ORDER BY sequence DESC) AS position FROM recall_output -) +) AS source_rows WHERE position <= {OUTPUT_TAIL_RECORDS}" ) } -/// Each supported view's name and SQLite body, in creation order. -pub(crate) fn sqlite_view_bodies() -> Vec<(&'static str, String)> { +/// Each supported view's name and relational body, in creation order. +pub(crate) fn relational_view_bodies(postgres: bool) -> Vec<(&'static str, String)> { vec![ - ("invocations", invocations()), - ("statements", STATEMENTS.to_owned()), - ("audit_results", AUDIT_RESULTS.to_owned()), - ("output_invocations", OUTPUT_INVOCATIONS.to_owned()), - ("output_tail", output_tail()), - ("output_transcripts", OUTPUT_TRANSCRIPTS.to_owned()), + ("invocations", invocations(postgres)), + ("statements", statements(postgres)), + ("audit_results", audit_results(postgres)), + ("output_invocations", output_invocations(postgres)), + ("output_tail", output_tail(postgres)), + ("output_transcripts", output_transcripts(postgres)), ("events", EVENTS.to_owned()), ("diagnostics", DIAGNOSTICS.to_owned()), - ("artifacts", ARTIFACTS.to_owned()), + ("artifacts", artifacts(postgres)), ("conflicts", CONFLICTS.to_owned()), ] } diff --git a/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs b/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs index 658aef267d..f7355f0442 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs @@ -36,11 +36,6 @@ pub(crate) fn clickhouse_type(column: &ColumnSpec) -> String { } } -/// `"name"` for SQLite. -pub(crate) fn sqlite_identifier(name: &str) -> String { - format!("\"{}\"", name.replace('"', "\"\"")) -} - /// `` `name` `` for ClickHouse. pub(crate) fn clickhouse_identifier(name: &str) -> String { format!("`{}`", name.replace('\\', "\\\\").replace('`', "\\`")) @@ -73,6 +68,22 @@ pub(crate) fn clickhouse_ttl(spec: &TableSpec, options: &DdlOptions) -> String { format!("{} + INTERVAL {days} DAY", spec.expiry_column) } +pub(crate) fn json_field(postgres: bool, (column, path): (&str, &str), cast: &str) -> String { + if postgres { + format!("({column} #>> '{{{}}}'){cast}", path.replace('.', ",")) + } else { + format!("json_extract({column}, '$.{path}')") + } +} + +pub(crate) fn relational_cast(postgres: bool, (expression, kind): (&str, &str)) -> String { + if postgres { + format!("CAST({expression} AS {kind})") + } else { + expression.into() + } +} + /// The engine clause, replicated with one shard and every replica on a cluster. pub(crate) fn clickhouse_engine(spec: &TableSpec, options: &DdlOptions) -> String { let version: &str = spec.clickhouse_version.unwrap_or_default(); diff --git a/crates/sqlbuild-recall/src/schema/constants.rs b/crates/sqlbuild-recall/src/schema/constants.rs index 48e5a69a36..05c9c15d98 100644 --- a/crates/sqlbuild-recall/src/schema/constants.rs +++ b/crates/sqlbuild-recall/src/schema/constants.rs @@ -17,13 +17,16 @@ pub const DDL_SCRIPT_INVOCATION: &str = "ddl-script"; pub const META_MAINTENANCE_COMPONENT: &str = "maintenance"; pub const META_MAINTENANCE_CLAIM_KEY: &str = "claim"; +/// Assigned by storage rather than accepted from a producer's row values. +pub const INGESTED_AT_COLUMN: &str = "ingested_at"; + /// Columns every Recall row carries, after the table's own columns. pub const COMMON_COLUMNS: [ColumnSpec; 6] = [ ColumnSpec::required("invocation_id", ColumnType::Text), ColumnSpec::required("invocation_started_at", ColumnType::Timestamp), ColumnSpec::required("schema_version", ColumnType::Integer), ColumnSpec::required("producer_version", ColumnType::Text), - ColumnSpec::required("ingested_at", ColumnType::Timestamp), + ColumnSpec::required(INGESTED_AT_COLUMN, ColumnType::Timestamp), ColumnSpec::required("content_hash", ColumnType::Text), ]; /// The artifact columns its content digest covers: what it is, not who saw it when. diff --git a/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs b/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs index cea0815488..dc69488373 100644 --- a/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs +++ b/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs @@ -1,8 +1,9 @@ //! `add_column_statement`: the additive upgrade of one table by one column. +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::schema::_helpers::rendering::{ - clickhouse_identifier, clickhouse_table, clickhouse_type, on_cluster, sqlite_identifier, - sqlite_type, + clickhouse_identifier, clickhouse_table, clickhouse_type, on_cluster, sqlite_type, }; use crate::schema::models::{ColumnSpec, DdlOptions, Dialect, TableSpec}; diff --git a/crates/sqlbuild-recall/src/schema/main/control_table_statements.rs b/crates/sqlbuild-recall/src/schema/main/control_table_statements.rs new file mode 100644 index 0000000000..71107c4f6e --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/control_table_statements.rs @@ -0,0 +1,13 @@ +//! Control stores preserve Recall's logical columns and add a committed-order event cursor. + +use crate::schema::_helpers::relational_tables::relational_table; +use crate::schema::constants::ALL_TABLES; +use crate::schema::main::table_spec::table_spec; + +#[must_use] +pub fn control_table_statements(postgres: bool) -> Vec { + ALL_TABLES + .iter() + .flat_map(|table| relational_table(table_spec(*table), postgres, true)) + .collect() +} diff --git a/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs b/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs new file mode 100644 index 0000000000..1645d1ab19 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs @@ -0,0 +1,35 @@ +//! Control stores expose Recall's supported views on both engines. + +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + +use crate::schema::_helpers::relational_views::relational_view_bodies; +use crate::schema::main::view_statements::view_statements; +use crate::schema::models::{DdlOptions, Dialect}; + +/// PostgreSQL validates each view at migration time; SQLite retains its existing view contract. +#[must_use] +pub fn control_view_statements(postgres: bool) -> Vec { + if postgres { + relational_view_bodies(true) + .into_iter() + .map(|(name, body)| { + format!( + "CREATE OR REPLACE VIEW {} AS\n{body}", + sqlite_identifier(name) + ) + }) + .collect() + } else { + view_statements( + Dialect::Sqlite, + &DdlOptions { + database: None, + retention_days: 30, + cluster: None, + }, + ) + .into_iter() + .map(|(_, statement)| statement) + .collect() + } +} diff --git a/crates/sqlbuild-recall/src/schema/main/mod.rs b/crates/sqlbuild-recall/src/schema/main/mod.rs index e2251bd838..58d39be9c9 100644 --- a/crates/sqlbuild-recall/src/schema/main/mod.rs +++ b/crates/sqlbuild-recall/src/schema/main/mod.rs @@ -1,5 +1,7 @@ pub mod add_column_statement; pub mod clickhouse_relation; +pub mod control_table_statements; +pub mod control_view_statements; pub mod create_database_statement; pub mod ddl_script; pub mod ddl_statements; diff --git a/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs b/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs index 1a44c7b2e4..44b5d5ee02 100644 --- a/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs +++ b/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs @@ -1,6 +1,7 @@ //! `sqlite_retention_sql`: the deletes SQLite maintenance runs, all rows of an invocation together. -use crate::schema::_helpers::rendering::sqlite_identifier; +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::schema::constants::{ INVOCATION_TABLES, META_MAINTENANCE_COMPONENT, RETENTION_CANDIDATE_TABLES, }; diff --git a/crates/sqlbuild-recall/src/schema/main/table_statements.rs b/crates/sqlbuild-recall/src/schema/main/table_statements.rs index 621f8e1f12..7829997b9b 100644 --- a/crates/sqlbuild-recall/src/schema/main/table_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/table_statements.rs @@ -1,48 +1,14 @@ //! `table_statements`: idempotent `CREATE TABLE` and index statements of every Recall table. +use crate::schema::_helpers::relational_tables::relational_table; use crate::schema::_helpers::rendering::{ all_columns, clickhouse_engine, clickhouse_identifier, clickhouse_table, clickhouse_ttl, - clickhouse_type, on_cluster, sqlite_identifier, sqlite_type, + clickhouse_type, on_cluster, }; use crate::schema::constants::ALL_TABLES; use crate::schema::main::table_spec::table_spec; use crate::schema::models::{DdlOptions, Dialect, TableSpec}; -fn sqlite_table(spec: &TableSpec) -> Vec { - let mut definitions: Vec = all_columns(spec) - .iter() - .map(|column| { - let not_null: &str = if column.nullable { "" } else { " NOT NULL" }; - format!( - " {} {}{not_null}", - sqlite_identifier(column.name), - sqlite_type(column.column_type) - ) - }) - .collect(); - let key: Vec = spec - .key - .iter() - .map(|name| sqlite_identifier(name)) - .collect(); - definitions.push(format!(" UNIQUE ({})", key.join(", "))); - let mut statements: Vec = vec![format!( - "CREATE TABLE IF NOT EXISTS {} (\n{}\n)", - sqlite_identifier(spec.name), - definitions.join(",\n") - )]; - for (suffix, columns) in spec.sqlite_indexes { - let columns: Vec = columns.iter().map(|name| sqlite_identifier(name)).collect(); - statements.push(format!( - "CREATE INDEX IF NOT EXISTS {} ON {} ({})", - sqlite_identifier(&format!("{}_{suffix}", spec.name)), - sqlite_identifier(spec.name), - columns.join(", ") - )); - } - statements -} - fn clickhouse_table_statement(spec: &TableSpec, options: &DdlOptions) -> String { let columns: Vec = all_columns(spec) .iter() @@ -76,7 +42,7 @@ pub fn table_statements(dialect: Dialect, options: &DdlOptions) -> Vec { .flat_map(|table| { let spec: &TableSpec = table_spec(*table); match dialect { - Dialect::Sqlite => sqlite_table(spec), + Dialect::Sqlite => relational_table(spec, false, false), Dialect::ClickHouse => vec![clickhouse_table_statement(spec, options)], } }) diff --git a/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs b/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs index 0b8c7430e2..913bf845df 100644 --- a/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs @@ -1,6 +1,7 @@ //! `upgrade_statements`: what an administrator runs to bring an observed store up to this build's schema. -use crate::schema::_helpers::rendering::sqlite_identifier; +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::schema::_helpers::version_row::version_row_statement; use crate::schema::constants::{ALL_TABLES, RECALL_SCHEMA_VERSION}; use crate::schema::main::add_column_statement::add_column_statement; diff --git a/crates/sqlbuild-recall/src/schema/main/view_statements.rs b/crates/sqlbuild-recall/src/schema/main/view_statements.rs index ce34504a3c..0351938b81 100644 --- a/crates/sqlbuild-recall/src/schema/main/view_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/view_statements.rs @@ -1,15 +1,17 @@ //! `view_statements`: the statements that create the supported views. +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::schema::_helpers::clickhouse_views::clickhouse_view_bodies; -use crate::schema::_helpers::rendering::{clickhouse_table, on_cluster, sqlite_identifier}; -use crate::schema::_helpers::sqlite_views::sqlite_view_bodies; +use crate::schema::_helpers::relational_views::relational_view_bodies; +use crate::schema::_helpers::rendering::{clickhouse_table, on_cluster}; use crate::schema::models::{DdlOptions, Dialect}; /// Each view's name and creation statement. SQLite views are created only when missing; the store drops them first when the schema version rises. ClickHouse views are replaced. #[must_use] pub fn view_statements(dialect: Dialect, options: &DdlOptions) -> Vec<(&'static str, String)> { match dialect { - Dialect::Sqlite => sqlite_view_bodies() + Dialect::Sqlite => relational_view_bodies(false) .into_iter() .map(|(name, body)| { ( diff --git a/crates/sqlbuild-recall/src/schema/models.rs b/crates/sqlbuild-recall/src/schema/models.rs index 022e4f0b84..e622f273fe 100644 --- a/crates/sqlbuild-recall/src/schema/models.rs +++ b/crates/sqlbuild-recall/src/schema/models.rs @@ -22,6 +22,22 @@ pub enum ColumnType { Boolean, } +impl ColumnType { + /// The PostgreSQL type used by both DDL and typed query placeholders. + #[must_use] + pub const fn postgres_type(self) -> &'static str { + match self { + Self::Text => "TEXT", + Self::Timestamp => "TIMESTAMPTZ", + Self::Json => "JSONB", + Self::Integer => "BIGINT", + Self::Boolean => "BOOLEAN", + Self::Real => "DOUBLE PRECISION", + Self::Blob => "BYTEA", + } + } +} + /// One column: its name, type and whether it may be null. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ColumnSpec { @@ -75,7 +91,20 @@ pub struct TableSpec { } /// The tables Recall writes. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] pub enum Table { Invocations, Output, diff --git a/crates/sqlbuild-recall/src/store/_helpers/mod.rs b/crates/sqlbuild-recall/src/store/_helpers/mod.rs index 3344c9a183..fab996484f 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod column_types; pub(crate) mod identity; pub(crate) mod quarantine; +pub(crate) mod relational; pub(crate) mod rows; pub(crate) mod schema; pub(crate) mod sqlite; diff --git a/crates/sqlbuild-recall/src/store/_helpers/relational.rs b/crates/sqlbuild-recall/src/store/_helpers/relational.rs new file mode 100644 index 0000000000..7122ec88b9 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/_helpers/relational.rs @@ -0,0 +1,11 @@ +//! Typed placeholders shared by native SQLite and control-plane transactions. + +use crate::schema::models::ColumnType; + +pub(crate) fn placeholder(index: usize, kind: ColumnType, postgres: bool) -> String { + if postgres { + format!("CAST(${index} AS {})", kind.postgres_type()) + } else { + format!("${index}") + } +} diff --git a/crates/sqlbuild-recall/src/store/_helpers/rows.rs b/crates/sqlbuild-recall/src/store/_helpers/rows.rs index 5fed0c3900..aa5b4d0b42 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/rows.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/rows.rs @@ -12,20 +12,14 @@ use crate::schema::models::Table; use crate::store::_helpers::schema::stored_columns; use crate::store::_helpers::sqlite::{SqlValue, checked}; use crate::store::errors::StoreError; +use crate::store::main::conflict_evidence::conflict_evidence; +use crate::store::main::relational_insert::relational_insert; +use crate::store::main::relational_lookup::relational_lookup; use crate::store::models::{ConflictEvidence, InsertOutcome}; -fn key_text(value: Option<&RecallValue>) -> String { - match value { - Some(RecallValue::Text(text)) => text.clone(), - Some(RecallValue::Integer(number)) => number.to_string(), - Some(RecallValue::Real(number)) => number.to_string(), - Some(RecallValue::Blob(_)) => "".to_owned(), - Some(RecallValue::Null) | None => String::new(), - } -} - fn text_of(row: &RecallRow, column: &str) -> String { - key_text(row.value(column)) + row.value(column) + .map_or_else(String::new, RecallValue::key_text) } /// Insert one row's values for the columns its table has; `Some(evidence)` when its key holds different content. @@ -34,28 +28,17 @@ fn write_row( (row, columns): (&RecallRow, &HashSet), ingested_at: &str, ) -> Result, StoreError> { - let spec = table_spec(row.table); let ingested: RecallValue = RecallValue::Text(ingested_at.to_owned()); - let stored: Vec<&(&str, RecallValue)> = row + let insert = relational_insert((row, columns), &ingested, false); + let parameters: Vec> = insert .values .iter() - .filter(|(name, _)| columns.contains(*name)) + .map(|(value, _)| SqlValue(value)) .collect(); - let mut names: Vec<&str> = stored.iter().map(|(name, _)| *name).collect(); - names.push("ingested_at"); - let placeholders: Vec = (1..=names.len()).map(|index| format!("?{index}")).collect(); - let sql = format!( - "INSERT INTO \"{}\" ({}) VALUES ({}) ON CONFLICT DO NOTHING", - spec.name, - names.join(", "), - placeholders.join(", ") - ); - let parameters: Vec> = stored - .iter() - .map(|(_, value)| SqlValue(value)) - .chain(std::iter::once(SqlValue(&ingested))) - .collect(); - let mut statement = checked(connection.prepare_cached(&sql), "prepare Recall insert")?; + let mut statement = checked( + connection.prepare_cached(&insert.sql), + "prepare Recall insert", + )?; let changed: usize = checked( statement.execute(rusqlite::params_from_iter(parameters.iter())), "write Recall rows", @@ -63,24 +46,16 @@ fn write_row( if changed > 0 { return Ok(None); } - let condition: Vec = spec - .key - .iter() - .enumerate() - .map(|(index, name)| format!("{name} = ?{}", index + 1)) - .collect(); - let lookup = format!( - "SELECT content_hash FROM \"{}\" WHERE {}", - spec.name, - condition.join(" AND ") - ); - let null = RecallValue::Null; - let key_values: Vec> = spec - .key + let lookup = relational_lookup(row, false, false); + let key_values: Vec> = lookup + .values .iter() - .map(|name| SqlValue(row.value(name).unwrap_or(&null))) + .map(|(value, _)| SqlValue(value)) .collect(); - let mut statement = checked(connection.prepare_cached(&lookup), "prepare Recall lookup")?; + let mut statement = checked( + connection.prepare_cached(&lookup.sql), + "prepare Recall lookup", + )?; let existing: Option = checked( statement .query_row(rusqlite::params_from_iter(key_values.iter()), |found| { @@ -89,21 +64,7 @@ fn write_row( .optional(), "read stored Recall row", )?; - let rejected: &str = row.content_hash().unwrap_or_default(); - match existing { - Some(existing) if existing != rejected => Ok(Some(ConflictEvidence { - table: row.table, - record_key: spec - .key - .iter() - .map(|name| key_text(row.value(name))) - .collect::>() - .join(":"), - existing_sha256: existing, - rejected_sha256: rejected.to_owned(), - })), - _ => Ok(None), - } + Ok(existing.and_then(|existing| conflict_evidence(row, &existing))) } /// Write `rows` on the current transaction; conflicts become `recall_conflicts` rows. diff --git a/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs b/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs new file mode 100644 index 0000000000..a4c04a92e6 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs @@ -0,0 +1,25 @@ +//! Duplicate rows preserve the first content and expose the rejected digest. + +use crate::records::models::{RecallRow, RecallValue}; +use crate::schema::main::table_spec::table_spec; +use crate::store::models::ConflictEvidence; + +/// Evidence only when a stored row's content differs, independent of the storage driver. +#[must_use] +pub fn conflict_evidence(row: &RecallRow, existing: &str) -> Option { + let rejected = row.content_hash().unwrap_or_default(); + (existing != rejected).then(|| ConflictEvidence { + table: row.table, + record_key: table_spec(row.table) + .key + .iter() + .map(|name| { + row.value(name) + .map_or_else(String::new, RecallValue::key_text) + }) + .collect::>() + .join(":"), + existing_sha256: existing.to_owned(), + rejected_sha256: rejected.to_owned(), + }) +} diff --git a/crates/sqlbuild-recall/src/store/main/mod.rs b/crates/sqlbuild-recall/src/store/main/mod.rs index 69ccb1cf7e..78f9b89028 100644 --- a/crates/sqlbuild-recall/src/store/main/mod.rs +++ b/crates/sqlbuild-recall/src/store/main/mod.rs @@ -1,5 +1,6 @@ pub mod checkpoint_sqlite_store; pub mod chosen_invocation; +pub mod conflict_evidence; pub mod describe_store; pub mod ensure_sqlite_schema; pub mod insert_rows; @@ -10,5 +11,7 @@ pub mod open_sqlite_store; pub mod quarantine_store; pub mod query_store; pub mod read_output_transcript; +pub mod relational_insert; +pub mod relational_lookup; pub mod resolve_invocation; pub mod store_location_problem; diff --git a/crates/sqlbuild-recall/src/store/main/relational_insert.rs b/crates/sqlbuild-recall/src/store/main/relational_insert.rs new file mode 100644 index 0000000000..70aa341299 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/relational_insert.rs @@ -0,0 +1,50 @@ +//! One insert contract for standalone SQLite Recall and transactional control stores. + +use std::collections::HashSet; + +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + +use crate::records::models::{RecallRow, RecallValue}; +use crate::schema::constants::INGESTED_AT_COLUMN; +use crate::schema::main::table_columns::table_columns; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::ColumnType; +use crate::store::_helpers::relational::placeholder; +use crate::store::models::RecallStatement; + +/// Ignore fields absent from an older additive schema; ingestion time is assigned by the store. +#[must_use] +pub fn relational_insert<'a>( + (row, columns): (&'a RecallRow, &HashSet), + ingested_at: &'a RecallValue, + postgres: bool, +) -> RecallStatement<'a> { + let metadata = table_columns(row.table); + let mut names: Vec = Vec::new(); + let mut values: Vec<(&RecallValue, ColumnType)> = Vec::new(); + for (name, value) in &row.values { + if let Some(column) = metadata.iter().find(|column| column.name == *name) + && columns.contains(*name) + && *name != INGESTED_AT_COLUMN + { + names.push(sqlite_identifier(name)); + values.push((value, column.column_type)); + } + } + names.push(INGESTED_AT_COLUMN.into()); + values.push((ingested_at, ColumnType::Timestamp)); + let placeholders: Vec = values + .iter() + .enumerate() + .map(|(index, (_, kind))| placeholder(index + 1, *kind, postgres)) + .collect(); + RecallStatement { + sql: format!( + "INSERT INTO {} ({}) VALUES ({}) ON CONFLICT DO NOTHING", + sqlite_identifier(table_spec(row.table).name), + names.join(", "), + placeholders.join(", ") + ), + values, + } +} diff --git a/crates/sqlbuild-recall/src/store/main/relational_lookup.rs b/crates/sqlbuild-recall/src/store/main/relational_lookup.rs new file mode 100644 index 0000000000..14d3936da0 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/relational_lookup.rs @@ -0,0 +1,43 @@ +//! Duplicate detection uses the same stable keys as physical unique constraints. + +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + +use crate::records::models::{RecallRow, RecallValue}; +use crate::schema::main::table_columns::table_columns; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::{ColumnType, Table}; +use crate::store::_helpers::relational::placeholder; +use crate::store::models::RecallStatement; + +/// Control event lookups return their durable cursor too; standalone stores have no cursor column. +#[must_use] +pub fn relational_lookup(row: &RecallRow, postgres: bool, control: bool) -> RecallStatement<'_> { + let spec = table_spec(row.table); + let keys = spec.key; + let metadata = table_columns(row.table); + let mut values: Vec<(&RecallValue, ColumnType)> = Vec::new(); + let mut conditions: Vec = Vec::new(); + for column in metadata.iter().filter(|column| keys.contains(&column.name)) { + let key = column.name; + let kind = column.column_type; + values.push((row.value(key).unwrap_or(&RecallValue::Null), kind)); + conditions.push(format!( + "{} = {}", + sqlite_identifier(key), + placeholder(values.len(), kind, postgres) + )); + } + let cursor = if control && row.table == Table::Events { + ", storage_id" + } else { + "" + }; + RecallStatement { + sql: format!( + "SELECT content_hash{cursor} FROM {} WHERE {}", + sqlite_identifier(spec.name), + conditions.join(" AND ") + ), + values, + } +} diff --git a/crates/sqlbuild-recall/src/store/models.rs b/crates/sqlbuild-recall/src/store/models.rs index a92d4ead30..fd53a8b5c7 100644 --- a/crates/sqlbuild-recall/src/store/models.rs +++ b/crates/sqlbuild-recall/src/store/models.rs @@ -4,7 +4,14 @@ use std::path::PathBuf; use std::time::{Duration, SystemTime}; use crate::records::models::{RecallValue, RowEnvelope}; -use crate::schema::models::Table; +use crate::schema::models::{ColumnType, Table}; + +/// A relational insert or lookup and its borrowed, typed parameters. +#[derive(Clone, Debug, PartialEq)] +pub struct RecallStatement<'a> { + pub sql: String, + pub values: Vec<(&'a RecallValue, ColumnType)>, +} /// One open store file. Only one thread uses a store. #[derive(Debug)] @@ -47,7 +54,7 @@ pub enum QuarantineOutcome { } /// A row whose key was already stored with different content; the first row is kept. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] pub struct ConflictEvidence { pub table: Table, pub record_key: String, @@ -56,7 +63,9 @@ pub struct ConflictEvidence { } /// What one insert batch did, per table. -#[derive(Clone, Debug, Default, PartialEq, Eq)] +#[derive( + Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize, utoipa::ToSchema, +)] pub struct InsertOutcome { /// Rows written or already present with equal content, by table. pub delivered: Vec<(Table, u64)>, diff --git a/crates/sqlbuild-state/src/freshness/constants.rs b/crates/sqlbuild-state/src/freshness/constants.rs index 8083997c83..184ed9e83b 100644 --- a/crates/sqlbuild-state/src/freshness/constants.rs +++ b/crates/sqlbuild-state/src/freshness/constants.rs @@ -41,7 +41,7 @@ pub const REQUIRED_STATE_COLUMNS: [&str; 6] = [ pub const STRING_STATE_VALUE_TYPE: &str = "string"; /// State value type of a timestamp stored as ISO text. pub const TEXT_TIMESTAMP_STATE_VALUE_TYPE: &str = "text_timestamp"; -/// Source freshness state table columns in table order, with the state value type each declares. +/// Legacy latest-reader columns; the nullable event ID is intentionally not required by reads. pub const SOURCE_FRESHNESS_COLUMN_TYPES: [(&str, &str); 10] = [ ("source_name", STRING_STATE_VALUE_TYPE), ("target_database", STRING_STATE_VALUE_TYPE), @@ -54,5 +54,8 @@ pub const SOURCE_FRESHNESS_COLUMN_TYPES: [(&str, &str); 10] = [ ("data_version_hash", STRING_STATE_VALUE_TYPE), ("observed_at", TEXT_TIMESTAMP_STATE_VALUE_TYPE), ]; +pub const SOURCE_FRESHNESS_EVENT_ID: &str = "event_id"; +pub const OBSERVED_AT_COLUMN: &str = "observed_at"; +pub const SQLITE_ADAPTER: &str = "sqlite"; pub(crate) const PHYSICAL_TABLE_SOURCE_FRAGMENT: &str = "requires a physical table source"; pub(crate) const INCOMPLETE_CONFIGURATION_FRAGMENT: &str = "incomplete"; diff --git a/crates/sqlbuild-state/src/freshness/main/event_id.rs b/crates/sqlbuild-state/src/freshness/main/event_id.rs new file mode 100644 index 0000000000..72bdeab470 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/event_id.rs @@ -0,0 +1,22 @@ +//! Observation identity excludes mutable payload and future storage columns. + +use sha2::{Digest, Sha256}; + +use crate::freshness::models::FreshnessEventIdentity; + +/// `observed_at` is UTC ISO text with six fractional digits and a `+00:00` offset. +pub fn event_id(row: &FreshnessEventIdentity<'_>) -> String { + let identity = serde_json::json!([ + "sqlbuild.source-freshness.event.v1", + row.source_name, + row.target_database, + row.target_schema, + row.target_name, + row.run_id, + row.observed_at, + row.strategy, + row.value_kind, + row.data_version_hash, + ]); + format!("{:x}", Sha256::digest(identity.to_string().as_bytes())) +} diff --git a/crates/sqlbuild-state/src/freshness/main/mod.rs b/crates/sqlbuild-state/src/freshness/main/mod.rs index 9ec47ed663..2e5f2dbe8f 100644 --- a/crates/sqlbuild-state/src/freshness/main/mod.rs +++ b/crates/sqlbuild-state/src/freshness/main/mod.rs @@ -1,6 +1,7 @@ pub mod age_status; pub mod configuration_error; pub mod data_version_hash; +pub mod event_id; pub mod normalized_data_version; pub mod read_latest_sql; pub mod records_equivalent; diff --git a/crates/sqlbuild-state/src/freshness/models.rs b/crates/sqlbuild-state/src/freshness/models.rs index dbe1341a71..9f7cbfb958 100644 --- a/crates/sqlbuild-state/src/freshness/models.rs +++ b/crates/sqlbuild-state/src/freshness/models.rs @@ -2,6 +2,21 @@ use sqlbuild_cursor::algebra::models::LocalDateTime; +/// Stable identity fields shared by warehouse writers and legacy history ingestion. +#[derive(Debug)] +pub struct FreshnessEventIdentity<'a> { + pub source_name: &'a str, + pub target_database: Option<&'a str>, + pub target_schema: Option<&'a str>, + pub target_name: Option<&'a str>, + pub run_id: &'a str, + pub strategy: &'a str, + pub value_kind: &'a str, + pub data_version_hash: &'a str, + /// UTC ISO text with six fractional digits and a `+00:00` offset. + pub observed_at: &'a str, +} + /// An observed data version of one value kind. #[derive(Clone, Debug, PartialEq, Eq)] pub enum DataVersion { diff --git a/crates/sqlbuild-state/src/state_tables/constants.rs b/crates/sqlbuild-state/src/state_tables/constants.rs new file mode 100644 index 0000000000..8bf6191df2 --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/constants.rs @@ -0,0 +1,3 @@ +//! Adapter decisions shared by append-only state history reads. + +pub(crate) const SQLSERVER_ADAPTER: &str = "sqlserver"; diff --git a/crates/sqlbuild-state/src/state_tables/main/mod.rs b/crates/sqlbuild-state/src/state_tables/main/mod.rs index 712abe33fc..6b46ba4e98 100644 --- a/crates/sqlbuild-state/src/state_tables/main/mod.rs +++ b/crates/sqlbuild-state/src/state_tables/main/mod.rs @@ -1,2 +1,3 @@ pub mod create_table_sql; +pub mod read_history_sql; pub mod values_insert_sql; diff --git a/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs b/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs new file mode 100644 index 0000000000..8a415a7352 --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs @@ -0,0 +1,120 @@ +//! Keyset pages preserve distinct observations without rereading or ranking all preceding rows. + +use crate::state_tables::constants::SQLSERVER_ADAPTER; +use crate::state_tables::models::HistoryRead; + +/// Cursor values are already rendered by the adapter; `None` sorts before every SQL value. +pub fn read_history_sql( + qualified_name: &str, + after: Option<&[Option]>, + read: HistoryRead<'_>, +) -> Result { + let HistoryRead { + columns, + key_columns, + timestamp_column, + observed_since, + adapter, + has_event_id, + limit, + } = read; + let mut projection = columns.join(", "); + projection.push_str(if has_event_id { + ", event_id" + } else { + ", NULL AS event_id" + }); + let mut columns = key_columns.unwrap_or(columns).to_vec(); + columns.push(if has_event_id { "event_id" } else { "NULL" }); + let ordered = columns + .iter() + .map(|column| { + if *column == timestamp_column { + Ok((*column).to_owned()) + } else { + binary(column, adapter) + } + }) + .collect::, _>>()?; + let mut greater: Vec = Vec::new(); + let mut equal: Vec = Vec::new(); + if let Some(after) = after { + for ((column, ordered), value) in columns.iter().zip(&ordered).zip(after) { + let literal = value + .as_ref() + .map(|value| { + if *column == timestamp_column { + Ok(value.clone()) + } else { + binary(value, adapter) + } + }) + .transpose()?; + let comparison = literal.as_ref().map_or_else( + || format!("{column} IS NOT NULL"), + |literal| format!("{ordered} > {literal}"), + ); + let prefix = if equal.is_empty() { + String::new() + } else { + format!("{} AND ", equal.join(" AND ")) + }; + greater.push(format!("({prefix}{comparison})")); + equal.push(literal.as_ref().map_or_else( + || format!("{column} IS NULL"), + |literal| format!("{ordered} = {literal}"), + )); + } + } + let mut filters: Vec = Vec::new(); + if !greater.is_empty() { + filters.push(format!("({})", greater.join(" OR "))); + } + if let Some(since) = observed_since { + filters.push(format!("{timestamp_column} >= {since}")); + } + let filter = if filters.is_empty() { + String::new() + } else { + format!(" WHERE {}", filters.join(" AND ")) + }; + let order: Vec = columns + .iter() + .zip(&ordered) + .map(|(column, ordered)| format!("CASE WHEN {column} IS NULL THEN 0 ELSE 1 END, {ordered}")) + .collect(); + let top = adapter == SQLSERVER_ADAPTER; + let head = if top { + format!("TOP {limit} ") + } else { + String::new() + }; + let tail = if top { + String::new() + } else { + format!(" LIMIT {limit}") + }; + Ok(format!( + "SELECT {head}{projection} FROM {qualified_name}{filter} ORDER BY {}{tail}", + order.join(", ") + )) +} + +/// Session and column collations must not merge case, accents or trailing spaces at page boundaries. +fn binary(value: &str, adapter: &str) -> Result { + Ok(match adapter { + "duckdb" | "motherduck" => format!("encode({value})"), + "sqlite" => format!("CAST({value} AS BLOB)"), + "postgres" => format!("convert_to({value}, 'UTF8')"), + "snowflake" => format!("TO_BINARY({value}, 'UTF-8')"), + "bigquery" => format!("CAST({value} AS BYTES)"), + "databricks" => format!("CAST({value} AS BINARY)"), + "sqlserver" => format!("CONVERT(VARBINARY(MAX), CONVERT(NVARCHAR(MAX), {value}))"), + "clickhouse" => value.to_owned(), + _ => { + return Err(format!( + "warehouse history does not support adapter '{adapter}'" + )); + } + }) +} diff --git a/crates/sqlbuild-state/src/state_tables/mod.rs b/crates/sqlbuild-state/src/state_tables/mod.rs index 1a70a48981..042268f15e 100644 --- a/crates/sqlbuild-state/src/state_tables/mod.rs +++ b/crates/sqlbuild-state/src/state_tables/mod.rs @@ -1,3 +1,4 @@ +pub mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-state/src/state_tables/models.rs b/crates/sqlbuild-state/src/state_tables/models.rs index 7a343e134d..5c41f277a2 100644 --- a/crates/sqlbuild-state/src/state_tables/models.rs +++ b/crates/sqlbuild-state/src/state_tables/models.rs @@ -7,3 +7,16 @@ pub struct TableColumn { pub rendered_type: String, pub required: bool, } + +/// A bounded legacy-compatible history projection, with its optional source event ID. +#[derive(Clone, Copy, Debug)] +pub struct HistoryRead<'a> { + pub columns: &'a [&'a str], + /// Stable identity and timestamp keys omit numeric or evidence payloads from ordering. + pub key_columns: Option<&'a [&'a str]>, + pub timestamp_column: &'a str, + pub observed_since: Option<&'a str>, + pub adapter: &'a str, + pub has_event_id: bool, + pub limit: usize, +} diff --git a/deploy/Dockerfile b/deploy/Dockerfile new file mode 100644 index 0000000000..5ecf300a8a --- /dev/null +++ b/deploy/Dockerfile @@ -0,0 +1,28 @@ +# Build wheels from this checkout before building these images; see deploy/README.md. +ARG PYTHON_VERSION=3.12 +FROM python:${PYTHON_VERSION}-slim-bookworm AS runtime +COPY dist/*.whl /wheels/ +RUN python -m pip install --no-cache-dir /wheels/*.whl && rm -rf /wheels \ + && useradd --uid 10001 --create-home sqlbuild \ + && mkdir -p /project/target /project/logs /warehouse \ + && chown sqlbuild:sqlbuild /project /project/target /project/logs /warehouse +ENV PYTHONUNBUFFERED=1 +WORKDIR /project +USER 10001:10001 +ENTRYPOINT ["sqb"] + +FROM runtime AS location +ARG SQLBUILD_GIT_REVISION +RUN test -n "$SQLBUILD_GIT_REVISION" +ENV SQLBUILD_GIT_REVISION=${SQLBUILD_GIT_REVISION} +COPY --chown=10001:10001 deploy/project/ /project/ + +# The control plane has no Python interpreter, project, adapter, or warehouse credentials. +FROM debian:bookworm-slim AS server +RUN apt-get update && apt-get install --no-install-recommends -y ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --uid 10001 --create-home sqlbuild +COPY --from=runtime /usr/local/bin/sqb /usr/local/bin/sqb +USER 10001:10001 +ENTRYPOINT ["sqb"] +CMD ["server", "--host", "0.0.0.0", "--port", "8000", "--launchers", "/etc/sqlbuild/launchers.json"] diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000000..eb85bcf12b --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,203 @@ +# Run the reference Docker Compose stack + +This Linux reference runs PostgreSQL, the native SQLBuild server/scheduler, and one outbound +Python code location. Workers use Docker only. The server image contains no Python or project. +The location image includes the example orders project and its Python dependencies. + +Use Docker Engine with a Unix socket and Docker Compose 2.23.1 or newer, Python 3.12, `uv`, +a Rust toolchain, and `jq`. Build and test from a trusted checkout. Docker socket access is +host-administrator authority: only the server gets that socket and control-store credentials. +Locations and workers get warehouse access, but no control-store credentials or inbound ports. +The default server port binds to loopback, not the public network. + +## Build the images + +Build matching wheels from this checkout before building the images: + +```sh +uv sync --locked --all-extras +uv run --no-sync maturin build --release --compatibility linux --out dist +uv run --no-sync maturin build --manifest-path crates/sqlbuild-cli/Cargo.toml --release --compatibility linux --out dist +cp deploy/compose.env.example .env +``` + +Set the ignored `.env` values. Use a unique PostgreSQL password and put its URL-encoded value +in `SQLBUILD_CONTROL_DATABASE_URL`, for example +`postgresql://sqlbuild:@postgres:5432/sqlbuild`. +Set `DOCKER_GID` to `stat -c %g /var/run/docker.sock` and `SQLBUILD_GIT_REVISION` to +`git rev-parse HEAD`. Keep only the matching SQLBuild and CLI wheels in `dist/`; their Python +version and architecture must match the image's `PYTHON_VERSION` build argument and Docker host. +The default image uses Python 3.12. Do not put credentials in build arguments or project files. +The Docker build context includes only wheels and the example project. + +```sh +docker compose build +docker compose up -d --wait postgres +``` + +## Register and start the code location + +Run these commands on the trusted control-store owner's host. They use the server image's +direct-store administration path, not an HTTP or person-role grant: + +```sh +docker compose run --rm --no-deps server admin create-location orders +``` + +Replace `.env`'s placeholder `SQLBUILD_LOCATION_ID` with the returned `location_id`. Issue a +bounded credential, using a future RFC 3339 expiry you choose: + +```sh +docker compose run --rm --no-deps server admin issue-location-token "$SQLBUILD_LOCATION_ID" --expires-at "$EXPIRY" +``` + +Shell variables above must be set explicitly; Compose's `.env` file is not sourced by the shell. +Store the one-time `bearer` in `.env` as `SQLBUILD_LOCATION_TOKEN`, restrict the file to its owner +(`chmod 600 .env`), and start the stack: + +```sh +docker compose up -d --wait server location +docker compose logs -f server location +``` + +The location watches and compiles its baked project, publishes a successful snapshot, and checks +its warehouse connection. The example schedule runs every minute. The server resolves the +running container's actual Docker image ID at publication; no `--image-id` or Docker socket is +needed in the location. Workers recompile in that pinned image and verify environment and node +hashes before writing. The warehouse persists in its named volume. + +For your project, build the `runtime` target as a base, then extend it with `COPY --chown=10001:10001` +of your project to `/project`, install its locked dependencies, and set `SQLBUILD_GIT_REVISION`. +Set `SQLBUILD_LOCATION_IMAGE` to that project image. Give location UID 10001 access to its +warehouse volumes. Never mount control credentials, location-token files, or Docker sockets +into the location: workers inherit its mounts and filtered warehouse environment. + +## Redeploy without interrupting a run + +Build the new project into a new revision-tagged image. Set `SQLBUILD_LOCATION_IMAGE` to that +image and update only the location: + +```sh +docker compose up -d --no-deps location +``` + +Active workers keep their existing image, process, mounts, and independent network endpoint. +New publications affect only new runs. A failed compile does not replace the last good snapshot; +its scheduled jobs still launch from the last good image even while the new location is broken. +Keep the warehouse mounts, network, working directory, and warehouse environment compatible with +that image. Changing those deployment settings requires a coordinated maintenance window. + +Retain every image referenced by a published snapshot or run. Do not run indiscriminate image +pruning or delete those images. Inspect `sqb_snapshots.image_id` and `sqb_runs.image_id` through +the trusted control-store owner before implementing retention. Removing a required image makes +a queued or historical run unlaunchable. + +For bind-mounted source development, explicitly add the development overlay: + +```sh +docker compose -f compose.yml -f deploy/compose.dev.yml up -d +``` + +The location watcher republishes changed source. The overlay keeps `target` and `logs` in +named volumes owned by the image's runtime user, so location and worker output does not require +changing the host source tree's ownership. Use the same overlay for `down --volumes` when removing +these development caches and logs. Workers reject source drift against a pinned snapshot; the +bind mount is not an immutable deployment and does not provide old-source fallback. + +## Credentials, identity, and platform events + +Rotate a location credential with `docker compose run --rm --no-deps server admin rotate-token +TOKEN_UUID`, store its one-time replacement bearer, and recreate the location. The scope and +expiry stay unchanged, and the old credential is revoked atomically. Service-account principal +tokens are separate from location/run tokens. Person identities do not become machine bearers. + +For OIDC sign-in, mount a server-owned JSON file with `origin`, `issuer`, +`client_id`, and explicit `identities` subject-to-existing-person UUID bindings; append +`--oidc-config /etc/sqlbuild/oidc.json` to the server command. Both origin and issuer require +HTTPS. Terminate HTTPS at a trusted proxy, preserve the external Host, restrict backend access, +and register `/api/auth/oidc/callback` at the issuer. The callback verifies an explicitly bound, +enabled person and issues a one-hour browser session. With a trusted proxy, every request +authenticates from the configured socket peer and the upstream-replaced `X-SQB-User` subject; no +browser session is issued, and mutations require the configured origin. Neither mechanism +provisions people or grants roles. A session bearer stays in a Secure, HttpOnly, host-only +cookie; mutations require the returned CSRF token and matching origin. Sessions work across replicas sharing the installation, with live +person and role checks. `POST /api/auth/logout` revokes the session across replicas; password +resets and `grant-admin` revoke all of that person's browser sessions. + +The local control-store owner initializes the first installation administrator with +`sqb admin create-user NAME`, then provisions its password separately with +`sqb admin set-password PRINCIPAL_ID`. `create-user` works only while no person exists; once +anyone exists, use `grant-admin` instead. `create-principal` creates a role-free identity, not +another administrator. With password, OIDC or trusted-proxy authentication, installation administrators +use `POST /api/admin/people/{principal_id}` with a JSON action of `grant`, `revoke`, `disable`, +or `enable`, their session cookie and CSRF header. This cannot remove or disable the final active +administrator. These endpoints never authorize a machine bearer or grant non-admin defaults. + +If administrator access is lost, run `sqb admin grant-admin PRINCIPAL_ID` as the installation's local +control-store owner; servers may keep running. It enables an existing person, grants installation +admin membership, revokes that person's browser sessions and records an audit event. It never +creates a person. Provision a password with `set-password` if needed. + +An installation administrator assigns or revokes additive built-in roles through +`POST /api/admin/roles`, using `principal_id`, `role`, optional `location_id` and `target`, +and `revoke` (false for a grant). An administrator assignment must be installation-wide. +Other built-ins can be installation-wide or restricted to a location or target. Revocation +also works while a person is disabled, before they are enabled again. + +Browser sessions use `GET /api/control/locations/{location_id}/targets/{target}/runs` +for recent run metadata, `POST /api/control/runs` for published-job requests, and +`POST /api/control/triggers` for trigger overrides. Use `POST /api/control/runs/cancel` +with `run_id`, `event_id` and `reason` to request cancellation; use +`POST /api/control/runs/rerun` with `run_id`, `source_run_id`, `from_failure`, `reason` +and `priority` to repeat a terminal run. Retrying an identical request is idempotent, but +still requires live permission. Reruns keep the original snapshot, image, selection and bounds; +`from_failure` narrows selection to durable failed/skipped outcomes. Cancellation of active +work remains a request until its worker confirms shutdown; it never releases ownership early. +Viewers can read metadata but cannot run, cancel or rerun jobs; operators and developers can +control ordinary jobs and toggle triggers. Ad-hoc reruns also require developer or admin access. +Full-refresh jobs, published cursor bounds, explicit backfill windows and their reruns require +an installation administrator in these endpoints. Sensitive work can still be cancelled without +history-rewrite permission. The server defaults to `--default-role none`: a person with no +assigned role cannot use them. Set `--default-role viewer` to give every authenticated person +installation-wide metadata access, including people whose explicit roles have been revoked. +This is a live server setting, not a persisted assignment; returning it to `none` removes that +ambient access without changing explicit grants. It never permits SQL, evidence, job execution +or administration. Metadata responses do not contain SQL, sampled rows or captured output. +These person routes do not accept machine +credentials; location and worker credentials continue to use the separate machine routes. + +Set `SQLBUILD_PLATFORM_WEBHOOK_URL` in `.env` before creating the server to receive the native +platform stream. Leave it unset to disable delivery. Receivers must deduplicate by `event_id` +or the matching `Idempotency-Key` header: delivery is at least once and retries after failure. +The native watchdog runs independently of location Python and scheduler leadership. It observes +heartbeats, overdue ticks, missing leadership, and locations silent for over 90 seconds; its +30-second scan and delivery polling add detection latency. Each alert is sent once when its +condition starts, not on every scan. Monitor platform heartbeats from +outside this stack too: a stopped server cannot report its own outage. + +## Back up and upgrade + +Back up PostgreSQL with `pg_dump` using your normal protected database backup tooling. Also +back up warehouse data, the restricted `.env`/identity configuration, and referenced project +images. Keep backup credentials and output outside the repository. Test restoration before an +upgrade. Ordinary `docker compose down` preserves named volumes; `down --volumes` destroys them. + +Upgrade matching server/location wheels and images together, retaining referenced old project +images. The native server applies additive control-store migrations before its API reports +ready. Do not start an older server against a newer schema without a documented compatible +rollback or restored backup. Scheduler leadership resumes from durable ticks after restart; +catch-up runs each trigger's latest missed time once, not every missed time. +The Docker launcher adopts an existing claim-labeled worker rather than starting a duplicate. + +## Verify the packaged lifecycle locally + +`make test-compose` builds matching wheels and images, then tests scheduled Docker execution, +redeploy during an active run, failed-deploy fallback, scheduler catch-up, credential/mount +isolation, the development overlay's source reload and worker outputs, and the stopped-location +platform webhook against PostgreSQL. It also backs up the +populated control store, restores into an empty database, compares every public table's row count +and content digest before restarting writers, and checks that retained credentials and last-good +scheduling work after restore without duplicate ticks. This is a control-store restore check, +not an image, secret-file, or warehouse disaster-recovery test. It creates disposable containers +and volumes and removes them on exit. It needs Docker, Compose, and network access for image +and Python dependency downloads; it never deploys to an external installation. diff --git a/deploy/compose.dev.yml b/deploy/compose.dev.yml new file mode 100644 index 0000000000..78c4d83f10 --- /dev/null +++ b/deploy/compose.dev.yml @@ -0,0 +1,11 @@ +# Opt in with: docker compose -f compose.yml -f deploy/compose.dev.yml up -d +services: + location: + volumes: + - ./deploy/project:/project + - development-cache:/project/target + - development-logs:/project/logs + +volumes: + development-cache: + development-logs: diff --git a/deploy/compose.env.example b/deploy/compose.env.example new file mode 100644 index 0000000000..a667e912e0 --- /dev/null +++ b/deploy/compose.env.example @@ -0,0 +1,8 @@ +# Copy to the ignored root .env; set the password and matching URL before starting PostgreSQL. +POSTGRES_PASSWORD= +SQLBUILD_CONTROL_DATABASE_URL= +DOCKER_GID= +SQLBUILD_GIT_REVISION= +# Replace this placeholder with create-location's assigned identity before starting the location. +SQLBUILD_LOCATION_ID=00000000-0000-0000-0000-000000000001 +SQLBUILD_LOCATION_TOKEN= diff --git a/deploy/project/models/orders.sql b/deploy/project/models/orders.sql new file mode 100644 index 0000000000..3c8bb14e45 --- /dev/null +++ b/deploy/project/models/orders.sql @@ -0,0 +1,3 @@ +MODEL (description 'Example orders.', materialized table); + +SELECT 37 AS order_id diff --git a/deploy/project/orchestration/orders.py b/deploy/project/orchestration/orders.py new file mode 100644 index 0000000000..47aeb4c1f3 --- /dev/null +++ b/deploy/project/orchestration/orders.py @@ -0,0 +1,10 @@ +"""A small scheduled project baked into the reference location image.""" + +from sqlbuild.orchestration import job, schedule + +orders = job( + "orders", + select="orders", + description="Refresh orders every minute.", + schedule=schedule(cron="* * * * *", timezone="UTC"), +) diff --git a/deploy/project/sqlbuild_project.toml b/deploy/project/sqlbuild_project.toml new file mode 100644 index 0000000000..7a45950c85 --- /dev/null +++ b/deploy/project/sqlbuild_project.toml @@ -0,0 +1,8 @@ +name = "orders" +adapter = "duckdb" + +[connections.local] +database = "/warehouse/orders.duckdb" + +[targets.development] +schema = "main" diff --git a/fensu.toml b/fensu.toml index e4b8adc906..ecc2f19c8d 100644 --- a/fensu.toml +++ b/fensu.toml @@ -229,7 +229,7 @@ reason = "Native code resolves these through the sqlbuild._native_host name tabl [[targets.python.dead_code.roots]] modules = ["sqlbuild.compiler.discovery._helpers.filesystem.project_modules"] -symbols = ["import_python_root_module", "install_python_root_package", "load_materialize_function", "load_provider_module", "load_python_node_module", "load_sink_module", "module_functions", "module_provider_classes", "parameter_annotations", "provider_usages", "requires_arguments", "restore_python_root_modules", "source_line", "validate_event_exporter_signature"] +symbols = ["import_python_root_module", "install_python_root_package", "load_materialize_function", "load_orchestration_module", "load_provider_module", "load_python_node_module", "load_sink_module", "module_functions", "module_provider_classes", "parameter_annotations", "provider_usages", "requires_arguments", "restore_python_root_modules", "source_line", "validate_event_exporter_signature"] reason = "Native code resolves these through the sqlbuild._native_host name table." [[targets.python.dead_code.roots]] @@ -1530,6 +1530,11 @@ rule = "FFR601" path = "src/sqlbuild/executor/run/_helpers/materializations/microbatch.py" reason = "Microbatch execution is one ordered failure-sensitive lifecycle; splitting reconciliation, DML, completion publication, and aggregate gates would hide the required phase ordering across module boundaries." +[[targets.python.rule_exceptions]] +rule = "FFT104" +path = "tests/integration/src/sqlbuild/compiler/compiled_project/publication/login/helpers.py" +reason = "The disposable HTTPS gateway and OIDC provider dispatch real HTTP protocol messages and authentication transports; assertions in the parametrized installed journey remain branch-free." + [targets.rust] analyzer = "rust" roots = ["crates"] @@ -1564,6 +1569,30 @@ rule = "FPRSS201" path = "crates/sqlbuild-python/src/bindings/models.rs" reason = "CompactAnalysisJob holds a prepared analysis closure, which cannot implement Debug; other models derive it." +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-orchestration/src/credentials/models.rs" +symbols = ["IssuedToken", "IssuedSession", "StartedLogin"] +reason = "Issued credentials and handoffs implement Debug manually to redact bearer, CSRF and approval secrets; deriving Debug would expose them." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-orchestration/src/api/models.rs" +symbols = ["PasswordLoginRequest", "PasswordChangeRequest", "PasswordSessionResponse", "ExternalIdentityResponse", "CliLoginApprovalRequest", "CliLoginClaimRequest", "CliLoginSessionResponse"] +reason = "Browser transport models implement Debug manually to redact passwords and CSRF secrets; other models derive Debug." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-cli/src/person_login/models.rs" +symbols = ["PersonCredential", "LoginFlow"] +reason = "Person credential and handoff models implement Debug manually to redact bearer, CSRF and approval secrets." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-orchestration/src/webhooks/models.rs" +symbols = ["Webhook"] +reason = "Webhook implements Debug manually to redact its endpoint, which may contain delivery credentials; other models derive Debug." + [[targets.rust.rule_exceptions]] rule = "FPRSS201" path = "crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs" @@ -1612,6 +1641,7 @@ crate_names = [ "sqlbuild-discovery", "sqlbuild-host", "sqlbuild-model-config", + "sqlbuild-orchestration", "sqlbuild-project", "sqlbuild-pysource", "sqlbuild-python", @@ -1638,6 +1668,14 @@ reason = "Resource asset projections intentionally mirror their public JSON sche paths = ["crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs"] reason = "Command document projections intentionally mirror the public envelope while preserving command-specific summaries: seeds require a plan and loads report skipped_count." +[[dupes.allowlist]] +paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerprints.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/freshness.rs"] +reason = "Typed observation field projections intentionally mirror provenance checks while preserving different required identity fields and timestamp names. Bounds, rejection rules and reconciliation completion share the ingestion owner." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_janitor_events.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_microbatches.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_migrations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_old_name_views.rs"] +reason = "Typed event projections retain distinct required identities and indexed payload columns. Event persistence, fencing, atomic retries and reconciliation share the events ingestion owner; no per-table lifecycle is copied." + [[targets.python.rule_exceptions]] rule = "XSB051" path = "src/sqlbuild/executor/migrations/_helpers/old_name_views.py" @@ -1663,17 +1701,17 @@ edge_paths = [ "crates/sqlbuild-python/src/bindings/_helpers/records/registry.rs", ] pending_budgets = [ - "crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/nullability_rules.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=20", + "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs=1", @@ -1688,43 +1726,43 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=22", + "crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=21", "crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/build_command/support.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/build_command/support.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress.rs=12", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_reporters.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_text.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress.rs=11", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_reporters.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_text.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs=23", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs=21", + "crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs=20", "crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=27", "crates/sqlbuild-python/src/bindings/_helpers/cli/launch_state.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=21", + "crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/cli/surrogates.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/adapters.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs=9", @@ -1733,19 +1771,19 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/storage_policies.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs=42", + "crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs=0", + "crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs=41", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs=30", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs=45", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs=40", + "crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs=38", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/startup_inputs.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/surrogates.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs=23", + "crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs=22", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs=39", "crates/sqlbuild-python/src/bindings/_helpers/compile_context/compile_timings.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compile_context/diagnostic_collector.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/compile_context/input_reads.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/compile_context/diagnostic_collector.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/compile_context/input_reads.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs=20", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs=3", @@ -1773,22 +1811,22 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs=0", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/records.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/conversion.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs=8", @@ -1805,8 +1843,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs=23", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs=9", @@ -1828,11 +1866,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs=27", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/event_stream.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/orchestration.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs=19", @@ -1844,8 +1882,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_coercion.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_connection.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_load.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs=10", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs=8", @@ -1854,7 +1892,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs=15", + "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/facts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/output.rs=4", @@ -1870,7 +1908,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/discovery/project_config.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs=6", @@ -1885,16 +1923,16 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=29", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=28", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs=17", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=12", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/functions.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgres.rs=28", - "crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs=28", + "crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgres.rs=27", + "crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs=27", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs=16", @@ -1919,7 +1957,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs=5", @@ -1930,7 +1968,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_calls.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_external.rs=4", @@ -1958,7 +1996,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs=28", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_facts.rs=3", @@ -1968,11 +2006,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=17", - "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=19", + "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/module_digests.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs=27", - "crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs=1", @@ -1983,16 +2021,16 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/errors.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/errors.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=0", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=33", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=31", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=12", @@ -2012,10 +2050,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs=1", @@ -2035,7 +2073,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs=4", @@ -2049,9 +2087,9 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs=2", @@ -2084,11 +2122,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs=15", + "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs=1", @@ -2096,12 +2134,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/document.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/document.rs=0", + "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs=1", @@ -2113,8 +2151,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/cursors.rs=1", @@ -2125,8 +2163,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/migrations.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/rule_shapes.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/records/models.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/records/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs=26", "crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_edits.rs=1", @@ -2141,24 +2179,24 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs=24", "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_classes.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs=17", + "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs=20", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs=19", + "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_values.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/result_output/resource_ids.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs=14", + "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/review_command/command.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs=1", @@ -2166,11 +2204,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_observation.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_observation.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs=21", - "crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/rules/fact_encoder.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/rules/fix_facts.rs=8", @@ -2185,7 +2223,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs=1", @@ -2197,26 +2235,26 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_cache.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs=33", + "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs=32", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs=23", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_tolerant.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs=16", + "crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs=6", @@ -2224,7 +2262,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=101", + "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=100", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs=4", @@ -2237,8 +2275,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs=4", @@ -2251,7 +2289,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/type_system/equality.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs=8", - "crates/sqlbuild-python/src/bindings/models.rs=7", + "crates/sqlbuild-python/src/bindings/models.rs=0", ] [targets.rust.rule_options.XSB073] @@ -2630,12 +2668,16 @@ module_layers = [ "lineage/lineage_command=commands", "lineage/lineage_facts=project", "load=commands", + "location_command=commands", "macro_bridge=front", "manifest=compile", "microbatches=plan", "model_config=front", "native_host=infra", "native_host/model_lookup=project", + "native_host/orchestration_sensors=commands", + "native_host/orchestration_triggers=project", + "native_host/warehouse_metadata=commands", "native_threads=infra", "output_capture=output", "plan_command=commands", @@ -2730,3 +2772,8 @@ rule = "FFS120" path = "src/sqlbuild/executor/scheduling/classes/recovering_connection_pool.py" symbols = ["RecoveringConnectionPool.put"] reason = "The override keeps queue.Queue.put's positional protocol, which every pool consumer calls." + +[[targets.python.rule_exceptions]] +rule = "FFS201" +path = "src/sqlbuild/orchestration_declarations/models/sensor_context.py" +reason = "Only SensorContext lives here: invocation-local ctx.cursor assignment proposes progress, committed solely by native fenced completion. The package regression guards this declaration-only scope; class-symbol exceptions are unsupported." diff --git a/pyproject.toml b/pyproject.toml index bef1f027d9..74496f9d4c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ dev = [ "pytest-cov>=7.0.0", "pytest-xdist>=3.8.0", "ruff>=0.12.0", - "fensu>=0.28.0", + "fensu>=0.28.3", "testcontainers>=4.8.1", "ty>=0.0.1a19", ] diff --git a/python-boundary.toml b/python-boundary.toml index ee2346f85d..5829d6322f 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -12,6 +12,31 @@ # data instead of projecting those public containers and immediately reading them back. remaining_lines = 0 +[[stays]] +path = "orchestration.py" +rule = 3 +note = "Public orchestration facade: re-exports only." + +[[stays]] +path = "orchestration_declarations/models/authoring.py" +rule = 3 +note = "Public job and schedule authoring records only; selection resolution, validation and scheduling stay native." + +[[stays]] +path = "orchestration_declarations/models/sensor_context.py" +rule = 3 +note = "Only the mutable SensorContext authoring declaration required by ctx.cursor assignment; no Python runtime allowance. Native fenced completion validates and commits proposed progress." + +[[stays]] +path = "orchestration_declarations/types.py" +rule = 3 +note = "Public PlatformEventKind, RunEventKind and NodeEventKind enums only, placed in types.py to satisfy enum ownership while retaining a zero implementation budget; no Python event processing allowance." + +[[stays]] +path = "orchestration_declarations/main/compile_snapshot.py" +rule = 3 +note = "Public declaration compiler: one-line delegation to build_orchestration_snapshot; no Python discovery, validation or snapshot logic." + [[stays]] path = "executor" rule = 1 diff --git a/scripts/delta_compile_series/README.md b/scripts/delta_compile_series/README.md new file mode 100644 index 0000000000..fed5784efb --- /dev/null +++ b/scripts/delta_compile_series/README.md @@ -0,0 +1,48 @@ +# Private comparison diagnostics + +CLI comparison tests keep bounded evidence only when the existing delta/full comparator +reports a mismatch. Acceptance, output normalization, progress routing and cache policy are +unchanged. Successful comparisons create no evidence. Local evidence directories use mode +0700 and files use 0600. Streams and named environment credentials are redacted, but the +remaining project content must still be treated as private. + +The CLI CI shard enables encrypted retention with `SQLBUILD_COMPARISON_ENCRYPTED_DIR` and +`SQLBUILD_COMPARISON_RECIPIENT`. The latter names the reviewed public recipient certificate; +no private key is available to the workflow. OpenSSL CMS uses AES-256-GCM and RSA-OAEP with +SHA-256 to encrypt each ZIP bundle. Only completed `failure.cms` files are uploaded, only +when the shard fails, with seven-day retention. Plaintext temporary evidence and partial +encryption output are never upload paths or printed by capture. + +Each comparison series retains at most four failures of 4 MiB each, with individual streams +capped at 512 KiB. One shard reserves at most twelve ciphertext slots, each capped at +4 MiB + 64 KiB (48.75 MiB total). Slots are reserved atomically across test workers. +Truncation, original lengths/digests and capture errors are explicit in the evidence. +Encryption failure cannot replace the original comparison result and has no plaintext +upload fallback. A missing encrypted artifact is not evidence that a failed comparison +was captured successfully. + +## Recover an encrypted failure + +Download the encrypted GitHub artifact to a private machine. Obtain the matching private +key through the private secret store, never through GitHub, repository content or logs. +Decrypt to a temporary file, and rename it only after authentication succeeds: + +```sh +umask 077 +if openssl cms -decrypt -binary -inform DER -in failure.cms \ + -inkey /private/path/recipient-key.pem -out evidence.zip.partial; then + mv evidence.zip.partial evidence.zip +else + rm -f evidence.zip.partial + exit 1 +fi +``` + +Do not inspect or extract partial decryption output: OpenSSL may write unauthenticated +bytes before rejecting tampering. Wrong keys, truncated ciphertext and modified GCM tags +must fail. Extract only into a private directory; do not publish raw evidence. + +The tracked certificate is public encryption material, not a trust root for authentication. +To rotate it, securely store and verify the replacement private key before replacing the +certificate. Retain prior private keys at least until every corresponding artifact has +expired. Neither key rotation nor diagnostics grants product access or changes CLI semantics. diff --git a/scripts/delta_compile_series/_helpers/encryption.py b/scripts/delta_compile_series/_helpers/encryption.py new file mode 100644 index 0000000000..b7abc7b414 --- /dev/null +++ b/scripts/delta_compile_series/_helpers/encryption.py @@ -0,0 +1,85 @@ +"""Authenticated, bounded CI retention; only complete ciphertext is uploadable.""" + +from __future__ import annotations + +import io +import os +import subprocess +import zipfile +from pathlib import Path + +from scripts.delta_compile_series.constants import ( + MAX_ENCRYPTED_BYTES, + MAX_ENCRYPTED_FAILURES, + MAX_EVIDENCE_FILES, + MAX_STEP_BYTES, +) + + +def encrypt_comparison_failure(directory: Path) -> None: + """Retain at most twelve encrypted failures when CI explicitly enables retention.""" + + configured: str | None = os.environ.get("SQLBUILD_COMPARISON_ENCRYPTED_DIR") + if not configured: + return + destination: Path = Path(configured) + destination.mkdir(parents=True, exist_ok=True, mode=0o700) + destination.chmod(0o700) + for index in range(MAX_ENCRYPTED_FAILURES): + slot: Path = destination / f"failure-{index:02d}" + try: + slot.mkdir(mode=0o700) + except FileExistsError: + continue + break + else: + return + pending: Path = slot / "encrypted.partial" + try: + files: list[Path] = list(directory.iterdir()) + if len(files) > MAX_EVIDENCE_FILES or any( + path.is_symlink() or not path.is_file() for path in files + ): + raise OSError("Invalid comparison evidence files") + archive: io.BytesIO = io.BytesIO() + remaining: int = MAX_STEP_BYTES + with zipfile.ZipFile(archive, mode="w", compression=zipfile.ZIP_DEFLATED) as bundle: + for path in sorted(files): + with path.open("rb") as stream: + data: bytes = stream.read(remaining + 1) + remaining -= len(data) + if remaining < 0: + raise OSError("Comparison evidence exceeds retention bound") + bundle.writestr(path.name, data) + certificate: str = os.environ.get("SQLBUILD_COMPARISON_RECIPIENT", "") + with pending.open("xb") as output: + pending.chmod(0o600) + result: subprocess.CompletedProcess[bytes] = subprocess.run( + [ + "openssl", + "cms", + "-encrypt", + "-binary", + "-aes-256-gcm", + "-outform", + "DER", + "-recip", + certificate, + "-keyopt", + "rsa_padding_mode:oaep", + "-keyopt", + "rsa_oaep_md:sha256", + ], + input=archive.getvalue(), + stdout=output, + stderr=subprocess.PIPE, + check=False, + timeout=30, + ) + if result.returncode != 0 or not 0 < pending.stat().st_size <= MAX_ENCRYPTED_BYTES: + raise OSError("Comparison evidence encryption failed") + pending.replace(slot / "failure.cms") + except subprocess.TimeoutExpired as error: + raise OSError("Comparison evidence encryption timed out") from error + finally: + pending.unlink(missing_ok=True) diff --git a/scripts/delta_compile_series/_helpers/evidence.py b/scripts/delta_compile_series/_helpers/evidence.py new file mode 100644 index 0000000000..852f68ae53 --- /dev/null +++ b/scripts/delta_compile_series/_helpers/evidence.py @@ -0,0 +1,216 @@ +"""Bounded, private evidence for a failed delta/full comparison, never CLI output.""" + +from __future__ import annotations + +import hashlib +import json +import os +import time +from itertools import islice +from pathlib import Path +from typing import Any, cast + +from scripts.delta_compile_series._helpers.encryption import encrypt_comparison_failure +from scripts.delta_compile_series._helpers.outcomes import _seen +from scripts.delta_compile_series.constants import ( + COMPARISON_MANIFEST_FILE, + MAX_CACHE_PATHS, + MAX_ENTRIES, + MAX_FAILURES, + MAX_FILE_BYTES, + MAX_READ_BYTES, + MAX_STEP_BYTES, +) +from scripts.delta_compile_series.models import DeltaCompileOutcome +from sqlbuild._native import redact_diagnostic_text, redact_diagnostic_value + +_MANIFEST_RESERVE: int = 64 * 1024 + + +def _file_state(*, path: Path, root: Path, limit: int = MAX_FILE_BYTES) -> dict[str, Any]: + """Stat and bounded content digest, without following paths outside the project.""" + + if not path.resolve().is_relative_to(root.resolve()): + return {"outside_project": True} + if not path.is_file(): + return {"missing": True} + status: os.stat_result = path.stat() + with path.open("rb") as stream: + data: bytes = stream.read(limit) + return { + "size": status.st_size, + "mtime_ns": status.st_mtime_ns, + "ctime_ns": status.st_ctime_ns, + "sha256": hashlib.sha256(data).hexdigest(), + "hashed_bytes": len(data), + "digest_truncated": len(data) != status.st_size, + } + + +def cache_state(project: Path) -> dict[str, Any]: + """Report evidence-collection errors without changing the comparison or CLI outcome.""" + + try: + return _cache_state(project) + except (OSError, ValueError) as error: + return {"capture_error": str(error)} + + +def _cache_state(project: Path) -> dict[str, Any]: + """Snapshot bounded SQL-test records and physical reuse evidence, not inferred identities.""" + + snapshots: dict[str, Any] = {} + remaining: int = MAX_READ_BYTES + paths: list[Path] = list( + islice((project / "target/cache").rglob("sql-test-artifacts.json"), MAX_CACHE_PATHS + 1) + ) + for path in sorted(paths[:MAX_CACHE_PATHS]): + snapshot: dict[str, Any] = _file_state(path=path, root=project) + snapshots[str(path.relative_to(project))] = snapshot + if snapshot.get("outside_project") or snapshot.get("missing"): + continue + with path.open("rb") as stream: + data: bytes = stream.read(MAX_FILE_BYTES + 1) + snapshot["cache_text"] = data[:MAX_FILE_BYTES].decode("utf-8", errors="replace") + try: + payload: Any = json.loads(data) if len(data) <= MAX_FILE_BYTES else None + except (ValueError, UnicodeError): + payload = None + snapshot["parseable"] = isinstance(payload, dict) + records: Any = payload.get("records") if isinstance(payload, dict) else None + if not isinstance(records, dict): + continue + snapshot["records_truncated"] = len(records) > MAX_ENTRIES + physical: dict[str, Any] = {} + for key, raw in islice(cast(dict[str, Any], records).items(), MAX_ENTRIES): + if not isinstance(raw, dict): + continue + record: dict[str, Any] = cast(dict[str, Any], raw) + relative_path: Any = record.get("relative_path") + if not isinstance(relative_path, str): + continue + state: dict[str, Any] = _file_state( + path=project / "target/compiled/tests" / relative_path, + root=project, + limit=min(MAX_FILE_BYTES, remaining), + ) + remaining -= state.get("hashed_bytes", 0) + state["recorded_identity"] = record.get("identity") + state["recorded_size_matches"] = state.get("size") == record.get("size") + state["recorded_mtime_matches"] = state.get("mtime_ns") == record.get("mtime_ns") + state["recorded_digest_matches"] = not state.get( + "digest_truncated", True + ) and state.get("sha256") == record.get("content_sha256") + physical[key] = state + snapshot["physical_artifacts"] = physical + return { + "captured_at_ns": time.time_ns(), + "caches": snapshots, + "cache_paths_truncated": len(paths) > MAX_CACHE_PATHS, + } + + +def _artifacts(project: Path) -> dict[str, Any]: + """Bounded per-file digests of actual compiled SQL, without copying query text.""" + + paths: list[Path] = list(islice((project / "target/compiled").rglob("*.sql"), MAX_ENTRIES + 1)) + files: dict[str, Any] = {} + remaining: int = MAX_READ_BYTES + for path in sorted(paths[:MAX_ENTRIES]): + state: dict[str, Any] = _file_state( + path=path, root=project, limit=min(MAX_FILE_BYTES, remaining) + ) + remaining -= state.get("hashed_bytes", 0) + files[str(path.relative_to(project))] = state + return { + "files": files, + "truncated": len(paths) > MAX_ENTRIES, + } + + +def _saved(*, destination: Path, name: str, text: str) -> dict[str, Any]: + """Redact named credentials, preserve stream order, and explicitly mark any byte limit.""" + + original: bytes = text.encode("utf-8", errors="surrogateescape") + data: bytes = redact_diagnostic_text(text).encode("utf-8", errors="surrogateescape") + used: int = sum(path.stat().st_size for path in destination.iterdir()) + limit: int = ( + min(_MANIFEST_RESERVE, max(0, MAX_STEP_BYTES - used)) + if name == COMPARISON_MANIFEST_FILE + else min(MAX_FILE_BYTES, max(0, MAX_STEP_BYTES - _MANIFEST_RESERVE - used)) + ) + kept: bytes = data[:limit] + with (destination / name).open("xb") as stream: + os.chmod(stream.name, 0o600) + _ = stream.write(kept) + return { + "original_sha256": hashlib.sha256(original).hexdigest(), + "original_bytes": len(original), + "redacted": data != original, + "retained_bytes": len(kept), + "truncated": len(kept) != len(data), + } + + +def keep_comparison_failure( + *, + destination: Path, + seed: int, + step: str, + case: str, + project: Path, + copy: Path, + outcome: DeltaCompileOutcome, + full: DeltaCompileOutcome, + before: dict[str, Any], + after: dict[str, Any], +) -> None: + """Keep at most four 4-MiB private failures; nothing is printed or uploaded.""" + + destination.mkdir(parents=True, exist_ok=True, mode=0o700) + destination.chmod(0o700) + failures: list[Path] = list(destination.iterdir()) + if len(failures) >= MAX_FAILURES: + return + directory: Path = destination / f"failure-{len(failures):02d}" + directory.mkdir(mode=0o700) + manifest: dict[str, Any] = {"seed": seed, "step": step, "case": case, "files": {}, "runs": {}} + for label, result, root in (("delta", outcome, project), ("full", full, copy)): + normalized: tuple[str, ...] = _seen(outcome=result, project_dir=project, other_dir=root) + try: + report: Any = json.loads(normalized[1]) + except ValueError: + report = None + manifest["runs"][label] = { + "returncode": result.returncode, + "command": result.command, + "environment": [ + (name, redact_diagnostic_value(name=name, value=value)) + for name, value in result.environment + ], + "project": str(root), + "semantic_json_sha256": hashlib.sha256( + json.dumps(report, sort_keys=True).encode() + ).hexdigest() + if report is not None + else None, + } + for stream in ("stdout", "stderr"): + name = f"{label}.{stream}" + manifest["files"][name] = _saved( + destination=directory, name=name, text=getattr(result, stream) + ) + name = f"{label}.artifacts.json" + manifest["files"][name] = _saved( + destination=directory, name=name, text=json.dumps(_artifacts(root), sort_keys=True) + ) + for name, state in (("cache-before.json", before), ("cache-after.json", after)): + manifest["files"][name] = _saved( + destination=directory, name=name, text=json.dumps(state, sort_keys=True) + ) + _ = _saved( + destination=directory, + name=COMPARISON_MANIFEST_FILE, + text=json.dumps(manifest, indent=2, sort_keys=True), + ) + encrypt_comparison_failure(directory) diff --git a/scripts/delta_compile_series/constants.py b/scripts/delta_compile_series/constants.py index edafd2c76e..3e276ef56b 100644 --- a/scripts/delta_compile_series/constants.py +++ b/scripts/delta_compile_series/constants.py @@ -24,6 +24,16 @@ FULL_COPY_SUFFIX: str = "_full_copy" DURATION_PATTERN: re.Pattern[str] = re.compile(r"\s*\(\d+(?:\.\d+)?s\)") COMPILE_TIMINGS_PATTERN: re.Pattern[str] = re.compile(r'\n "compile_timings": \{[^{}]*\n \}') +MAX_FAILURES: int = 4 +MAX_STEP_BYTES: int = 4 * 1024 * 1024 +MAX_FILE_BYTES: int = 512 * 1024 +MAX_ENTRIES: int = 2048 +MAX_READ_BYTES: int = 16 * 1024 * 1024 +MAX_CACHE_PATHS: int = 16 +COMPARISON_MANIFEST_FILE: str = "manifest.json" +MAX_EVIDENCE_FILES: int = 9 +MAX_ENCRYPTED_FAILURES: int = 12 +MAX_ENCRYPTED_BYTES: int = MAX_STEP_BYTES + 64 * 1024 EDITS: tuple[DeltaCompileEdit, ...] = ( DeltaCompileEdit("body_constant", r"COALESCE\(b\.amount, \d+\)", "COALESCE(b.amount, {value})"), diff --git a/scripts/delta_compile_series/main/run_delta_compile_series.py b/scripts/delta_compile_series/main/run_delta_compile_series.py index ac38d9beba..dd8e31d65d 100644 --- a/scripts/delta_compile_series/main/run_delta_compile_series.py +++ b/scripts/delta_compile_series/main/run_delta_compile_series.py @@ -6,8 +6,10 @@ import shutil from collections.abc import Callable from pathlib import Path +from typing import Any from scripts.delta_compile_series._helpers.edits import edited_file +from scripts.delta_compile_series._helpers.evidence import cache_state, keep_comparison_failure from scripts.delta_compile_series._helpers.outcomes import full_compile_difference, judged from scripts.delta_compile_series.constants import ( DELTA_COMPILE_LINE, @@ -29,16 +31,43 @@ def _compared( project_dir: Path, compile_project: Callable[[], DeltaCompileOutcome], compile_in_full: Callable[[Path], DeltaCompileOutcome], + diagnostics_dir: Path | None, + seed: int, + step: str, + case: str, ) -> tuple[DeltaCompileOutcome, str | None]: """Compile, and compile a copy of the project as it was in full; how what they print differs.""" copy: Path = project_dir.with_name(project_dir.name + FULL_COPY_SUFFIX) shutil.rmtree(copy, ignore_errors=True) _ = shutil.copytree(project_dir, copy, symlinks=True) + before: dict[str, Any] = ( + {"delta": cache_state(project_dir), "full": cache_state(copy)} if diagnostics_dir else {} + ) outcome: DeltaCompileOutcome = compile_project() + after: dict[str, Any] = {"delta": cache_state(project_dir)} if diagnostics_dir else {} + full: DeltaCompileOutcome = compile_in_full(copy) + if diagnostics_dir: + after["full"] = cache_state(copy) problem: str | None = full_compile_difference( - outcome=outcome, full=compile_in_full(copy), project_dir=project_dir, full_project_dir=copy + outcome=outcome, full=full, project_dir=project_dir, full_project_dir=copy ) + if problem is not None and diagnostics_dir is not None: + try: + _ = keep_comparison_failure( + destination=diagnostics_dir, + seed=seed, + step=step, + case=case, + project=project_dir, + copy=copy, + outcome=outcome, + full=full, + before=before, + after=after, + ) + except OSError: + pass return outcome, problem @@ -49,6 +78,8 @@ def run_delta_compile_series( steps: int, compile_project: Callable[[], DeltaCompileOutcome], compile_in_full: Callable[[Path], DeltaCompileOutcome] | None = None, + diagnostics_dir: Path | None = None, + case: str = "", ) -> DeltaCompileSeries: """Edit or revert `steps` times, compiling after each, and judge or compare every compile.""" @@ -87,6 +118,10 @@ def run_delta_compile_series( project_dir=project_dir, compile_project=compile_project, compile_in_full=compile_in_full, + diagnostics_dir=diagnostics_dir, + seed=seed, + step=step, + case=case, ) delta: bool = DELTA_COMPILE_LINE in outcome.stderr deltas += int(delta) diff --git a/scripts/delta_compile_series/models.py b/scripts/delta_compile_series/models.py index 26628e2dfc..367bce381d 100644 --- a/scripts/delta_compile_series/models.py +++ b/scripts/delta_compile_series/models.py @@ -25,6 +25,8 @@ class DeltaCompileOutcome: returncode: int stdout: str stderr: str + command: tuple[str, ...] = () + environment: tuple[tuple[str, str], ...] = () @dataclass(frozen=True) diff --git a/scripts/fensu_policy/_helpers/native_crossings.py b/scripts/fensu_policy/_helpers/native_crossings.py index 430cc3fb78..27df706697 100644 --- a/scripts/fensu_policy/_helpers/native_crossings.py +++ b/scripts/fensu_policy/_helpers/native_crossings.py @@ -27,6 +27,11 @@ _PYTHON_FIELD: re.Pattern[str] = re.compile(NATIVE_PYTHON_FIELD_PATTERN) _STRUCT_OPEN: re.Pattern[str] = re.compile(r"^\s*(?:pub(?:\([^)]*\))?\s+)?struct\s+\w+[^;]*\{\s*$") _FIELD: re.Pattern[str] = re.compile(r"^\s*(?:pub(?:\([^)]*\))?\s+)?\w+\s*:") +_PYCLASS_MODULE: re.Pattern[str] = re.compile( + r"(^[ \t]*#\[\s*(?:pyo3::)?pyclass\s*\([^]]*?\bmodule\s*=\s*(?:r#*)?)" + r'"(?:\\.|[^"\\\n])*"', + re.MULTILINE, +) def is_checked_binding(*, path: str) -> bool: @@ -72,8 +77,9 @@ def _code_lines(*, source: str) -> list[tuple[int, str]]: def crossing_lines(*, source: str) -> list[int]: - """Line numbers of Rust-to-Python crossings, one entry per crossing.""" + """Line numbers of Rust-to-Python crossings, excluding inert PyO3 module metadata.""" + source = _PYCLASS_MODULE.sub(r'\1""', source) found: list[int] = [] number: int line: str diff --git a/scripts/native_tests/__init__.py b/scripts/native_tests/__init__.py new file mode 100644 index 0000000000..4175a3327b --- /dev/null +++ b/scripts/native_tests/__init__.py @@ -0,0 +1 @@ +"""Shared native integration-test infrastructure.""" diff --git a/scripts/native_tests/constants.py b/scripts/native_tests/constants.py new file mode 100644 index 0000000000..3473a3cec7 --- /dev/null +++ b/scripts/native_tests/constants.py @@ -0,0 +1,6 @@ +"""Reproducible native integration database image.""" + +POSTGRES_IMAGE: str = ( + "postgres:17.6-alpine3.22@" + "sha256:ef257d85f76e48da1c64832459b59fcaba1a4dac97bf5d7450c77753542eee94" +) diff --git a/scripts/native_tests/main/__init__.py b/scripts/native_tests/main/__init__.py new file mode 100644 index 0000000000..22055e8817 --- /dev/null +++ b/scripts/native_tests/main/__init__.py @@ -0,0 +1 @@ +"""Native test command entrypoints.""" diff --git a/scripts/native_tests/main/run.py b/scripts/native_tests/main/run.py new file mode 100644 index 0000000000..e8c9ea392e --- /dev/null +++ b/scripts/native_tests/main/run.py @@ -0,0 +1,33 @@ +"""One disposable PostgreSQL server per suite; Rust fixtures isolate their own databases.""" + +from __future__ import annotations + +import os +import subprocess +import sys + +from testcontainers.postgres import PostgresContainer + +from scripts.native_tests.constants import POSTGRES_IMAGE + + +def run_native_tests(argv: list[str] | None = None) -> int: + """Fail closed on Docker/startup errors and propagate Cargo's result after cleanup.""" + arguments: list[str] = sys.argv[1:] if argv is None else argv + print("Starting disposable PostgreSQL for native integration tests...", flush=True) + try: + with PostgresContainer(POSTGRES_IMAGE, driver=None) as postgres: + print("PostgreSQL ready. Running native tests with isolated databases...", flush=True) + result: subprocess.CompletedProcess[bytes] = subprocess.run( + ["cargo", "test", *arguments], + env=os.environ + | { + "SQLBUILD_TEST_CONTROL_POSTGRES_URL": postgres.get_connection_url(), + }, + check=False, + ) + print(f"PostgreSQL removed. Native tests exited {result.returncode}.", flush=True) + return result.returncode + except Exception as error: + print(f"Native integration tests failed: {error}", file=sys.stderr, flush=True) + return 1 diff --git a/scripts/run_native_tests.py b/scripts/run_native_tests.py new file mode 100644 index 0000000000..2f673d1fc7 --- /dev/null +++ b/scripts/run_native_tests.py @@ -0,0 +1,12 @@ +"""Direct wrapper for native tests with disposable PostgreSQL.""" + +from scripts.native_tests.main.run import run_native_tests + + +def main(argv: list[str] | None = None) -> int: + """Run Cargo tests with a shared Testcontainers PostgreSQL instance.""" + return run_native_tests(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index 35373fb57b..468d2f250c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -114,6 +114,8 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [sqb init](cli/init.md) (`cli/init`) - Scaffold a new SQLBuild project. - [sqb playground](cli/playground.md) (`cli/playground`) - Create a self-contained SQLBuild project to explore locally. - [sqb skills](cli/skills.md) (`cli/skills`) - Install SQLBuild skill files for AI coding agents. +- [sqb server](cli/server.md) (`cli/server`) - Run the control service and scheduler. +- [sqb login and logout](cli/login.md) (`cli/login`) - Authorize a bounded CLI session through your server's browser sign-in. - [sqb compile](cli/compile.md) (`cli/compile`) - Compile models into resolved SQL, validate contracts, and write target artifacts - fully offline. - [sqb format](cli/format.md) (`cli/format`) - Apply canonical SQLBuild SQL formatting. - [sqb contract](cli/contract.md) (`cli/contract`) - Compare or generate repository contracts from physical warehouse schemas. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/login.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/login.md new file mode 100644 index 0000000000..eb42ef116e --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/login.md @@ -0,0 +1,65 @@ + + +# sqb login and logout + +> Authorize a bounded CLI session through your server's browser sign-in. + +Online: https://sqlbuild.com/docs/cli/login/ + +Sign in to a SQLBuild server using its configured password, SSO or organization authentication: + +```bash +sqb login --server https://control.example.com +``` + +Only authorize a request you started yourself. Compare the browser's verification code with +your terminal, sign in, and choose **Authorize CLI**. Return to the terminal for the completion +message. Choose **Cancel** to refuse the request. + +The browser confirmation expires after five minutes and can be exchanged once. The CLI receives +its own bounded person session, not your browser cookie or a personal API token. Your existing +permissions still apply; signing in does not create a person, assign a role or grant access. + +## Use a browser on another device + +```bash +sqb login --server https://control.example.com --no-browser +``` + +Open the printed URL in your browser and compare its verification code with the terminal. +Keep the terminal running until it confirms completion. If the request expires or is cancelled, +start `sqb login` again. + +## Check your saved session + +```bash +sqb login status --server https://control.example.com +``` + +This command verifies the saved session with the live server and prints only your person and +session identity as JSON. Disabled people, revoked sessions and password changes are enforced +by the server. If the session is unavailable, sign in again. + +## Sign out + +```bash +sqb logout --server https://control.example.com +``` + +Logout revokes the current CLI session and removes its local credential. It does not sign your +browser out. If the server cannot be reached, local cleanup still runs and the command reports +that remote revocation could not be confirmed. + +Credentials use your OS credential store. On headless Linux without a desktop session, an +unavailable credential service permits a private fallback under +`~/.config/sqlbuild/credentials`, restricted to your user. A locked or ambiguous OS credential +store is an error: unlock it and retry. Credentials are never printed or written into a project. + +| Option | Purpose | +| --- | --- | +| `--server HTTPS_ORIGIN` | Required server origin, without a path, query or embedded credentials | +| `--ca-cert PEM` | Trust an additional CA certificate for this server; certificate and hostname verification remain enabled | +| `--no-browser` | Print the confirmation URL without opening a browser; applies to sign-in | + +Lifecycle messages go to stderr. Successful login and logout leave stdout empty; status writes +identity JSON to stdout. A failure exits nonzero and never prints a credential. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md new file mode 100644 index 0000000000..271c324077 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -0,0 +1,87 @@ + + +# sqb server + +> Run the control service and scheduler. + +Online: https://sqlbuild.com/docs/cli/server/ + +Run the control API and scheduler without loading project Python: + +```bash +sqb server +``` + +Use `sqb server --help` for service, authentication and launcher options. + +## Set retention + +Set how much history the server keeps: + +```bash +sqb server --statistics-retention-days 30 --audit-retention-days 730 --snapshot-retention-count 20 +``` + +| Data | Option | Default | Range | +| --- | --- | --- | --- | +| Hourly warehouse statistics | `--statistics-retention-days` | `90` | `1`–`65535` | +| Security audit records | `--audit-retention-days` | `400` | `1`–`65535` | +| Published snapshots per location and target | `--snapshot-retention-count` | `10` | `1`–`65535` | +| Skipped and covered schedule ticks | None | 30 days | Fixed | + +Statistics samples older than the horizon disappear from statistics reads at once and are then +deleted in the background. + +A security audit record older than the horizon is deleted only when it is no longer needed: + +- The webhook in `SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL`, if set, has received it. +- No retained run's retry depends on it. A manual job or rerun request keeps its record while + the run exists. +- It is not the newest record, so audit page cursors never see a reused position. + +Records kept past the horizon for these reasons are reported on stderr as +`Security audit retention: retained past horizon: N` whenever the count changes. A growing count +usually means the audit webhook is failing. + +The server keeps the newest published snapshots for each location and target. It deletes an older +snapshot only when no retained run, job, trigger, trigger change or location request uses it, and +deletes its node definitions and SQL text when no other snapshot shares them. Repeating or +interrupting this cleanup is safe. + +The scheduler deletes covered ticks older than 30 days. It keeps each trigger's latest tick and +every tick that started a run. A new schedule, a resumed trigger or a cursor reset catches up only +on the latest missed time, so deleted ticks never come back. + +An invalid value stops the server before it opens the control database. Servers that share one +control database delete at the shortest horizon among them and count only their own audit +webhook, so give them the same values and webhook. + +## Inspect published SQL + +Sign in with [`sqb login`](login.md), then inspect one exact node in the current +successful publication: + +```bash +sqb control sql model:orders \ + --server https://control.example.com \ + --location-id "$LOCATION_ID" \ + --target development +``` + +The command uses your saved person session and prints JSON. It does not load a local project +or open a warehouse connection. Use `--ca-cert PATH` for an additional trusted PEM certificate. +The node key is exact: `model:orders+` does not expand a selector. Progress and failures go to +stderr; a failed request prints no SQL to stdout. Permission denials include their decision ID. + +The equivalent person-authenticated API is a `GET` request to +`/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/sql`. + +Developer or Admin access must cover the requested location and target. Viewer, Operator and +the configurable default Viewer role do not grant SQL inspection. The response contains the +snapshot ID, exact node key, raw SQL and compiled SQL. Missing SQL fields are `null`; an absent +current publication or node returns JSON `null` after authorization. + +SQL inspection does not execute SQL or return Python source, execution artifacts, logs or sample +data. Every successful inspection records an audit entry without copying SQL into it. If the +audit cannot be saved, the request fails without returning SQL. Revocation takes effect on the +next request, including sessions created by [`sqb login`](login.md). diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index a64b35d2da..8a67d7d699 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -199,6 +199,7 @@ COLUMN_SCHEMA_FINGERPRINT: str COLUMN_DEFINITION_B64: str COLUMN_METADATA_JSON_B64: str COLUMN_TIMESTAMP: str +COLUMN_EVENT_ID: str NODE_TYPE_MODEL: str NODE_TYPE_SEED: str AUDIT_GATE_METADATA_KEY: str @@ -1738,6 +1739,48 @@ class NativeCompiledAudit: @property def meta(self) -> dict[str, object] | None: ... +class NativeOrchestrationSnapshot: + """Compiled jobs, schedules and event/source-version/age sensors as publishable data.""" + + def to_json(self) -> str: ... + def publish(self, server_url: str, bearer: str) -> str: ... + def start_worker( + self, server_url: str, bearer: str, identity: tuple[str, str] + ) -> NativeOrchestrationWorker: ... + +class NativeOrchestrationWorker: + """A verified claim owner with joined native heartbeat teardown.""" + + def execution_json(self) -> str: ... + def cancel_requested(self) -> bool: ... + def report(self, event_id: str, change_json: str) -> bool: ... + def close(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, + kind: type[BaseException] | None, + value: BaseException | None, + traceback: TracebackType | None, + ) -> None: ... + +def build_orchestration_snapshot( + project: CompiledProjectView, + metadata_json: str, + project_dir: Path, + *, + declarations: Sequence[object] | None = None, +) -> NativeOrchestrationSnapshot: ... +def run_location_command( + *, + project_dir: Path, + location_id: str, + selected_target: str, + generation: int, + server_url: str, + image_id: str | None = None, + action: str = "publish", +) -> int: ... + class NativeProjectView: @property def run_id(self) -> str: ... @@ -1780,6 +1823,8 @@ class NativeProjectView: @property def loader_functions(self) -> tuple[DiscoveredLoaderFunction, ...]: ... @property + def python_execution_inputs(self) -> dict[str, tuple[str, Path]]: ... + @property def hook_functions(self) -> tuple[DiscoveredHookFunction, ...]: ... @property def enforce_explicit_references(self) -> bool: ... @@ -2309,6 +2354,13 @@ def resolve_target_connection_config( target_name: str, cli_vars: dict[str, object] | None = None, ) -> dict[str, object]: ... +def build_effective_connection_config( + *, + discovered_inputs: DiscoveredProjectInputs, + selected_target: str | None = None, + cli_vars: dict[str, object] | None = None, + catalog: bool = False, +) -> dict[str, object]: ... def normalize_dbt_profile_connection( *, resolved: ResolvedDbtProfileOutput ) -> NormalizedDbtProfileConnection: ... @@ -4146,6 +4198,8 @@ def write_fingerprint( render_create_table_sql: Callable[..., str] | None = None, render_create_index_sqls: Callable[..., tuple[str, ...]] | None = None, ensure_table: bool = True, + read_columns: Callable[..., tuple[ColumnInfo, ...]] | None = None, + render_add_columns: Callable[..., tuple[str, ...]] | None = None, ) -> bool: ... def write_fingerprints( *, @@ -4159,6 +4213,8 @@ def write_fingerprints( render_create_table_sql: Callable[..., str] | None = None, render_create_index_sqls: Callable[..., tuple[str, ...]] | None = None, on_progress: FingerprintWriteProgress | None = None, + read_columns: Callable[..., tuple[ColumnInfo, ...]] | None = None, + render_add_columns: Callable[..., tuple[str, ...]] | None = None, ) -> None: ... def select_latest_fingerprints( *, @@ -5005,6 +5061,8 @@ class BuildProgressCallbacks: def redraw_transient_line(self) -> None: ... def on_node_complete(self, node_result: object) -> None: ... def write_execution_event(self, result: object) -> None: ... + def on_check_start(self, name: str) -> None: ... + def on_check_complete(self, result: PythonCheckExecutionResult) -> None: ... def close(self) -> None: ... class AuditProgressReporter: @@ -7274,6 +7332,7 @@ class CliEntrypointHandlers(NativeRecord): run_refactor: Callable[..., int] | None run_recall: Callable[..., int] | None run_review: Callable[..., int] | None + run_location: Callable[..., int] | None def __init__( self, run_compile: Callable[[CompileCommandRequest], int], @@ -7310,6 +7369,7 @@ class CliEntrypointHandlers(NativeRecord): run_refactor: Callable[..., int] | None = None, run_recall: Callable[..., int] | None = None, run_review: Callable[..., int] | None = None, + run_location: Callable[..., int] | None = None, ) -> None: ... class CommandWarehouseScope(NativeRecord): @@ -7798,6 +7858,8 @@ class SourceFreshnessRenderers(NativeRecord): render_insert_records_sql: Callable[..., str] render_create_table_sql: Callable[..., str] | None render_create_index_sqls: Callable[..., tuple[str, ...]] | None + read_columns: Callable[..., tuple[ColumnInfo, ...]] | None + render_add_columns: Callable[..., tuple[str, ...]] | None def __init__( self, render_qualified_name: Callable[..., str | None], @@ -7805,6 +7867,8 @@ class SourceFreshnessRenderers(NativeRecord): render_insert_records_sql: Callable[..., str], render_create_table_sql: Callable[..., str] | None = None, render_create_index_sqls: Callable[..., tuple[str, ...]] | None = None, + read_columns: Callable[..., tuple[ColumnInfo, ...]] | None = None, + render_add_columns: Callable[..., tuple[str, ...]] | None = None, ) -> None: ... class SourceFreshnessObservation(NativeRecord): @@ -8025,6 +8089,7 @@ class CliCommand(StrEnum): DIFF = "diff" REVIEW = "review" DEBUG = "debug" + LOCATION = "location" LINEAGE = "lineage" QUERY = "query" COST = "cost" @@ -11755,6 +11820,7 @@ class TargetConfig(NativeRecord): diff: TargetDiffConfig warehouses: TargetWarehousesConfig concurrency: int | None + catalog_connection: str | None def __init__( self, connection: dict[str, object] = ..., @@ -11778,6 +11844,7 @@ class TargetConfig(NativeRecord): diff: TargetDiffConfig = ..., warehouses: TargetWarehousesConfig = ..., concurrency: int | None = ..., + catalog_connection: str | None = ..., ) -> None: ... class LocalTargetConfig(NativeRecord): @@ -11804,6 +11871,7 @@ class LocalTargetConfig(NativeRecord): diff: TargetDiffConfig warehouses: TargetWarehousesConfig concurrency: int | None + catalog_connection: str | None def __init__( self, connection: dict[str, object] = ..., @@ -11828,6 +11896,7 @@ class LocalTargetConfig(NativeRecord): diff: TargetDiffConfig = ..., warehouses: TargetWarehousesConfig = ..., concurrency: int | None = ..., + catalog_connection: str | None = ..., ) -> None: ... class SettingsConfig(NativeRecord): @@ -15550,6 +15619,7 @@ class CompiledProject(NativeRecord): sql_expansions: dict[Path, CompiledSqlExpansion] native_session: Any | None native_project: NativeCompiledProject | None + python_execution_inputs: dict[str, tuple[str, Path]] def __init__( self, run_id: str, @@ -15586,6 +15656,7 @@ class CompiledProject(NativeRecord): sql_expansions: dict[Path, CompiledSqlExpansion] = ..., native_session: Any | None = ..., native_project: NativeCompiledProject | None = ..., + python_execution_inputs: dict[str, tuple[str, Path]] = ..., ) -> None: ... @property def lineage_graph(self) -> NativeProjectGraph: ... diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 9dca6ce419..fde2109986 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -167,6 +167,10 @@ "sqlbuild.compiler.discovery._helpers.filesystem.project_modules", "load_sink_module", ), + "load_orchestration_module": ( + "sqlbuild.compiler.discovery._helpers.filesystem.project_modules", + "load_orchestration_module", + ), "module_functions": ( "sqlbuild.compiler.discovery._helpers.filesystem.project_modules", "module_functions", diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index c68e33d507..4ca6064432 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -130,6 +130,9 @@ from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( load_materialize_function as load_materialize_function, ) +from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( + load_orchestration_module as load_orchestration_module, +) from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( load_provider_module as load_provider_module, ) diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 6acc8921ad..683da84321 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -1661,6 +1661,7 @@ def estimate_relation_row_count( database: str | None, schema: str | None, name: str, + metadata_only: bool = False, ) -> RelationRowCountEstimate: """Report an unknown row count for custom adapters without metadata support.""" diff --git a/src/sqlbuild/adapter/contract/classes/diff.py b/src/sqlbuild/adapter/contract/classes/diff.py index ccb8b2078a..376cc6debe 100644 --- a/src/sqlbuild/adapter/contract/classes/diff.py +++ b/src/sqlbuild/adapter/contract/classes/diff.py @@ -84,8 +84,9 @@ def estimate_relation_row_count( database: str | None, schema: str | None, name: str, + metadata_only: bool = False, ) -> RelationRowCountEstimate: - """Return the metadata row count for one relation without scanning its data.""" + """Estimate counts, forbidding data scans when metadata_only is requested.""" ... @abstractmethod diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index d376a6a228..119592a31a 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -6,6 +6,7 @@ import json import logging import re +import time from datetime import date, datetime from decimal import Decimal from importlib.machinery import ModuleSpec @@ -1075,12 +1076,31 @@ def render_cursor_bound_literal(self, *, value: str, cursor_type: str | None) -> return f"'{value}'" def connect(self, config: dict[str, Any]) -> Any: - """Open a DuckDB connection from the resolved connection config.""" + """Open DuckDB, waiting up to five seconds for another process's database file lock.""" import duckdb database: str = str(config.get("database", ":memory:")) - raw_connection: duckdb.DuckDBPyConnection = duckdb.connect(database=database) + deadline: float = time.monotonic() + 5 + while True: + try: + raw_connection: duckdb.DuckDBPyConnection = duckdb.connect( + database=database, read_only=bool(config.get("read_only", False)) + ) + break + except duckdb.IOException as error: + if ( + not any( + marker in str(error) + for marker in ( + ": Conflicting lock is held in ", + "\nFile is already open in ", + ) + ) + or time.monotonic() >= deadline + ): + raise + time.sleep(0.05) return self._configured_connection(raw_connection=raw_connection, config=config) def connect_local_replay(self, database: Path) -> Any: @@ -2250,6 +2270,7 @@ def estimate_relation_row_count( database: str | None, schema: str | None, name: str, + metadata_only: bool = False, ) -> RelationRowCountEstimate: """Read DuckDB's estimated table size from the catalog.""" diff --git a/src/sqlbuild/adapter/contract/models.py b/src/sqlbuild/adapter/contract/models.py index 5c0a97c92a..33862792c6 100644 --- a/src/sqlbuild/adapter/contract/models.py +++ b/src/sqlbuild/adapter/contract/models.py @@ -369,10 +369,12 @@ class RowDiffCoverage: @dataclass(frozen=True) class RelationRowCountEstimate: - """Metadata row count read without scanning data; ``row_count`` is None when unknown.""" + """Relation estimates; unavailable counts and storage bytes remain None.""" row_count: int | None detail: str | None = None + byte_count: int | None = None + byte_count_basis: str | None = None @dataclass(frozen=True) diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 1247bbc73c..7c6500412d 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -2569,6 +2569,7 @@ def estimate_relation_row_count( database: str | None, schema: str | None, name: str, + metadata_only: bool = False, ) -> RelationRowCountEstimate: """Read ``numRows`` from the tables API; only native tables report a count.""" @@ -2579,11 +2580,19 @@ def estimate_relation_row_count( ) table_type: str = str(getattr(table, "table_type", NATIVE_TABLE_TYPE)).upper() num_rows: object | None = getattr(table, "num_rows", None) - if table_type != NATIVE_TABLE_TYPE or not isinstance(num_rows, int): + if table_type != NATIVE_TABLE_TYPE: return RelationRowCountEstimate( row_count=None, detail=f"no row count ({table_type.lower()})" ) - return RelationRowCountEstimate(row_count=num_rows) + byte_count: object | None = getattr(table, "num_bytes", None) if metadata_only else None + return RelationRowCountEstimate( + row_count=num_rows if isinstance(num_rows, int) else None, + detail=f"no row count ({table_type.lower()})" + if not isinstance(num_rows, int) + else None, + byte_count=byte_count if isinstance(byte_count, int) else None, + byte_count_basis="tables_api_num_bytes" if isinstance(byte_count, int) else None, + ) def inspect_row_diff_coverage( self, diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 0820e86f25..7a4bf15518 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -465,22 +465,34 @@ def physical_relation_generation( return generation_comment(tuple(row)).removeprefix(GENERATION_COMMENT_PREFIX) def estimate_relation_row_count( - self, *, connection: Any, database: str | None, schema: str | None, name: str + self, + *, + connection: Any, + database: str | None, + schema: str | None, + name: str, + metadata_only: bool = False, ) -> RelationRowCountEstimate: del database row: Any = self.execute( connection=connection, - sql="SELECT engine, total_rows FROM system.tables WHERE " + sql=( + "SELECT engine, total_rows, total_bytes FROM system.tables WHERE " + if metadata_only + else "SELECT engine, total_rows FROM system.tables WHERE " + ) + _database_filter(schema) + f" AND name = {quote_string(name)}", ).fetchone() if row is None: return RelationRowCountEstimate(row_count=None, detail="relation not found") - if row[1] is None: - return RelationRowCountEstimate( - row_count=None, detail=f"engine {row[0]} does not report row counts" - ) - return RelationRowCountEstimate(row_count=int(row[1])) + byte_count: int | None = int(row[2]) if metadata_only and row[2] is not None else None + return RelationRowCountEstimate( + row_count=int(row[1]) if row[1] is not None else None, + detail=f"engine {row[0]} does not report row counts" if row[1] is None else None, + byte_count=byte_count, + byte_count_basis="system_tables_total_bytes" if byte_count is not None else None, + ) def read_view_definition( self, *, connection: Any, database: str | None, schema: str, name: str diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index edf2f88a2f..a7eae3813d 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -2565,6 +2565,7 @@ def estimate_relation_row_count( database: str | None, schema: str | None, name: str, + metadata_only: bool = False, ) -> RelationRowCountEstimate: """Read ``ANALYZE TABLE`` statistics; ``DESCRIBE DETAIL`` has no row count.""" @@ -2596,12 +2597,19 @@ def estimate_relation_row_count( statistics_match: re.Match[str] | None = re.search( r"(\d+)\s+rows", details.get("statistics", "") ) - if statistics_match is None: - return RelationRowCountEstimate( - row_count=None, - detail="table statistics not collected; run ANALYZE TABLE ... COMPUTE STATISTICS", - ) - return RelationRowCountEstimate(row_count=int(statistics_match.group(1))) + bytes_match: re.Match[str] | None = ( + re.search(r"(\d+)\s+bytes", details.get("statistics", "")) if metadata_only else None + ) + return RelationRowCountEstimate( + row_count=int(statistics_match.group(1)) if statistics_match is not None else None, + detail=( + "table statistics not collected; run ANALYZE TABLE ... COMPUTE STATISTICS" + if statistics_match is None + else None + ), + byte_count=int(bytes_match.group(1)) if bytes_match is not None else None, + byte_count_basis="describe_table_statistics_bytes" if bytes_match is not None else None, + ) def inspect_row_diff_coverage( self, diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 3232567671..05bb55cd76 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -2356,6 +2356,7 @@ def estimate_relation_row_count( database: str | None, schema: str | None, name: str, + metadata_only: bool = False, ) -> RelationRowCountEstimate: """Read the planner's ``reltuples`` estimate, treating unanalyzed data as unknown.""" @@ -2381,12 +2382,16 @@ def estimate_relation_row_count( row_count=None, detail="no table statistics (view or partitioned table)" ) estimated_rows: float = float(row[1]) + row_count: int | None = None if estimated_rows > 0: - return RelationRowCountEstimate(row_count=int(estimated_rows)) - if int(row[2]) == 0: - return RelationRowCountEstimate(row_count=0) + row_count = int(estimated_rows) + elif int(row[2]) == 0: + row_count = 0 return RelationRowCountEstimate( - row_count=None, detail="table statistics not collected; run ANALYZE" + row_count=row_count, + detail="table statistics not collected; run ANALYZE" if row_count is None else None, + byte_count=int(row[2]) if metadata_only else None, + byte_count_basis="pg_relation_size_main_fork" if metadata_only else None, ) def diff_schema( diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 49736eb0af..c66c388a16 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -3548,6 +3548,7 @@ def estimate_relation_row_count( database: str | None, schema: str | None, name: str, + metadata_only: bool = False, ) -> RelationRowCountEstimate: """Read ``ROW_COUNT`` from the information schema; views report no count.""" @@ -3556,13 +3557,17 @@ def estimate_relation_row_count( database=database, schema=schema, name=name, - projection="row_count", + projection="row_count,bytes" if metadata_only else "row_count", ) if row is None: return RelationRowCountEstimate(row_count=None, detail="relation not found") - if row[0] is None: - return RelationRowCountEstimate(row_count=None, detail="no row count (view)") - return RelationRowCountEstimate(row_count=int(row[0])) + byte_count: int | None = int(row[1]) if metadata_only and row[1] is not None else None + return RelationRowCountEstimate( + row_count=int(row[0]) if row[0] is not None else None, + detail="no row count (view)" if row[0] is None else None, + byte_count=byte_count, + byte_count_basis="information_schema_bytes" if byte_count is not None else None, + ) def inspect_row_diff_coverage( self, diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py index 235d9f9169..a04a722fec 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py @@ -1502,9 +1502,14 @@ def estimate_relation_row_count( database: str | None, schema: str | None, name: str, + metadata_only: bool = False, ) -> RelationRowCountEstimate: - """Count table rows: a local file has no metadata count, and counting reads only pages.""" + """Keep interactive counts; background metadata sampling never reads table pages.""" + if metadata_only: + return RelationRowCountEstimate( + row_count=None, detail="SQLite does not report table row counts from metadata" + ) del database table_name: str = physical_name(schema=schema, name=name) row: tuple[Any, ...] | None = connection.execute_with( diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 2f200b531d..7bafe69563 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -340,6 +340,7 @@ def render_create_fingerprint_table_sql( from sqlbuild._native import ( COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, + COLUMN_EVENT_ID, COLUMN_METADATA_JSON_B64, COLUMN_NODE_NAME, COLUMN_NODE_TYPE, @@ -373,7 +374,8 @@ def render_create_fingerprint_table_sql( f"{COLUMN_SCHEMA_FINGERPRINT} NVARCHAR(450) NOT NULL, " f"{COLUMN_DEFINITION_B64} NVARCHAR(MAX) NOT NULL, " f"{COLUMN_METADATA_JSON_B64} NVARCHAR(MAX) NOT NULL, " - f"{COLUMN_TIMESTAMP} DATETIME2 NOT NULL" + f"{COLUMN_TIMESTAMP} DATETIME2 NOT NULL, " + f"{COLUMN_EVENT_ID} NVARCHAR(MAX)" f")" ) return self._create_table_if_missing_sql( @@ -1342,6 +1344,7 @@ def estimate_relation_row_count( database: str | None, schema: str | None, name: str, + metadata_only: bool = False, ) -> RelationRowCountEstimate: """Sum heap or clustered-index partition row counts from the catalog.""" diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index a093f3faa9..d55f426473 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -254,6 +254,8 @@ def sql_scenarios(self) -> tuple[CompiledSqlScenario, ...]: ... @property def loader_functions(self) -> tuple[DiscoveredLoaderFunction, ...]: ... @property + def python_execution_inputs(self) -> dict[str, tuple[str, Path]]: ... + @property def hook_functions(self) -> tuple[DiscoveredHookFunction, ...]: ... @property def enforce_explicit_references(self) -> bool: ... diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py index f28340d71b..1eef8c4987 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py @@ -244,6 +244,17 @@ def load_sink_module(*, file_path: Path, project_dir: Path) -> ModuleType: ) +def load_orchestration_module(*, file_path: Path, project_dir: Path) -> ModuleType: + """Execute an orchestration module with the project's normal package imports.""" + + return _load_project_package_module( + file_path=file_path, + project_dir=project_dir, + file_label="orchestration", + error_type=PythonNodeDiscoveryError, + ) + + def load_materialize_function( *, file_path: Path, project_dir: Path ) -> Callable[..., object] | None: @@ -338,6 +349,8 @@ def _exec_project_module( sys.modules[module_name] = module spec.loader.exec_module(module) except Exception as error: + if sys.modules.get(module_name) is module: + sys.modules.pop(module_name, None) raise error_type( f"Failed to import {file_label} file {file_path.relative_to(project_dir)}: {error}" ) from error diff --git a/src/sqlbuild/executor/clone/_helpers/fingerprinting.py b/src/sqlbuild/executor/clone/_helpers/fingerprinting.py index 5c2abd241e..0378d85973 100644 --- a/src/sqlbuild/executor/clone/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/clone/_helpers/fingerprinting.py @@ -205,6 +205,8 @@ def _on_group_progress( fingerprints=group, render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, on_progress=_on_group_progress, diff --git a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py index d8d0c9f088..f4559e8027 100644 --- a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py +++ b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py @@ -118,6 +118,8 @@ def materialize_query_diff_artifact( ), render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/executor/functions/main/_execute.py b/src/sqlbuild/executor/functions/main/_execute.py index a95ec0cd3b..c7f935e162 100644 --- a/src/sqlbuild/executor/functions/main/_execute.py +++ b/src/sqlbuild/executor/functions/main/_execute.py @@ -182,6 +182,8 @@ def _try_write_function_fingerprint( fingerprint=fingerprint, render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py b/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py index edc76da34c..310f994670 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py @@ -55,6 +55,8 @@ def try_write_python_node_identity_fingerprint( ), render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py index 87027f7f5c..220af44090 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py @@ -148,6 +148,8 @@ def _execute_one_python_check( ) as lifecycle, ): if blocked is None: + if callbacks.on_check_start is not None: + callbacks.on_check_start(check_function.name) node: DiscoveredPythonNode = python_graph.nodes_by_name[check_function.name] warnings: list[str] = [] context: CheckContext = CheckContext( diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py index 4237e12f99..e41a47874f 100644 --- a/src/sqlbuild/executor/python_nodes/models.py +++ b/src/sqlbuild/executor/python_nodes/models.py @@ -404,3 +404,4 @@ class PythonCheckCallbacks: logger: logging.Logger | None = None identity_recorder: PythonIdentityRecorder | None = None on_check_complete: Callable[[PythonCheckExecutionResult], None] | None = None + on_check_start: Callable[[str], None] | None = None diff --git a/src/sqlbuild/executor/review/_helpers/marker.py b/src/sqlbuild/executor/review/_helpers/marker.py index ee87afbfbf..1f2f23e173 100644 --- a/src/sqlbuild/executor/review/_helpers/marker.py +++ b/src/sqlbuild/executor/review/_helpers/marker.py @@ -52,6 +52,8 @@ def write_review_marker( ), render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py index e084b6ab5d..065fe30123 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py @@ -97,6 +97,8 @@ def try_write_fingerprint( fingerprint=fingerprint, render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py index a103b8bb6b..02a44cc063 100644 --- a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py @@ -56,6 +56,8 @@ def try_write_seed_fingerprint( fingerprint=fingerprint, render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py new file mode 100644 index 0000000000..f6f022e65d --- /dev/null +++ b/src/sqlbuild/orchestration.py @@ -0,0 +1,49 @@ +"""Public authoring and snapshot compilation API for orchestration declarations.""" + +from sqlbuild.orchestration_declarations.main.compile_snapshot import compile_snapshot +from sqlbuild.orchestration_declarations.models.authoring import ( + CalendarSchedule as calendar_schedule, +) +from sqlbuild.orchestration_declarations.models.authoring import ( + CalendarWindow, + RunRequest, + SkipReason, +) +from sqlbuild.orchestration_declarations.models.authoring import Job as job +from sqlbuild.orchestration_declarations.models.authoring import JobGroup as job_group +from sqlbuild.orchestration_declarations.models.authoring import NodeEventCondition as node_event +from sqlbuild.orchestration_declarations.models.authoring import ( + PlatformEventCondition as platform_event, +) +from sqlbuild.orchestration_declarations.models.authoring import RunEventCondition as run_event +from sqlbuild.orchestration_declarations.models.authoring import Schedule as schedule +from sqlbuild.orchestration_declarations.models.authoring import Sensor as sensor +from sqlbuild.orchestration_declarations.models.authoring import ( + SourceNewDataCondition as source_new_data, +) +from sqlbuild.orchestration_declarations.models.authoring import ( + SourceStaleCondition as source_stale, +) +from sqlbuild.orchestration_declarations.models.sensor_context import SensorContext +from sqlbuild.orchestration_declarations.types import NodeEventKind, PlatformEventKind, RunEventKind + +__all__ = ( + "CalendarWindow", + "NodeEventKind", + "PlatformEventKind", + "RunEventKind", + "RunRequest", + "SensorContext", + "SkipReason", + "calendar_schedule", + "compile_snapshot", + "job", + "job_group", + "node_event", + "platform_event", + "run_event", + "schedule", + "sensor", + "source_new_data", + "source_stale", +) diff --git a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py new file mode 100644 index 0000000000..0f9fc8b052 --- /dev/null +++ b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py @@ -0,0 +1,28 @@ +"""Delegate authoring-record compilation to the native retained snapshot owner.""" + +from __future__ import annotations + +from collections.abc import Sequence +from pathlib import Path + +from sqlbuild._native import NativeOrchestrationSnapshot, build_orchestration_snapshot +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.orchestration_declarations.models.authoring import ( + CalendarSchedule, + Job, + JobGroup, + Sensor, +) + + +def compile_snapshot( + *, + project: CompiledProjectView, + metadata_json: str, + project_dir: Path, + declarations: Sequence[Job | JobGroup | CalendarSchedule | Sensor] | None = None, +) -> NativeOrchestrationSnapshot: + """Compile authoring records and retained project facts into a data-only snapshot.""" + return build_orchestration_snapshot( + project, metadata_json, project_dir, declarations=declarations + ) diff --git a/src/sqlbuild/orchestration_declarations/models/__init__.py b/src/sqlbuild/orchestration_declarations/models/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/orchestration_declarations/models/authoring.py b/src/sqlbuild/orchestration_declarations/models/authoring.py new file mode 100644 index 0000000000..9c5d72f044 --- /dev/null +++ b/src/sqlbuild/orchestration_declarations/models/authoring.py @@ -0,0 +1,157 @@ +"""Authoring records only; selection, validation and scheduling belong to Rust.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable, Sequence +from dataclasses import dataclass, field, replace +from datetime import datetime +from typing import Literal + +from sqlbuild.orchestration_declarations.types import NodeEventKind, PlatformEventKind, RunEventKind + + +@dataclass(frozen=True, kw_only=True) +class Schedule: + """Cron with an RFC 3339 start; missed times collapse into one run of the latest.""" + + cron: str + timezone: str + starts_at: str | None = None + default_status: Literal["running", "paused"] = "running" + + +@dataclass(frozen=True, eq=False) +class Job: + """A named selection to execute, optionally on a wall-clock schedule.""" + + name: str + description: str = field(kw_only=True) + select: str = field(kw_only=True) + exclude: str = field(default="", kw_only=True) + settings: dict[str, object] = field(default_factory=dict, kw_only=True) + concurrency: int | None = field(default=None, kw_only=True) + concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) + schedule: Schedule | None = field(default=None, kw_only=True) + targets: tuple[str, ...] = field(default=(), kw_only=True) + + +@dataclass(frozen=True, eq=False) +class JobGroup: + """Jobs ordered by compiled lineage and optional explicit dependencies.""" + + name: str + description: str = field(kw_only=True) + jobs: Sequence[Job] = field(kw_only=True) + after: dict[Job, Sequence[Job]] = field(default_factory=dict, kw_only=True) + concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) + schedule: Schedule | None = field(default=None, kw_only=True) + targets: tuple[str, ...] = field(default=(), kw_only=True) + + +@dataclass(frozen=True, kw_only=True) +class CalendarWindow: + """Timezone-aware, half-open interval for an upcoming calendar evaluation.""" + + start: datetime + end: datetime + + +@dataclass(frozen=True, kw_only=True, eq=False) +class CalendarSchedule: + """Decorate a location callback returning upcoming timezone-aware datetimes.""" + + job: Job | JobGroup + description: str + horizon: str + timezone: str + starts_at: str | None = None + default_status: Literal["running", "paused"] = "running" + targets: tuple[str, ...] = () + function: Callable[..., Iterable[datetime]] | None = field(default=None, repr=False) + + def __call__(self, function: Callable[..., Iterable[datetime]]) -> CalendarSchedule: + return replace(self, function=function) + + +@dataclass(frozen=True, kw_only=True) +class PlatformEventCondition: + """Consume matching server observations in durable storage order.""" + + kinds: Sequence[PlatformEventKind] + kind: Literal["platform_event"] = field(default="platform_event", init=False) + + +@dataclass(frozen=True, kw_only=True) +class RunEventCondition: + """Match scoped terminal outcomes, optionally requiring genuinely executed successful nodes.""" + + job: Job | JobGroup + kinds: Sequence[RunEventKind] + executed_successfully: Sequence[str] | None = None + kind: Literal["run_event"] = field(default="run_event", init=False) + + +@dataclass(frozen=True, kw_only=True) +class NodeEventCondition: + """Match each successful rebuild in any run within the sensor's location and target.""" + + select: str + kinds: Sequence[NodeEventKind] + kind: Literal["node_event"] = field(default="node_event", init=False) + + +@dataclass(frozen=True, kw_only=True) +class SourceNewDataCondition: + """Match the first non-null source version and later changed versions, in receipt order.""" + + select: str + kind: Literal["source_new_data"] = field(default="source_new_data", init=False) + + +@dataclass(frozen=True, kw_only=True) +class SourceStaleCondition: + """Match timestamp source versions older than after_seconds, once per version.""" + + select: str + after_seconds: int + kind: Literal["source_stale"] = field(default="source_stale", init=False) + + +@dataclass(frozen=True, eq=False) +class Sensor: + """Start a job for each matching event; event identity deduplicates replayed requests.""" + + name: str + description: str = field(kw_only=True) + when: ( + PlatformEventCondition + | RunEventCondition + | NodeEventCondition + | SourceNewDataCondition + | SourceStaleCondition + | None + ) = field(default=None, kw_only=True) + run: Job | JobGroup = field(kw_only=True) + minimum_interval_seconds: int = field(default=30, kw_only=True) + default_status: Literal["running", "paused"] = field(default="running", kw_only=True) + targets: tuple[str, ...] = field(default=(), kw_only=True) + function: Callable[..., Iterable[RunRequest] | SkipReason] | None = field( + default=None, repr=False, kw_only=True + ) + + def __call__(self, function: Callable[..., Iterable[RunRequest] | SkipReason]) -> Sensor: + return replace(self, function=function) + + +@dataclass(frozen=True, kw_only=True) +class RunRequest: + """A sensor-local key deduplicates requests across evaluations and retries.""" + + run_key: str + + +@dataclass(frozen=True) +class SkipReason: + """Complete an evaluation without starting a run.""" + + reason: str diff --git a/src/sqlbuild/orchestration_declarations/models/sensor_context.py b/src/sqlbuild/orchestration_declarations/models/sensor_context.py new file mode 100644 index 0000000000..648ce981db --- /dev/null +++ b/src/sqlbuild/orchestration_declarations/models/sensor_context.py @@ -0,0 +1,12 @@ +"""Invocation-local context; native fenced completion alone commits a proposed cursor.""" + +from dataclasses import dataclass + + +@dataclass(kw_only=True) +class SensorContext: + """Cursor assignment proposes progress; failure or timeout retains the committed cursor.""" + + cursor: str | None + target: str + evaluation_id: str diff --git a/src/sqlbuild/orchestration_declarations/types.py b/src/sqlbuild/orchestration_declarations/types.py new file mode 100644 index 0000000000..8149b173a6 --- /dev/null +++ b/src/sqlbuild/orchestration_declarations/types.py @@ -0,0 +1,33 @@ +"""Public enum vocabulary only; event processing belongs to Rust.""" + +from enum import StrEnum + + +class PlatformEventKind(StrEnum): + """Launch failures include response errors and do not prove worker absence.""" + + HEARTBEAT = "heartbeat" + SCHEDULER_NO_LEADER = "scheduler_no_leader" + TICK_OVERDUE = "tick_overdue" + CALENDAR_HORIZON_LOW = "calendar_horizon_low" + LOCATION_BROKEN = "location_broken" + LAUNCH_FAILED = "launch_failed" + + +class RunEventKind(StrEnum): + """COMPLETED selects every terminal outcome; it is not a run status.""" + + COMPLETED = "completed" + SUCCESS = "success" + WARN = "warn" + PARTIAL = "partial" + FAILED = "failed" + CANCELLED = "cancelled" + + +class NodeEventKind(StrEnum): + """Actual execution drives success and transitions into or out of failure.""" + + SUCCESS = "success" + FAILED = "failed" + RECOVERED = "recovered" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py index 4fe9306891..ebb83d9c96 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py @@ -146,7 +146,10 @@ def test_given_unchanged_project_when_planning_after_build_then_models_are_not_q try: stale_row: tuple[object, ...] = tuple(fingerprint_rows[0]) connection.execute( - "INSERT INTO main._sqlbuild_fingerprints VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO main._sqlbuild_fingerprints " + "(node_type, node_name, target_database, target_schema, target_name, run_id, " + "definition_hash, version_hash, schema_fingerprint, definition_b64, " + "metadata_json_b64, ts) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( stale_row[0], stale_row[1], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index caaf69d744..801b9dc17d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -5318,7 +5318,11 @@ def run_delta_compile( project_dir=project_dir, args=args, env={REUSE_DISABLE_ENV_VAR: "0", **env} ) return DeltaCompileOutcome( - returncode=result.returncode, stdout=result.stdout, stderr=result.stderr + returncode=result.returncode, + stdout=result.stdout, + stderr=result.stderr, + command=tuple(result.args), + environment=tuple(sorted({REUSE_DISABLE_ENV_VAR: "0", **env}.items())), ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py index 5786204a87..87b79f8819 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py @@ -2,9 +2,11 @@ from __future__ import annotations +import json import shutil from functools import partial from pathlib import Path +from typing import Any import pytest @@ -262,6 +264,60 @@ def test_given_seeded_edit_series_when_compiling_each_edit_then_deltas_match_ful assert series.deltas >= test_case.expected_minimum_deltas, series.steps +@pytest.mark.parametrize( + "test_case", + [ + DeltaCompileSeriesTestCase( + description="cache mode diagnostic control", + seed=16, + steps=1, + json_output=True, + adapter=_UNTYPED_ADAPTER, + expected_minimum_deltas=1, + ) + ], + ids=lambda case: case.description, +) +def test_given_cache_mode_mismatch_when_comparing_then_keeps_actual_cli_evidence( + tmp_path: Path, + test_case: DeltaCompileSeriesTestCase, +) -> None: + project_dir: Path = tmp_path / "orders" + destination: Path = tmp_path / "private-evidence" + write_delta_compile_project(project_dir=project_dir, model_count=30, adapter=test_case.adapter) + + series: DeltaCompileSeries = run_delta_compile_series( + project_dir=project_dir, + seed=test_case.seed, + steps=test_case.steps, + compile_project=partial( + run_delta_compile, project_dir=project_dir, args=_JSON_COMPILE, env={} + ), + compile_in_full=partial(run_full_compile, args=(*_JSON_COMPILE, "--no-cache")), + diagnostics_dir=destination, + case=test_case.description, + ) + + assert series.deltas >= test_case.expected_minimum_deltas + assert len(series.unexpected) == 1 + assert "stderr differs from a full compile" in series.unexpected[0] + directory: Path = destination / "failure-00" + manifest: dict[str, Any] = json.loads((directory / "manifest.json").read_text()) + assert manifest["runs"]["delta"]["returncode"] == manifest["runs"]["full"]["returncode"] == 0 + assert ( + manifest["runs"]["delta"]["semantic_json_sha256"] + == manifest["runs"]["full"]["semantic_json_sha256"] + ) + assert manifest["runs"]["full"]["command"][-3:] == ["compile", "--json", "--no-cache"] + assert "SQL test planning START" in (directory / "full.stderr").read_text() + assert "SQL test planning START" not in (directory / "delta.stderr").read_text() + delta: dict[str, Any] = json.loads((directory / "delta.artifacts.json").read_text())["files"] + full: dict[str, Any] = json.loads((directory / "full.artifacts.json").read_text())["files"] + assert {path: state["sha256"] for path, state in delta.items()} == { + path: state["sha256"] for path, state in full.items() + } + + @pytest.mark.parametrize( "test_case", [ @@ -291,6 +347,8 @@ def test_given_seeded_edit_series_when_compiling_each_edit_then_output_matches_f steps=test_case.steps, compile_project=partial(run_delta_compile, project_dir=project_dir, args=args, env={}), compile_in_full=partial(run_full_compile, args=args), + diagnostics_dir=tmp_path / "private-comparison-evidence", + case=test_case.description, ) assert series.unexpected == test_case.expected_unexpected_outcomes, series.steps diff --git a/tests/goldens/compiler/failure/target-unknown-key.json b/tests/goldens/compiler/failure/target-unknown-key.json index 036d4c22ef..457bf232d0 100644 --- a/tests/goldens/compiler/failure/target-unknown-key.json +++ b/tests/goldens/compiler/failure/target-unknown-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: clone, compile_cache, concurrency, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", + "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: catalog_connection, clone, compile_cache, concurrency, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", "help": "did you mean 'warehouses'?" } ] diff --git a/tests/integration/scripts/fensu_policy/__init__.py b/tests/integration/scripts/fensu_policy/__init__.py new file mode 100644 index 0000000000..6aab21b701 --- /dev/null +++ b/tests/integration/scripts/fensu_policy/__init__.py @@ -0,0 +1 @@ +"""Integration checks for repository-authored Fensu policies.""" diff --git a/tests/integration/scripts/fensu_policy/_test_types.py b/tests/integration/scripts/fensu_policy/_test_types.py new file mode 100644 index 0000000000..ef287d1b65 --- /dev/null +++ b/tests/integration/scripts/fensu_policy/_test_types.py @@ -0,0 +1,10 @@ +"""Native boundary metadata cases for the real Fensu CLI.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class NativeBoundaryTestCase: + description: str + metadata: str + expected_fault_count: int diff --git a/tests/integration/scripts/fensu_policy/test_native_boundary.py b/tests/integration/scripts/fensu_policy/test_native_boundary.py new file mode 100644 index 0000000000..1b996fae04 --- /dev/null +++ b/tests/integration/scripts/fensu_policy/test_native_boundary.py @@ -0,0 +1,78 @@ +"""Exercise native boundary policy through Fensu's real Rust analyzer and rule host.""" + +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from tests.integration.scripts.fensu_policy._test_types import NativeBoundaryTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + NativeBoundaryTestCase( + description="pyclass module metadata", + metadata='#[pyclass(frozen, module = "sqlbuild._native")]\npub struct Snapshot;\n', + expected_fault_count=2, + ), + NativeBoundaryTestCase( + description="namespaced pyclass raw module metadata", + metadata='#[pyo3::pyclass(frozen, module = r#"sqlbuild._native"#)]\npub struct Snapshot;\n', + expected_fault_count=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_pyclass_metadata_when_checking_rust_then_only_real_python_crossings_fault( + tmp_path: Path, + test_case: NativeBoundaryTestCase, +) -> None: + repository: Path = Path(__file__).resolve().parents[4] + shutil.copytree( + repository / "scripts/fensu_policy", + tmp_path / "scripts/fensu_policy", + ignore=shutil.ignore_patterns("__pycache__"), + ) + (tmp_path / "scripts/__init__.py").write_text("") + (tmp_path / "Cargo.toml").write_text( + '[workspace]\nmembers = ["crates/sqlbuild-python"]\nresolver = "2"\n' + ) + crate: Path = tmp_path / "crates/sqlbuild-python" + (crate / "src").mkdir(parents=True) + (crate / "Cargo.toml").write_text( + '[package]\nname = "sqlbuild-python"\nversion = "0.1.0"\nedition = "2021"\n' + ) + (tmp_path / "fensu.toml").write_text( + '[targets.rust]\nanalyzer = "rust"\nroots = ["crates"]\n' + 'rule_modules = ["scripts.fensu_policy.rules.native_boundary"]\n' + 'select = ["XSB072"]\n' + ) + source: Path = crate / "src/lib.rs" + source.write_text(test_case.metadata) + command: list[str] = [ + str(Path(sys.executable).with_name("fensu")), + "check", + "--target", + "rust", + "--no-cache", + "--no-color", + ] + clean: subprocess.CompletedProcess[str] = subprocess.run( + command, cwd=tmp_path, capture_output=True, text=True, timeout=15, check=False + ) + assert clean.returncode == 0, clean.stdout + clean.stderr + assert "Found 0 faults" in clean.stdout + source.write_text( + test_case.metadata + 'fn read(py: Python<\'_>) { py.import("sqlbuild.compiler"); }\n' + ) + crossing: subprocess.CompletedProcess[str] = subprocess.run( + command, cwd=tmp_path, capture_output=True, text=True, timeout=15, check=False + ) + assert crossing.returncode == 1, crossing.stdout + crossing.stderr + assert crossing.stdout.count("XSB072") == test_case.expected_fault_count, crossing.stdout + assert crossing.stdout.count(" --> crates/sqlbuild-python/src/lib.rs:3:0") == 2 + assert " --> crates/sqlbuild-python/src/lib.rs:1:0" not in crossing.stdout + assert source.read_text().startswith(test_case.metadata) diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py b/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py index 492b5c02ef..d6b97a4713 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py @@ -1,3 +1,4 @@ +from contextlib import AbstractContextManager from dataclasses import dataclass, field from sqlbuild._native import InferredNullability, SeedCsvSettings @@ -41,6 +42,15 @@ class ConnectTestCase: expected_connects: bool +@dataclass(frozen=True) +class ContendedConnectTestCase: + description: str + read_only: bool + release_key: str + expected_connection: AbstractContextManager[object] + expected_rows: list[tuple[int]] + + @dataclass(frozen=True) class ConnectSettingsTestCase: description: str diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py index d7c27e5bf5..607354a7fb 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py @@ -1,7 +1,12 @@ from __future__ import annotations +import json +import subprocess +import sys +from contextlib import nullcontext from decimal import Decimal from pathlib import Path +from threading import Timer from typing import Any import duckdb @@ -32,6 +37,7 @@ CompositeRowDiffSamplingTestCase, ConnectSettingsTestCase, ConnectTestCase, + ContendedConnectTestCase, DeleteInsertTestCase, DiffRowsErrorTestCase, DiffRowsTestCase, @@ -172,6 +178,80 @@ def test_given_config_when_connecting_then_returns_usable_connection( adapter.close(connection) +@pytest.mark.parametrize( + "test_case", + [ + ContendedConnectTestCase( + description="writer waits for a catalog reader", + read_only=False, + release_key="release_after_seconds", + expected_connection=nullcontext(), + expected_rows=[(23,)], + ), + ContendedConnectTestCase( + description="catalog reader waits for a writer", + read_only=True, + release_key="release_after_seconds", + expected_connection=nullcontext(), + expected_rows=[(23,)], + ), + ContendedConnectTestCase( + description="persistent contention remains an error", + read_only=False, + release_key="persistent_release_after_seconds", + expected_connection=pytest.raises(duckdb.IOException, match="lock|already open"), + expected_rows=[(23,)], + ), + ], + ids=lambda case: case.description, +) +def test_given_catalog_contention_when_connecting_then_temporary_locks_wait_and_persistent_locks_fail( + test_case: ContendedConnectTestCase, adapter: DuckDbAdapter, tmp_path: Path +) -> None: + inputs: dict[str, Any] = json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/local_catalog_contention.json" + ).read_text() + ) + database: Path = tmp_path / "orders.duckdb" + connection: Any = adapter.connect({"database": str(database)}) + adapter.execute(connection=connection, sql=inputs["setup_sql"]) + adapter.close(connection) + with subprocess.Popen( + [ + sys.executable, + "-c", + "import duckdb,sys; c=duckdb.connect(sys.argv[1],read_only=sys.argv[2]=='true'); " + "print('ready',flush=True); sys.stdin.read(1); c.close()", + str(database), + str(not test_case.read_only).lower(), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + ) as holder: + assert holder.stdout is not None + assert holder.stdout.readline().strip() == "ready" + release: Timer = Timer( + inputs[test_case.release_key], + holder.communicate, + kwargs={"input": "\n", "timeout": 5}, + ) + release.start() + try: + with test_case.expected_connection: + connection = adapter.connect( + {"database": str(database), "read_only": test_case.read_only} + ) + try: + assert connection.execute(inputs["query"]).fetchall() == test_case.expected_rows + finally: + adapter.close(connection) + finally: + release.join() + assert holder.returncode == 0 + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py b/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py index 09ec18f774..64b140c19c 100644 --- a/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py @@ -219,3 +219,10 @@ class SqliteUnkeyedSampleTestCase: limit: int expected_row_count: int expected_order_ids: frozenset[object] + + +@dataclass(frozen=True) +class SqliteStatisticsTestCase: + description: str + fixture_name: str + expected_interactive_row_count: int diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py b/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py index 389f56d9ed..5afc6b4e14 100644 --- a/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from pathlib import Path from typing import Any @@ -9,6 +10,7 @@ from sqlbuild.adapter.contract.models import ( ColumnInfo, RelationInfo, + RelationRowCountEstimate, RowDiffResult, RowDiffSampleRow, ) @@ -25,6 +27,7 @@ SqliteSchemaEvolutionTestCase, SqliteSeedTestCase, SqliteStateTableTestCase, + SqliteStatisticsTestCase, SqliteStrategyTestCase, SqliteSwapTestCase, SqliteTypedTableTestCase, @@ -766,3 +769,49 @@ def test_given_view_when_renaming_then_sqlite_recreates_it( ) == test_case.expected_names ) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteStatisticsTestCase( + description="background metadata avoids scans while interactive counts remain exact", + fixture_name="sqlite", + expected_interactive_row_count=3, + ), + ], + ids=lambda case: case.description, +) +def test_given_sqlite_rows_when_sampling_metadata_then_unknown_without_scanning( + test_case: SqliteStatisticsTestCase, adapter: SqliteAdapter, connection: Any +) -> None: + fixture: Any = json.loads( + ( + Path(__file__).resolve().parents[6] + / "benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json" + ).read_text() + )[test_case.fixture_name] + run_sql(adapter=adapter, connection=connection, statements=tuple(fixture["setup_sql"])) + statements: list[str] = [] + connection.raw_connection.set_trace_callback(statements.append) + + metadata: RelationRowCountEstimate = adapter.estimate_relation_row_count( + connection=connection, + database=None, + schema="main", + name=fixture["name"], + metadata_only=True, + ) + + assert metadata.row_count is None + assert metadata.detail == fixture["metadata_detail"] + assert metadata.byte_count is None + assert metadata.byte_count_basis is None + assert statements == [] + + interactive: RelationRowCountEstimate = adapter.estimate_relation_row_count( + connection=connection, database=None, schema="main", name=fixture["name"] + ) + + assert interactive.row_count == test_case.expected_interactive_row_count + assert interactive.detail is None diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index bfb3ca0eab..9fd65cab7a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -4,6 +4,14 @@ from dataclasses import dataclass, field +from sqlbuild.orchestration_declarations.models.authoring import ( + CalendarSchedule, + Job, + JobGroup, + Schedule, + Sensor, +) + @dataclass(frozen=True) class HeaderProjectionTestCase: @@ -190,3 +198,82 @@ class QuotedHookBoundaryTestCase: description: str argument: str expected_sql: str + + +@dataclass(frozen=True) +class SnapshotCatalogTestCase: + description: str + files: dict[str, str] + expected_kinds: frozenset[str] + + +@dataclass(frozen=True) +class SnapshotIdentityTestCase: + description: str + path: str + before: str + after: str + expected_changed: list[str] + + +@dataclass(frozen=True) +class SnapshotRejectionTestCase: + description: str + files: dict[str, str] + nodes: dict[str, object] + expected_compiler_errors: bool + expected_message: str + + +@dataclass(frozen=True) +class SnapshotEnvironmentTestCase: + description: str + first_lockfile: str + second_lockfile: str + expected_environment_count: int + + +@dataclass(frozen=True) +class SnapshotScheduleTestCase: + description: str + schedule: Schedule + expected_selection: tuple[str, ...] + expected_controls: dict[str, object] = field( + default_factory=lambda: { + "starts_at": None, + "default_status": "running", + } + ) + + +@dataclass(frozen=True) +class SnapshotDeclarationRejectionTestCase: + description: str + job_description: str = "Orders." + select: str = "orders" + cron: str = "0 2 * * *" + timezone: str = "UTC" + settings: dict[str, object] = field(default_factory=dict) + concurrency: int | None = None + copies: int = 1 + expected_message: str = "" + schedule: Schedule | None = None + + +@dataclass(frozen=True) +class SnapshotDiscoveryTestCase: + description: str + files: dict[str, str] + expected_jobs: tuple[str, ...] + declarations: tuple[Job | JobGroup | CalendarSchedule | Sensor, ...] | None = None + expected_groups: dict[str, dict[str, list[str]]] = field(default_factory=dict) + expected_sensors: dict[str, dict[str, object]] = field(default_factory=dict) + expected_schedules: tuple[str, ...] = () + + +@dataclass(frozen=True) +class SnapshotDiscoveryRejectionTestCase: + description: str + files: dict[str, str] + expected_error: type[Exception] + expected_message: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index f351592f4d..024306d0a4 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -8,7 +8,8 @@ from collections.abc import Callable, Iterator from pathlib import Path from types import ModuleType -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any +from uuid import UUID import pytest @@ -47,6 +48,20 @@ if TYPE_CHECKING: from sqlbuild._native import CompileInputsView +METADATA: dict[str, Any] = { + "location_id": str(UUID(int=1)), + "target": "development", + "generation": 1, + "git_revision": None, + "git_dirty": False, + "git_author": None, + "image_id": None, + "environment_hash": "test-environment", + "sqb_version": "not-the-installed-version", + "nodes": {}, + "documents": {}, +} + TYPE_PROOF_FILES: dict[str, str] = { "sqlbuild_project.toml": ( 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRCONTRACT105"]\n\n' diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/__init__.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/__init__.py new file mode 100644 index 0000000000..75debee041 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/__init__.py @@ -0,0 +1 @@ +"""Retained compiler snapshots published through the real native control API.""" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py new file mode 100644 index 0000000000..0b5cd7e114 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -0,0 +1,313 @@ +"""Publication cases and isolated server credentials.""" + +from collections.abc import Callable +from contextlib import AbstractContextManager +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from subprocess import Popen +from typing import Any, Literal, NamedTuple + + +class SnapshotApi(NamedTuple): + url: str + location_id: str + bearer: str + + def __repr__(self) -> str: + return f"SnapshotApi(url={self.url!r}, location_id={self.location_id!r}, bearer=)" + + +class ClaimedWorker(NamedTuple): + run_id: str + claim_id: str + bearer: str + + +class ScheduledWorker(NamedTuple): + api: SnapshotApi + project: Path + control_store: Path + process: Popen[bytes] + log: Path + docker: "DockerLocation | None" = None + + +class DockerLocation(NamedTuple): + image_id: str + container: str + socket: Path + + +class WorkerDeployment(NamedTuple): + kind: str + configuration: dict[str, Any] + publication_arguments: tuple[str, ...] = () + docker: DockerLocation | None = None + + +@dataclass(frozen=True) +class SnapshotPublicationTestCase: + description: str + target: str + expected_generation: int + expected_nodes: frozenset[str] + + +@dataclass(frozen=True) +class SnapshotWorkerTestCase: + description: str + model_file: str + original_sql: str + replacement_sql: str + expected_error: str + + +@dataclass(frozen=True) +class LocationReloadTestCase: + description: str + target: str + expected_generation: int + expected_stop_code: int + database: str + expected_connection: bool + + +@dataclass(frozen=True) +class LocationShutdownTestCase: + description: str + signal: int + expected_exit_code: int + minimum_heartbeats: int + blocked_command: str + + +@dataclass(frozen=True) +class LocationDatabaseTestCase: + description: str + database: str + expected_generation: int + connection_toml: str + + +@dataclass(frozen=True) +class LocationWorkTestCase: + description: str + database: str + expected_connection: bool + + +@dataclass(frozen=True) +class WorkerExecutionTestCase: + description: str + model_sql: str + warehouse_query: str + expected_status: str + expected_rows: list[tuple[int]] + expected_node_count: int = 1 + expected_origin: Literal["schedule", "sensor"] = "schedule" + files: dict[str, str] = field(default_factory=dict) + files_after_publication: dict[str, str] = field(default_factory=dict) + store_setup: str = "" + concurrency_limits: dict[str, int] = field(default_factory=dict) + launcher: Literal["native", "uncooperative", "docker"] = "native" + + +@dataclass(frozen=True) +class CheckExecutionTestCase(WorkerExecutionTestCase): + description: str + expected_check_outcome: str = "success" + + +@dataclass(frozen=True, kw_only=True) +class PersonJourneyTestCase(WorkerExecutionTestCase): + description: str + default_role: Literal["none", "viewer"] + expected_metadata_access: Callable[[], AbstractContextManager[Any]] + + +@dataclass(frozen=True) +class GroupExecutionTestCase(WorkerExecutionTestCase): + description: str + expected_child_statuses: dict[str, tuple[str, ...]] = field(default_factory=dict) + expected_totals: tuple[str, str | None] = ("skipped", "model:broken_orders") + expected_marts_started: bool = True + + +@dataclass(frozen=True) +class WorkerRestartTestCase(WorkerExecutionTestCase): + description: str + shutdown_signal: int = 15 + expected_scheduler_exit: int = 0 + + +@dataclass(frozen=True) +class BackfillExecutionTestCase(WorkerExecutionTestCase): + description: str + window_arguments: tuple[str, ...] = () + expected_window: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class CalendarRejectionTestCase: + description: str + expression: str + expected_message: str + horizon: str = "8s" + timezone: str = "UTC" + + +@dataclass(frozen=True) +class CalendarRefreshTestCase: + description: str + expected_timezone: str + + +@dataclass(frozen=True) +class WarehouseHistoryTestCase: + description: str + adapter_name: str + expected_page_lengths: tuple[int, ...] + connection_key: str + journal_modes: tuple[str, ...] + append_fingerprints: Callable[..., None] + + +@dataclass(frozen=True) +class CatalogConnectionTestCase: + description: str + adapter_name: str + connection_key: str + expected_run_id: str + + +@dataclass(frozen=True) +class StateEventsTestCase(CatalogConnectionTestCase): + description: str + expected_run_id: str + fixture_name: str + append_history: Callable[..., None] + legacy_columns: tuple[str, ...] = () + + +@dataclass(frozen=True) +class AutomaticHistoryTestCase: + description: str + adapter_name: str + connection_key: str + expected_run_id: str + wide: bool = False + initial_seconds: int = 8 + expected_average_bytes: int = 0 + + +@dataclass(frozen=True) +class FingerprintHistoryTestCase: + description: str + adapter_name: str + connection_key: str + observed_at: datetime + journal_modes: tuple[str, ...] + fixture_paths: tuple[Path, ...] + expected_legacy_runs: tuple[str, ...] + + +@dataclass(frozen=True) +class DevelopmentTestCase: + description: str + target: str + arguments: tuple[str, ...] + control_path: str + expected_rows: list[tuple[int, int]] + local_config: str = "" + + +@dataclass(frozen=True) +class ComposeDeploymentTestCase: + description: str + expected_warehouse_rows: list[list[int]] + + +@dataclass(frozen=True) +class InstalledRecoveryTestCase: + description: str + executable: str + expected_rejection: str + + +@dataclass(frozen=True) +class CatalogInspectionTestCase: + description: str + adapter_name: str + connection_key: str + expected_table_type: str + expected_view_type: str + expected_metadata_rows: int | None = None + + +@dataclass(frozen=True) +class CatalogBoundaryTestCase: + description: str + expected_page_delivered: bool + + +@dataclass(frozen=True) +class StatisticsRecoveryTestCase: + description: str + expected_log: str + + +@dataclass(frozen=True) +class StatisticsHorizonTestCase: + description: str + server_arguments: tuple[str, ...] + expected_relations: tuple[str, ...] + + +@dataclass(frozen=True) +class AuditHorizonTestCase: + description: str + fixture: str + expected_retained: int + + +@dataclass(frozen=True) +class RetentionRejectionTestCase: + description: str + arguments: tuple[str, ...] + expected_error: str + + +@dataclass(frozen=True) +class StatisticsHorizonRejectionTestCase: + description: str + days: str + expected_error: str + + +@dataclass(frozen=True) +class DenialMetadataTestCase: + description: str + expected_denial_count: int + + +@dataclass(frozen=True) +class CliLoginTestCase: + description: str + authentication: str + expected_person_name: str + + +@dataclass(frozen=True) +class CliCleanupTestCase: + description: str + server: str + expected_exit: int + + +@dataclass(frozen=True) +class SchedulerCrashTestCase: + description: str + fixture: str + expected_anomalies: dict[str, int] + expected_minimum_kills: int diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py new file mode 100644 index 0000000000..3a2de107cd --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py @@ -0,0 +1,47 @@ +"""Observe the installed location's catalog projection and statistics samples.""" + +import json +import sqlite3 +import time +from contextlib import closing +from pathlib import Path +from subprocess import Popen +from typing import Any + + +def wait_for_catalog( + control: Path, expected: list[dict[str, Any]], process: Popen[bytes], log: Path +) -> None: + """Wait for the exact synthetic relation, not a count that unrelated rows can satisfy.""" + deadline: float = time.monotonic() + 45 + observed: list[dict[str, Any]] = [{}] + while observed != expected: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, (observed, expected, log.read_text()) + with closing(sqlite3.connect(control)) as connection: + rows: list[tuple[str]] = connection.execute( + "SELECT record_json FROM sqb_warehouse_catalog " + "WHERE json_extract(record_json, '$.name')='orders_🧾'" + ).fetchall() + observed = [json.loads(row[0]) for row in rows] + time.sleep(0.05) + + +def wait_for_statistics( + control: Path, name: str, process: Popen[bytes], log: Path +) -> list[dict[str, Any]]: + """Observe actual metadata-only sampling through the installed location and HTTP ingestion.""" + deadline: float = time.monotonic() + 45 + records: list[dict[str, Any]] = [] + while not records: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + with closing(sqlite3.connect(control)) as connection: + rows: list[tuple[str]] = connection.execute( + "SELECT record_json FROM sqb_warehouse_statistics " + "WHERE relation_name=? ORDER BY observed_at,event_id", + (name,), + ).fetchall() + records = [json.loads(row[0]) for row in rows] + time.sleep(0.05) + return records diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/__init__.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/__init__.py new file mode 100644 index 0000000000..8d7411a51a --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/__init__.py @@ -0,0 +1 @@ +"""Real container launcher verification.""" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py new file mode 100644 index 0000000000..74b846395e --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py @@ -0,0 +1,316 @@ +"""Own a disposable packaged PostgreSQL/HTTP/Docker stack without logging credentials.""" + +from __future__ import annotations + +import json +import os +import secrets +import shutil +import subprocess +import time +from collections.abc import Iterator +from contextlib import contextmanager +from datetime import UTC, datetime, timedelta +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from threading import Thread +from typing import Any +from uuid import uuid4 + + +class ComposeStack: + """One test owns its containers, images, PostgreSQL volume, and warehouse volume.""" + + def __init__(self, directory: Path) -> None: + self.directory = directory + self.name = f"sqlbuild-compose-{uuid4().hex}" + self.location_id = "00000000-0000-0000-0000-000000000001" + password: str = secrets.token_hex(24) + self.environment = os.environ | { + "COMPOSE_PROJECT_NAME": self.name, + "SQLBUILD_PORT": "0", + "DOCKER_GID": str(Path("/var/run/docker.sock").stat().st_gid), + "POSTGRES_PASSWORD": password, + "SQLBUILD_CONTROL_DATABASE_URL": f"postgresql://sqlbuild:{password}@postgres:5432/sqlbuild", + "SQLBUILD_SERVER_IMAGE": os.environ.get( + "SQLBUILD_TEST_SERVER_IMAGE", "sqlbuild-server:test" + ), + "SQLBUILD_LOCATION_IMAGE": f"{self.name}:first", + "SQLBUILD_LOCATION_ID": self.location_id, + "SQLBUILD_GIT_REVISION": "compose-acceptance", + } + self.override = directory / "override.json" + self.override.write_text( + json.dumps( + {"services": {"server": {"extra_hosts": ["host.docker.internal:host-gateway"]}}} + ) + ) + self.compose_files = [Path("compose.yml").resolve(), self.override] + + def docker( + self, *arguments: str, check: bool = True, timeout: float = 120 + ) -> subprocess.CompletedProcess[str]: + started: float = time.monotonic() + result: subprocess.CompletedProcess[str] = subprocess.run( + ["docker", *arguments], + env=self.environment, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + with (self.directory / "commands.jsonl").open("a") as log: + log.write( + json.dumps( + { + "args": arguments, + "exit": result.returncode, + "seconds": time.monotonic() - started, + } + ) + + "\n" + ) + assert not check or result.returncode == 0, result.stderr + return result + + def compose(self, *arguments: str, check: bool = True) -> str: + configuration: list[str] = [] + for file in self.compose_files: + configuration.extend(("--file", str(file))) + return self.docker( + "compose", + *configuration, + *arguments, + check=check, + ).stdout.strip() + + def query(self, sql: str) -> list[dict[str, Any]]: + result: str = self.compose( + "exec", + "-T", + "postgres", + "psql", + "-U", + "sqlbuild", + "-d", + "sqlbuild", + "-At", + "-c", + f"SELECT COALESCE(json_agg(result),'[]'::json)::text FROM ({sql}) result", + ) + return json.loads(result) + + def database_manifest(self) -> dict[str, list[dict[str, Any]]]: + """Compare persisted rows without exposing credential hashes or payloads in failures.""" + tables: list[dict[str, Any]] = self.query( + "SELECT tablename,quote_ident(tablename) AS identifier FROM pg_tables " + "WHERE schemaname='public' ORDER BY tablename" + ) + return { + table["tablename"]: self.query( + "SELECT count(*) AS rows,md5(COALESCE(string_agg(row_to_json(t)::text," + "E'\\n' ORDER BY row_to_json(t)::text),'')) AS digest " + f"FROM public.{table['identifier']} t" + ) + for table in tables + } + + def verify_control_store_restore(self, snapshot: dict[str, Any]) -> None: + """Restore the populated stack into an empty store before allowing either writer back.""" + self.compose("stop", "--timeout", "5", "server") + manifest: dict[str, list[dict[str, Any]]] = self.database_manifest() + assert manifest["sqb_runs"][0]["rows"] > 0 + assert manifest["sqb_ticks"][0]["rows"] > 0 + assert manifest["sqb_access_tokens"][0]["rows"] > 0 + self.compose( + "exec", + "-T", + "postgres", + "pg_dump", + "-U", + "sqlbuild", + "--format=custom", + "--file=/tmp/control.backup", + "sqlbuild", + ) + self.compose("exec", "-T", "postgres", "dropdb", "-U", "sqlbuild", "sqlbuild") + self.compose("exec", "-T", "postgres", "createdb", "-U", "sqlbuild", "sqlbuild") + assert self.database_manifest() == {} + self.compose( + "exec", + "-T", + "postgres", + "pg_restore", + "-U", + "sqlbuild", + "--exit-on-error", + "--dbname=sqlbuild", + "/tmp/control.backup", + ) + assert self.database_manifest() == manifest + (self.directory / "restore-manifest.json").write_text(json.dumps(manifest, indent=2)) + self.compose("exec", "-T", "postgres", "rm", "/tmp/control.backup") + + # Retained credentials authenticate the broken location without replacing last-good. + self.redeploy("broken") + resumed_at: str = datetime.now(UTC).isoformat() + self.compose("up", "-d", "--wait", "server") + self.wait( + "SELECT event_id FROM recall_events WHERE event_type='location_health' " + "AND payload->'report'->'health'->>'kind'='compile_failed' " + f"AND occurred_at>'{resumed_at}'" + ) + self.wait( + f"SELECT run_id FROM sqb_runs WHERE origin='schedule' AND queued_at>'{resumed_at}' " + f"AND snapshot_id='{snapshot['snapshot_id']}' AND status='success'" + ) + assert ( + self.query( + "SELECT trigger_id,scheduled_for,count(*) FROM sqb_ticks " + "GROUP BY trigger_id,scheduled_for HAVING count(*)>1" + ) + == [] + ) + assert self.query( + "SELECT image_id,snapshot_id FROM sqb_snapshots " + "JOIN sqb_current_snapshots USING(snapshot_id,location_id,target)" + ) == [snapshot] + + def wait(self, sql: str, *, timeout: float = 150) -> list[dict[str, Any]]: + deadline: float = time.monotonic() + timeout + rows: list[dict[str, Any]] = self.query(sql) + while not rows: + assert time.monotonic() < deadline, f"Timed out: {sql} (logs in {self.directory})" + time.sleep(0.5) + rows = self.query(sql) + with (self.directory / "observations.jsonl").open("a") as log: + log.write( + json.dumps({"at": datetime.now(UTC).isoformat(), "query": sql, "rows": rows}) + "\n" + ) + return rows + + def build_projects(self) -> list[str]: + cases: dict[str, str] = json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/container_deployment.json" + ).read_text() + ) + base: str = os.environ.get("SQLBUILD_TEST_LOCATION_IMAGE", "sqlbuild-location:test") + images: list[str] = [] + for version, sql in cases.items(): + project: Path = self.directory / version + (project / "models").mkdir(parents=True) + (project / "orchestration").mkdir() + for relative in ("sqlbuild_project.toml", "orchestration/orders.py"): + shutil.copyfile(Path("deploy/project") / relative, project / relative) + (project / "models/orders.sql").write_text( + "MODEL (description 'Example orders.', materialized table);\n" + sql + "\n" + ) + # Only source enters this build context; credentials stay in the parent process. + (project / "Dockerfile").write_text( + f"FROM {base}\nENV SQLBUILD_GIT_REVISION={version}\n" + "COPY --chown=10001:10001 . /project/\n" + ) + self.docker("build", "--tag", f"{self.name}:{version}", str(project)) + images.append( + self.docker( + "image", "inspect", "--format", "{{.Id}}", f"{self.name}:{version}" + ).stdout.strip() + ) + return images + + def register(self) -> None: + self.compose("up", "-d", "--wait", "postgres") + location: dict[str, Any] = json.loads( + self.compose("run", "--rm", "--no-deps", "server", "admin", "create-location", "orders") + ) + self.location_id = location["location_id"] + self.environment["SQLBUILD_LOCATION_ID"] = self.location_id + expiry: str = (datetime.now(UTC) + timedelta(hours=1)).isoformat() + credential: dict[str, Any] = json.loads( + self.compose( + "run", + "--rm", + "--no-deps", + "server", + "admin", + "issue-location-token", + self.location_id, + "--expires-at", + expiry, + ) + ) + self.environment["SQLBUILD_LOCATION_TOKEN"] = credential["bearer"] + + def redeploy(self, version: str) -> None: + self.environment["SQLBUILD_LOCATION_IMAGE"] = f"{self.name}:{version}" + self.compose("up", "-d", "--no-deps", "location") + + def close(self) -> None: + self.compose("stop", "--timeout", "5", "server", "location", check=False) + for service in ("server", "location", "postgres"): + (self.directory / f"{service}.log").write_text( + self.compose("logs", "--no-color", service, check=False) + ) + workers: list[str] = self.docker( + "ps", "--all", "--quiet", "--filter", f"name=sqlbuild-{self.location_id}-" + ).stdout.split() + for worker in workers: + output: subprocess.CompletedProcess[str] = self.docker( + "logs", "--timestamps", worker, check=False + ) + (self.directory / f"worker-{worker}.log").write_text(output.stdout + output.stderr) + self.docker("rm", "--force", worker) + self.compose("down", "--volumes", "--remove-orphans") + for version in ("first", "second", "broken"): + self.docker("image", "rm", f"{self.name}:{version}", check=False) + + +@contextmanager +def platform_receiver() -> Iterator[tuple[int, list[dict[str, Any]]]]: + """Own the real webhook HTTP boundary even if stack cleanup itself fails.""" + events: list[dict[str, Any]] = [] + + class Receiver(BaseHTTPRequestHandler): + def do_POST(self) -> None: + event: dict[str, Any] = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + assert self.headers["Idempotency-Key"] == event["event_id"] + events.append(event) + self.send_response(204) + self.end_headers() + + def log_message(self, format: str, *args: object) -> None: + """Keep HTTP chatter out of test output.""" + + with ThreadingHTTPServer(("0.0.0.0", 0), Receiver) as receiver: + thread: Thread = Thread(target=receiver.serve_forever, daemon=True) + thread.start() + try: + yield receiver.server_port, events + finally: + receiver.shutdown() + thread.join(timeout=5) + assert not thread.is_alive() + + +def wait_for_location_outage( + events: list[dict[str, Any]], prior_events: set[str] +) -> dict[str, Any]: + """Wait for fresh delivery, not an outage already received during a prior restart.""" + deadline: float = time.monotonic() + 135 + matches: list[dict[str, Any]] = [] + while not matches: + assert time.monotonic() < deadline, ( + "Native platform sink did not report the stopped location" + ) + time.sleep(0.5) + matches = list( + filter( + lambda event: ( + event["event_id"] not in prior_events + and event["payload"]["platform"]["kind"] == "location_broken" + ), + events, + ) + ) + return matches[0] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py new file mode 100644 index 0000000000..e85570e8ee --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py @@ -0,0 +1,196 @@ +"""A real Docker location with the installed runtime and an isolated synthetic warehouse.""" + +import json +import os +import shutil +import socket +import subprocess +import sys +from collections.abc import Iterator +from contextlib import contextmanager +from http.client import HTTPConnection, HTTPResponse +from http.server import BaseHTTPRequestHandler +from pathlib import Path +from socketserver import UnixStreamServer +from threading import Event, Thread +from uuid import uuid4 + +import pytest + +import sqlbuild +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + DockerLocation, + WorkerDeployment, +) + +DOCKER_SOCKET: Path = Path(os.environ.get("SQLBUILD_TEST_DOCKER_SOCKET", "/var/run/docker.sock")) +REQUIRES_DOCKER: pytest.MarkDecorator = pytest.mark.skipif( + os.environ.get("SQLBUILD_REQUIRE_DOCKER_TESTS") != "1" + and (sys.platform != "linux" or shutil.which("docker") is None or not DOCKER_SOCKET.exists()), + reason="Docker worker tests require a Linux Docker daemon", +) + + +def docker_command(socket: Path, *arguments: str) -> str: + """Keep daemon errors useful without dumping a container's credential-bearing environment.""" + result: subprocess.CompletedProcess[str] = subprocess.run( + ["docker", "--host", f"unix://{socket}", *arguments], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=20, + check=False, + ) + assert result.returncode == 0, result.stdout + return result.stdout.strip() + + +@contextmanager +def docker_deployment(directory: Path, executable: str) -> Iterator[WorkerDeployment]: + """The worker inherits only these runtime/project mounts, never the store or daemon socket.""" + socket: Path = DOCKER_SOCKET + project: Path = directory / "project" + image: str = os.environ.get("SQLBUILD_TEST_WORKER_IMAGE", "alpine:3.23") + image_id: str = docker_command(socket, "image", "inspect", "--format", "{{.Id}}", image) + name: str = f"sqlbuild-test-location-{uuid4()}" + python: Path = Path(sys.executable) + installations: list[Path] = [Path(sys.prefix), Path(sys.base_prefix)] + while python.is_symlink(): + python = python.parent / python.readlink() + installations.append(python.parent.parent) + mounts: list[str] = [] + for path in filter( + Path.exists, + dict.fromkeys( + [ + Path("/usr"), + Path("/lib"), + Path("/lib64"), + *installations, + Path(sqlbuild.__file__).parent.parent, + ] + ), + ): + mounts.extend(["--volume", f"{path}:{path}:ro"]) + mounts.extend(["--volume", f"{project}:{project}:rw"]) + container: str = docker_command( + socket, + "run", + "--detach", + "--name", + name, + "--network", + "host", + "--user", + f"{os.getuid()}:{os.getgid()}", + "--workdir", + str(project), + "--env", + "HOME=/tmp", + "--env", + "SQLBUILD_WORKER_TEST_VALUE=orders-context", + "--env", + "SQLBUILD_LOCATION_TOKEN=synthetic-location-credential", + "--env", + "SQLBUILD_CONTROL_DATABASE_URL=synthetic-control-store", + "--env", + "SQLBUILD_PLATFORM_WEBHOOK_URL=http://127.0.0.1:1/events?token=synthetic-delivery-credential", + "--env", + "SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL=http://127.0.0.1:1/audit?token=synthetic-audit-credential", + "--env", + "SQLBUILD_SCHEDULER_TEST_STORE=synthetic-configured-store", + *mounts, + image_id, + "/usr/bin/sleep", + "300", + ) + try: + yield WorkerDeployment( + "docker", + {"socket": str(socket), "location_container": container, "executable": executable}, + ("--image-id", image_id), + DockerLocation(image_id, container, socket), + ) + finally: + containers: list[str] = docker_command( + socket, "ps", "--all", "--quiet", "--filter", f"volume={project}" + ).split() + for owned in containers: + _ = (directory / f"docker-{owned}.log").write_text( + docker_command(socket, "logs", owned) + ) + _ = docker_command(socket, "rm", "--force", owned) + + +def worker_container(location: DockerLocation, location_id: str, run_id: str) -> dict[str, object]: + """Return only physical identity/exit evidence, not Docker's secret-bearing configuration.""" + name: str = f"sqlbuild-{location_id}-{run_id}" + result: dict[str, object] = json.loads( + docker_command(location.socket, "inspect", "--format", "{{json .State}}", name) + ) + assert ( + docker_command(location.socket, "inspect", "--format", "{{.Image}}", name) + == location.image_id + ) + return result + + +@contextmanager +def lost_start_response(directory: Path, upstream: Path, inspection: Event) -> Iterator[Path]: + """Fail the real start result and hold physical inspection until explicitly restored.""" + proxy: Path = directory / "docker-proxy.sock" + + class Handler(BaseHTTPRequestHandler): + def forward(self) -> None: + connection: HTTPConnection = HTTPConnection("localhost", timeout=5) + connection.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + connection.sock.settimeout(5) + connection.sock.connect(str(upstream)) + try: + body: bytes = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + connection.request( + self.command, self.path, body, {"Content-Type": "application/json"} + ) + response: HTTPResponse = connection.getresponse() + payload: bytes = response.read() + status: int = response.status + started: bool = self.command == "POST" and self.path.endswith("/start") + unavailable: bool = ( + self.command == "GET" + and self.path.startswith("/containers/sqlbuild-") + and not inspection.is_set() + ) + assert not started or status == 204, ( + "The actual Docker start must succeed before faulting" + ) + status, payload = { + (False, False): (status, payload), + (True, False): (503, b'{"message":"synthetic lost start response"}'), + (False, True): (503, b'{"message":"synthetic inspection outage"}'), + }[(started, unavailable)] + self.send_response(status) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + finally: + connection.close() + + def do_GET(self) -> None: + self.forward() + + def do_POST(self) -> None: + self.forward() + + def log_message(self, format: str, *args: object) -> None: + """Never copy credential-bearing Docker requests to test output.""" + + with UnixStreamServer(str(proxy), Handler) as server: + thread: Thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield proxy + finally: + server.shutdown() + thread.join(timeout=6) + assert not thread.is_alive(), "Docker fault proxy did not stop" + proxy.unlink(missing_ok=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py new file mode 100644 index 0000000000..ba156087b9 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -0,0 +1,1221 @@ +"""A standalone native API and real SQLite store, with test-only credential provisioning.""" + +from __future__ import annotations + +import hashlib +import json +import os +import random +import secrets +import signal +import sqlite3 +import subprocess +import sys +import time +from collections.abc import Callable, Iterator +from contextlib import AbstractContextManager, closing, contextmanager, nullcontext, suppress +from dataclasses import replace +from datetime import UTC, datetime, timedelta +from io import StringIO +from pathlib import Path +from typing import Any, Literal, cast +from urllib.error import HTTPError +from urllib.parse import quote +from urllib.request import Request, urlopen +from uuid import uuid4 + +import duckdb +import pytest + +from sqlbuild._native import ( + Fingerprint, + MigrationDecision, + MigrationDiscovery, + MigrationEvent, + MigrationRelation, + OldNameViewEvent, + OldNameViewEventType, + write_fingerprint, + write_migration_state_event, +) +from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter +from sqlbuild.executor.audit_results.main._write import write_audit_result_records +from sqlbuild.executor.audit_results.models import AuditResultRecord +from sqlbuild.executor.janitor_events.classes.janitor_event_writer import JanitorEventWriter +from sqlbuild.executor.janitor_events.main.create_table_sql import ( + build_janitor_events_create_table_sql, +) +from sqlbuild.executor.janitor_events.models import JanitorEventRecord +from sqlbuild.executor.janitor_events.types import JanitorEventType +from sqlbuild.microbatches.classes.direct_store import DirectMicrobatchEventStore +from sqlbuild.microbatches.classes.event_codec import MicrobatchEventCodec +from sqlbuild.microbatches.constants import MICROBATCH_COLUMNS +from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchWriteResult +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ClaimedWorker, + ScheduledWorker, + SnapshotApi, + WorkerDeployment, + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.docker.helpers import ( + docker_deployment, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files +from tests.integration.src.sqlbuild.executor.audit_results.main.helpers import build_record + + +class RejectedPublicationOutput(StringIO): + """A caller loses only the result stream, after the real API accepts a publication.""" + + def write(self, text: str) -> int: + raise OSError("synthetic publication result delivery failure") + + +@contextmanager +def location_process( + arguments: list[str], log: Path, publications: Path, *, env: dict[str, str] | None = None +) -> Iterator[subprocess.Popen[bytes]]: + """Own and reap the supervisor even when an assertion interrupts a reload.""" + with log.open("ab") as stderr, publications.open("ab") as stdout: + process: subprocess.Popen[bytes] = subprocess.Popen( + arguments, stderr=stderr, stdout=stdout, env=env + ) + try: + yield process + finally: + process.terminate() + try: + _ = process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + _ = process.wait(timeout=5) + + +def stop_location(process: subprocess.Popen[bytes], interpreter_pid: int, log: Path) -> None: + """Exercise graceful supervisor shutdown and prove its interpreter was reaped.""" + process.send_signal(signal.SIGTERM) + assert process.wait(timeout=15) == 0, log.read_text() + with pytest.raises(ProcessLookupError): + os.kill(interpreter_pid, 0) + + +def open_legacy_warehouse(project: Path, adapter_name: str, connection_key: str) -> tuple[Any, Any]: + """The caller owns the returned real connection; declarations must never run during reads.""" + write_files( + project_dir=project, + files={ + "sqlbuild_project.toml": ( + f'name = "orders"\nadapter = "{adapter_name}"\n' + f'[connections.local]\n{connection_key} = "warehouse.db"\n' + '[targets.development]\nschema = "main"\n' + ), + "orchestration/orders.py": 'raise AssertionError("unexpected declaration import")\n', + }, + ) + adapter: Any = resolve_adapter(adapter_name=adapter_name) + connection: Any = adapter.connect({connection_key: str(project / "warehouse.db")}) + legacy_sql: str = Path( + "benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql" + ).read_text() + for statement in legacy_sql.rstrip().removesuffix(";").split(";"): + _ = adapter.execute(connection=connection, sql=statement) + return adapter, connection + + +def append_legacy_history( + adapter: Any, connection: Any, rows: list[dict[str, str]], statements: list[str] +) -> None: + """Append synthetic arrivals with independent observation times to real history tables.""" + for row in rows: + for statement in statements: + _ = adapter.execute(connection=connection, sql=statement.format(**row)) + + +def append_audit_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: + """Use the public warehouse writer, including duplicate IDs and an external CLI run.""" + record: AuditResultRecord = replace( + build_record(), + run_id="", + evidence_json=fixture["record"]["evidence_json"], + evidence_count=1, + evidence_truncated=True, + audit_meta_json=fixture["record"]["audit_meta_json"], + ) + records: list[AuditResultRecord] = [ + replace(record, result_id=f"result-{index:03}") for index in range(fixture["source_rows"]) + ] + records.extend([records[0]] * fixture["duplicate_rows"]) + records.append(replace(records[0], occurred_at=datetime.fromisoformat(fixture["later_time"]))) + write_audit_result_records( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + records=records, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + render_create_table_sql=adapter.render_create_audit_result_table_sql, + render_create_index_sqls=adapter.render_create_audit_result_index_sqls, + ) + + +def append_janitor_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: + """Real chunked archive/delete writes, including repeated write attempts.""" + data: dict[str, Any] = fixture["record"].copy() + del data["state_database"] + del data["state_schema"] + for key in ("occurred_at", "archived_at"): + data[key] = datetime.fromisoformat(data[key]) + data["event_type"] = JanitorEventType(data["event_type"]) + record: JanitorEventRecord = JanitorEventRecord(**data) + records: list[JanitorEventRecord] = [ + replace(record, event_id=f"event-{index:03}") for index in range(fixture["source_rows"]) + ] + records[-1] = replace( + records[-1], + event_type=JanitorEventType.DELETE, + occurred_at=datetime.fromisoformat(fixture["later_time"]), + ) + _ = adapter.execute( + connection=connection, + sql=build_janitor_events_create_table_sql( + database=None, + schema="main", + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + ), + ) + writer: JanitorEventWriter = JanitorEventWriter( + execute=adapter.execute, + render_qualified_name=adapter.render_qualified_name, + run_id=record.run_id, + ) + writer.write(connection=connection, records=tuple(records)) + writer.write(connection=connection, records=(records[0],) * fixture["duplicate_rows"]) + + +def append_microbatch_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: + """Real append-only coordination writer; reading never enters the executor's state path.""" + rows: list[dict[str, Any]] = [ + fixture["record"] + | {"event_id": f"{fixture['id_prefix']}-{index:03}"} + | fixture["record_overrides"].get(f"{fixture['id_prefix']}-{index:03}", {}) + for index in range(fixture["source_rows"]) + ] + records: list[MicrobatchEvent] = [] + for row in rows: + values: tuple[Any, ...] = tuple(row[column] for column in MICROBATCH_COLUMNS) + records.append(MicrobatchEventCodec.from_row(values)) + writer: DirectMicrobatchEventStore = DirectMicrobatchEventStore( + adapter=adapter, connection=connection + ) + first: MicrobatchWriteResult = writer.write_many(tuple(records)) + assert first.inserted == fixture["source_rows"] + repeated: MicrobatchWriteResult = writer.write_many((records[0],) * fixture["duplicate_rows"]) + assert repeated.inserted == 0 + + +def append_migration_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: + """Real model-move facts, including idempotent duplicate attempts.""" + data: dict[str, Any] = migration_history_data(fixture, ("origin", "destination")) + data["discovery"] = MigrationDiscovery(data["discovery"]) + data["decision"] = MigrationDecision(data["decision"]) + record: MigrationEvent = MigrationEvent(**data) + write_migration_history( + adapter, + connection, + fixture, + record, + "model", + adapter.render_create_migration_state_table_sql(database=None, schema="main"), + {}, + ) + + +def append_old_name_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: + """Real compatibility-view facts, including nullable and populated expiry.""" + data: dict[str, Any] = migration_history_data(fixture, ("old", "new")) + data["event_type"] = OldNameViewEventType(data["event_type"]) + data["column_aliases"] = () + record: OldNameViewEvent = OldNameViewEvent(**data) + write_migration_history( + adapter, + connection, + fixture, + record, + "old_name", + adapter.render_create_old_name_view_state_table_sql(database=None, schema="main"), + { + "column_aliases": tuple(json.loads(fixture["last_record"]["column_aliases"]).items()), + "expires_at": datetime.fromisoformat(fixture["last_record"]["expires_at"]), + "view_sql": fixture["last_record"]["view_sql"], + }, + ) + + +def migration_history_data(fixture: dict[str, Any], relations: tuple[str, str]) -> dict[str, Any]: + """Reconstruct native relation values from the warehouse-shaped fixture.""" + data: dict[str, Any] = fixture["record"].copy() + del data["state_database"] + del data["state_schema"] + data["created_at"] = datetime.fromisoformat(data["created_at"]) + for relation in relations: + data[relation] = MigrationRelation( + database=data.pop(f"{relation}_database"), + schema=data.pop(f"{relation}_schema"), + name=data.pop(f"{relation}_name"), + ) + return data + + +def write_migration_history( + adapter: Any, + connection: Any, + fixture: dict[str, Any], + record: MigrationEvent | OldNameViewEvent, + kind: Literal["model", "old_name"], + create_sql: str, + last_values: dict[str, Any], +) -> None: + """Exercise the shared immutable writer with paginated, retried source facts.""" + records: list[Any] = [ + replace(record, event_id=f"{fixture['id_prefix']}-{index:03}") + for index in range(fixture["source_rows"]) + ] + records[-1] = replace( + records[-1], created_at=datetime.fromisoformat(fixture["later_time"]), **last_values + ) + records.extend([records[0]] * fixture["duplicate_rows"]) + for record in records: + write_migration_state_event( + kind=kind, + connection=connection, + execute=adapter.execute, + event=record, + render_qualified_name=adapter.render_qualified_name, + create_table_sql=create_sql, + ) + + +def warehouse_history_page( + project: Path, + table: str, + after: list[str | None] | None = None, + observed_since: str | None = None, +) -> Any: + """Exercise the real isolated reader without a server, state database or declaration import.""" + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "runtime", + "--server", + "http://127.0.0.1:1", + "--location-id", + "00000000-0000-0000-0000-000000000001", + "--target", + "development", + ], + input=json.dumps( + { + table: { + "location": {"database": None, "schema": "main"}, + "after": after, + "observed_since": observed_since, + } + } + ) + + "\n", + text=True, + capture_output=True, + timeout=20, + env=os.environ | {"SQLBUILD_CONTROL_DATABASE_URL": "unsupported://must-not-open"}, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout) + + +def warehouse_history_pages( + project: Path, table: str, observed_since: str | None = None +) -> list[dict[str, Any]]: + """Follow bounded source pages, including empty filtered pages with a continuation.""" + pages: list[dict[str, Any]] = [] + after: list[str | None] | None = None + more: bool = True + while more: + page: dict[str, Any] = warehouse_history_page(project, table, after, observed_since) + assert page is not None + pages.append(page) + after = page["next_after"] + more = after is not None + assert len(pages) <= 4, "history cursor must advance past duplicate physical rows" + return pages + + +def sync_warehouse_history(api: SnapshotApi, batch: dict[str, Any]) -> dict[str, Any]: + """Deliver one metadata page through the public location transport.""" + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/targets/development/metadata", + data=json.dumps(batch).encode(), + headers={"Authorization": f"Bearer {api.bearer}", "Content-Type": "application/json"}, + ) + with urlopen(request, timeout=5) as response: + result: dict[str, Any] = json.load(response) + return result + + +def wait_for_generation( + api: SnapshotApi, target: str, generation: int, process: subprocess.Popen[bytes], log: Path +) -> dict[str, Any]: + deadline: float = time.monotonic() + 15 + current: dict[str, Any] = {"record": {"generation": 0}} + while current["record"]["generation"] < generation: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, ( + f"Generation {generation} was not published: {log.read_text()}" + ) + try: + current = current_snapshot(api, target) + except HTTPError as error: + assert error.code == 404 + time.sleep(0.02) + assert current["record"]["generation"] == generation, log.read_text() + return current + + +def wait_for_log( + process: subprocess.Popen[bytes], log: Path, text: str, *, timeout: float = 15 +) -> None: + deadline: float = time.monotonic() + timeout + while text not in log.read_text(): + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.02) + + +def location_health_reports(api: SnapshotApi) -> list[dict[str, Any]]: + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/health", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + with urlopen(request, timeout=2) as response: + return json.load(response) + + +def wait_for_warehouse_sync_health( + api: SnapshotApi, + expected: dict[str, Any], + process: subprocess.Popen[bytes], + log: Path, + *, + after: datetime, +) -> list[dict[str, Any]]: + deadline: float = time.monotonic() + 45 + reports: list[dict[str, Any]] = [] + while {report["report"]["health"]["table"] for report in reports} != set(expected["tables"]): + assert process.poll() is None, log.read_text() + by_outcome: dict[tuple[str, str | None, bool], list[dict[str, Any]]] = {} + for report in location_health_reports(api): + health: dict[str, Any] = report["report"]["health"] + key: tuple[str, str | None, bool] = ( + health["kind"], + health.get("failure"), + datetime.fromisoformat(report["received_at"]) > after, + ) + by_outcome.setdefault(key, []).append(report) + reports = list( + filter( + lambda report: report["report"]["health"]["table"] in expected["tables"], + by_outcome.get(("warehouse_sync", expected["failure"], True), []), + ) + ) + assert time.monotonic() < deadline, (reports, log.read_text()) + time.sleep(0.05) + for report in reports: + assert report["report"]["target"] == "development" + health = report["report"]["health"] + assert health["generation"] == 1 + assert health["location"] == {"database": None, "schema": "main"} + assert 0 <= health["records_sent"] <= 100 + assert health["elapsed_millis"] > 0 + return reports + + +def wait_for_location_health( + api: SnapshotApi, + target: str, + health: dict[str, Any], + process: subprocess.Popen[bytes], + log: Path, + *, + minimum_reports: int = 1, +) -> dict[str, Any]: + deadline: float = time.monotonic() + 15 + 30 * (minimum_reports - 1) + key: tuple[str, str] = (target, json.dumps(health, sort_keys=True)) + observations: dict[tuple[str, str], dict[str, dict[str, Any]]] = {} + while len(observations.get(key, ())) < minimum_reports: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + for report in reversed(location_health_reports(api)): + event_id: str = report["report"]["event_id"] + observed_key: tuple[str, str] = ( + report["report"]["target"], + json.dumps(report["report"]["health"], sort_keys=True), + ) + observations.setdefault(observed_key, {})[event_id] = report + time.sleep(0.02) + return list(observations[key].values())[-1] + + +def wait_for_interpreter(process: subprocess.Popen[bytes], pidfile: Path, log: Path) -> int: + deadline: float = time.monotonic() + 15 + while not pidfile.exists(): + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.02) + return int(pidfile.read_text()) + + +def wait_for_location_work( + api: SnapshotApi, request_id: str, process: subprocess.Popen[bytes], log: Path +) -> dict[str, Any]: + """Observe completion through the scoped API rather than private coordination tables.""" + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/work/{request_id}", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + deadline: float = time.monotonic() + 15 + result: dict[str, Any] = {"result_json": None} + while result["result_json"] is None: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + with urlopen(request, timeout=2) as response: + result = json.load(response) + time.sleep(0.02) + return result + + +def api_address(process: subprocess.Popen[bytes], log: Path) -> str: + """Wait for bounded native startup, never publishing an unready address.""" + deadline: float = time.monotonic() + 10 + while "Control API ready: " not in log.read_text(): + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, "Native API did not become ready" + time.sleep(0.02) + return "http://" + log.read_text().partition("Control API ready: ")[2].splitlines()[0] + + +@contextmanager +def snapshot_api( + directory: Path, *, server_arguments: tuple[str, ...] = () +) -> Iterator[SnapshotApi]: + """Only this fixture seeds the control database; the code location uses HTTP exclusively.""" + directory.mkdir() + log: Path = directory / "server.log" + environment: dict[str, str] = os.environ.copy() + environment.pop("SQLBUILD_CONTROL_DATABASE_URL", None) + environment.pop("SQLBUILD_PLATFORM_WEBHOOK_URL", None) + environment.pop("SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL", None) + with log.open("wb") as stderr: + process: subprocess.Popen[bytes] = subprocess.Popen( + [ + str(Path(sys.executable).with_name("sqb")), + "server", + "--port", + "0", + "--no-scheduler", + *server_arguments, + ], + cwd=directory, + env=environment, + stdout=subprocess.DEVNULL, + stderr=stderr, + ) + try: + url: str = api_address(process, log) + location_id: str = str(uuid4()) + token_id: str = str(uuid4()) + bearer: str = f"{token_id}.{secrets.token_hex(32)}" + now: datetime = datetime.now(UTC) + with sqlite3.connect(directory / ".sqlbuild/control.sqlite3") as connection: + _ = connection.execute( + "INSERT INTO sqb_locations (location_id,name,created_by,created_at) " + "VALUES (?, 'orders', 'test-admin', ?)", + (location_id, now.isoformat()), + ) + _ = connection.execute( + "INSERT INTO sqb_access_tokens " + "(token_id,secret_hash,location_id,scope_json,created_at,expires_at) " + "VALUES (?,?,?,?,?,?)", + ( + token_id, + hashlib.sha256(bearer.encode()).hexdigest(), + location_id, + json.dumps({"kind": "location", "location_id": location_id}), + now.isoformat(), + (now + timedelta(hours=1)).isoformat(), + ), + ) + yield SnapshotApi(url=url, location_id=location_id, bearer=bearer) + finally: + process.terminate() + try: + _ = process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + _ = process.wait(timeout=5) + + +def current_snapshot(api: SnapshotApi, target: str) -> dict[str, Any]: + """Observe the public API instead of inspecting stored snapshot implementation details.""" + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/targets/{quote(target, safe='')}/snapshots", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + result: dict[str, Any] = json.load(response) + return result + + +def native_launcher(directory: Path, executable: str) -> AbstractContextManager[WorkerDeployment]: + """Use the installed native launcher without a synthetic child.""" + return nullcontext( + WorkerDeployment( + "local", {"project_dir": str(directory / "project"), "executable": executable} + ) + ) + + +@contextmanager +def uncooperative_launcher(directory: Path, executable: str) -> Iterator[WorkerDeployment]: + """Own a synthetic process that ignores TERM, including cleanup after a failed assertion.""" + del executable + script: Path = directory / "worker" + _ = script.write_text( + f"#!{sys.executable}\nimport os, signal, time\nfrom pathlib import Path\n" + "signal.signal(signal.SIGTERM, signal.SIG_IGN)\n" + "Path('worker.pid').write_text(str(os.getpid()))\n" + "time.sleep(90)\n" + ) + script.chmod(0o700) + try: + yield WorkerDeployment( + "local", {"project_dir": str(directory / "project"), "executable": str(script)} + ) + finally: + with suppress(FileNotFoundError, ProcessLookupError): + os.kill(int((directory / "project/worker.pid").read_text()), signal.SIGKILL) + + +@contextmanager +def scheduled_worker( + directory: Path, + monkeypatch: pytest.MonkeyPatch, + test_case: WorkerExecutionTestCase, + *, + server_arguments: tuple[str, ...] = (), +) -> Iterator[ScheduledWorker]: + """Declare, publish and schedule isolated DuckDB work without provisioning claims.""" + project: Path = directory / "project" + write_files( + project_dir=project, + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n' + '[targets.development]\nschema = "main"\n' + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized table);\n" + test_case.model_sql + "\n" + ), + "orchestration/orders.py": ( + "import os\nfrom sqlbuild.orchestration import job, schedule\n" + "if 'SQLBUILD_RUN_TOKEN' in os.environ:\n" + " assert 'SQLBUILD_PLATFORM_WEBHOOK_URL' not in os.environ\n" + " assert 'SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL' not in os.environ\n" + 'annual_orders = job("annual_orders", select="orders", ' + 'description="Build orders.", ' + 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ), + } + | test_case.files, + ) + executable: str = str(Path(sys.executable).with_name("sqb")) + launchers: dict[str, Callable[[Path, str], AbstractContextManager[WorkerDeployment]]] = { + "native": native_launcher, + "docker": docker_deployment, + "uncooperative": uncooperative_launcher, + } + with ( + snapshot_api(directory / "control", server_arguments=server_arguments) as api, + launchers[test_case.launcher](directory, executable) as deployment, + ): + published: subprocess.CompletedProcess[str] = subprocess.run( + [ + executable, + "location", + "publish", + "--project-dir", + str(project), + "--location-id", + api.location_id, + "--target", + "development", + "--server", + api.url, + "--generation", + "1", + *deployment.publication_arguments, + ], + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert published.returncode == 0, published.stderr + write_files(project_dir=project, files=test_case.files_after_publication) + configuration: Path = directory / "launchers.json" + launcher: dict[str, Any] = { + "location_id": api.location_id, + "target": "development", + "server_url": api.url, + "max_workers": 1, + } | deployment.configuration + _ = configuration.write_text( + json.dumps( + { + deployment.kind: [launcher], + "concurrency_limits": test_case.concurrency_limits, + } + ) + ) + control_store: Path = directory / "control/.sqlbuild/control.sqlite3" + with sqlite3.connect(control_store) as control: + _ = control.executescript(test_case.store_setup) + monkeypatch.setenv("SQLBUILD_SCHEDULER_TEST_STORE", f"sqlite://{control_store}") + monkeypatch.setenv( + "SQLBUILD_PLATFORM_WEBHOOK_URL", + "http://127.0.0.1:1/events?token=synthetic-delivery-credential", + ) + monkeypatch.setenv( + "SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL", + "http://127.0.0.1:1/audit?token=synthetic-audit-credential", + ) + log: Path = directory / "scheduler.log" + with location_process( + [ + executable, + "scheduler", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "--launchers", + str(configuration), + ], + log, + directory / "scheduler.stdout", + ) as process: + yield ScheduledWorker(api, project, control_store, process, log, deployment.docker) + + +def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) -> dict[str, Any]: + """Run/node outcomes, Recall, claims and the physical warehouse agree.""" + api, project, control_store, process, log, _ = worker + run: dict[str, Any] = wait_for_run(api, process, log) + assert run["node_counts_json"] == {test_case.expected_status: test_case.expected_node_count}, ( + log.read_text() + ) + assert run["job_key"] == "annual_orders" + runs_url: str = f"{api.url}/api/locations/{api.location_id}/targets/development/runs" + headers: dict[str, str] = {"Authorization": f"Bearer {api.bearer}"} + with urlopen( + Request(f"{runs_url}/{run['run_id']}/nodes", headers=headers), timeout=5 + ) as response: + nodes: list[dict[str, Any]] = json.load(response) + assert nodes[0]["node_key"] == "model:orders" + assert nodes[0]["started_at"] is not None + assert nodes[0]["ended_at"] is not None + with sqlite3.connect(control_store) as control: + assert control.execute( + "SELECT origin FROM sqb_runs WHERE run_id=?", (run["run_id"],) + ).fetchone() == (test_case.expected_origin,) + assert control.execute( + "SELECT count(*) FROM recall_events " + "WHERE invocation_id=? AND event_type='run_launched'", + (f"scheduler/{run['run_id']}",), + ).fetchone() == (1,) + contexts: list[tuple[str | None]] = control.execute( + "SELECT external_context FROM recall_invocations WHERE command = 'location'" + ).fetchall() + assert contexts, log.read_text() + assert all(context is not None for (context,) in contexts) + expected_run_ids: set[str] = { + run_id + for (run_id,) in control.execute( + "SELECT run_id FROM sqb_runs WHERE kind<>'group' AND ended_at IS NOT NULL" + ).fetchall() + } + assert { + json.loads(cast(str, context))["run_id"] for (context,) in contexts + } == expected_run_ids + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + with duckdb.connect(str(project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + return run + + +def running_statement(worker: ScheduledWorker, *, job_key: str | None = None) -> dict[str, Any]: + """The worker has started the real long query, not merely entered its Python interpreter.""" + active: dict[str, Any] = wait_for_run( + worker.api, + worker.process, + worker.log, + pending=frozenset({"queued", "launched"}), + job_key=job_key, + ) + assert active["status"] == "running", worker.log.read_text() + deadline: float = time.monotonic() + 15 + with sqlite3.connect(worker.control_store) as control: + while not control.execute( + "SELECT 1 FROM recall_artifacts WHERE CAST(content AS TEXT) LIKE ?", + ("%range(10000000000)%",), + ).fetchone(): + assert worker.process.poll() is None, worker.log.read_text() + assert time.monotonic() < deadline, worker.log.read_text() + time.sleep(0.02) + return active + + +def cancel_running_statement( + worker: ScheduledWorker, *, job_key: str | None = None, run_id: str | None = None +) -> None: + """Wait for the real long-running query before exercising the administrative cancel command.""" + active: dict[str, Any] = running_statement(worker, job_key=job_key) + cancelled: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "cancel-run", + run_id or active["run_id"], + "--event-id", + str(uuid4()), + "--reason", + "Cancel the active test statement.", + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert cancelled.returncode == 0, cancelled.stderr + + +def wait_for_run( + api: SnapshotApi, + process: subprocess.Popen[bytes], + log: Path, + *, + pending: frozenset[str] = frozenset({"queued", "launched", "running", "unresponsive"}), + job_key: str | None = None, + run_id: str | None = None, + target: str = "development", +) -> dict[str, Any]: + """A terminal API projection must follow the scheduler's actual isolated warehouse execution.""" + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/targets/{quote(target, safe='')}/runs", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + deadline: float = time.monotonic() + 30 + current: dict[str, Any] = {"status": "queued"} + while current["status"] in pending: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + with urlopen(request, timeout=2) as response: + runs: list[dict[str, Any]] = json.load(response) + by_job: dict[str | None, dict[str, Any]] = {run["job_key"]: run for run in runs} + by_job[None] = next(iter(runs), current) + by_id: dict[str | None, dict[str, Any]] = {run["run_id"]: run for run in runs} + by_id[None] = by_job.get(job_key, current) + current = by_id.get(run_id, current) + time.sleep(0.02) + return current + + +def claimed_worker(directory: Path, api: SnapshotApi, publication: dict[str, Any]) -> ClaimedWorker: + """Provision only the launch boundary; verification and lifecycle reporting use real HTTP.""" + run_id: str = str(uuid4()) + claim_id: str = str(uuid4()) + token_id: str = str(uuid4()) + bearer: str = f"{token_id}.{secrets.token_hex(32)}" + now: datetime = datetime.now(UTC) + selection: list[str] = ["model:orders"] + request: dict[str, Any] = { + "run_id": run_id, + "location_id": api.location_id, + "target": publication["target"], + "snapshot_id": publication["snapshot_id"], + "image_id": None, + "kind": "job", + "origin": "manual", + "job_key": None, + "parent_run_id": None, + "root_run_id": run_id, + "trigger_id": None, + "tick_id": None, + "rerun_of_run_id": None, + "requested_by": "test-admin", + "priority": 0, + "selection": selection, + "write_set": selection, + "settings": {"full_refresh": True}, + "concurrency_groups": [], + } + with sqlite3.connect(directory / ".sqlbuild/control.sqlite3") as connection: + _ = connection.execute( + "INSERT INTO sqb_runs (run_id,location_id,target,snapshot_id,kind,origin,root_run_id," + "requested_by,selection_json,request_json,write_set_digest,status,queued_at," + "node_counts_json,last_event_storage_id) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + ( + run_id, + api.location_id, + publication["target"], + publication["snapshot_id"], + "job", + "manual", + run_id, + "test-admin", + json.dumps(selection), + json.dumps(request), + hashlib.sha256(json.dumps(selection).encode()).hexdigest(), + "queued", + now.isoformat(), + json.dumps({"pending": 1}), + 0, + ), + ) + _ = connection.execute( + "INSERT INTO sqb_run_nodes (run_id,node_key,status,writes) " + "VALUES (?,'model:orders','pending',TRUE)", + (run_id,), + ) + _ = connection.execute( + "INSERT INTO sqb_write_claims (target,node_key,run_id,claimed_at) VALUES (?,?,?,?)", + (publication["target"], "model:orders", run_id, now.isoformat()), + ) + _ = connection.execute( + "UPDATE sqb_runs SET worker_instance=?, claimed_at=?, last_heartbeat_at=? " + "WHERE run_id=?", + (claim_id, now.isoformat(), now.isoformat(), run_id), + ) + _ = connection.execute( + "INSERT INTO sqb_access_tokens " + "(token_id,secret_hash,run_id,claim_id,scope_json,created_at,expires_at) " + "VALUES (?,?,?,?,?,?,?)", + ( + token_id, + hashlib.sha256(bearer.encode()).hexdigest(), + run_id, + claim_id, + json.dumps({"kind": "run", "run_id": run_id, "claim_id": claim_id}), + now.isoformat(), + (now + timedelta(hours=1)).isoformat(), + ), + ) + return ClaimedWorker(run_id=run_id, claim_id=claim_id, bearer=bearer) + + +def wait_for_sensor_phase( + store: Path, phase: str, *, empty_runs: bool = False, timeout: float = 45 +) -> dict[str, Any]: + """Wait for real scheduler facts, optionally requiring a completed skip.""" + deadline: float = time.monotonic() + timeout + row: tuple[str] | None = None + while row is None: + assert time.monotonic() < deadline, f"Sensor did not record {phase}" + with sqlite3.connect(store) as control: + row = control.execute( + "SELECT fact_json FROM sqb_sensor_evaluation_events WHERE phase=? " + "AND (? = FALSE OR fact_json->'outcome'->'run_ids'='[]') " + "ORDER BY occurred_at DESC, evaluation_id DESC LIMIT 1", + (phase, empty_runs), + ).fetchone() + time.sleep(0.05) + return json.loads(row[0]) + + +def sensor_calls(path: Path, fields: dict[str, Any]) -> list[dict[str, Any]]: + """Select recorded callback invocations without inferring execution from declarations.""" + records: list[dict[str, Any]] = [json.loads(line) for line in path.read_text().splitlines()] + return list(filter(lambda record: fields.items() <= record.items(), records)) + + +def append_retried_fingerprints( + *, + adapter: Any, + connection: Any, + database_path: Path, + fingerprint: Fingerprint, + read_columns: Callable[..., Any] | None = None, +) -> None: + """Retry the same observation without substituting its identity or live catalog.""" + del database_path + for _attempt in range(2): + _ = write_fingerprint( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + fingerprint=fingerprint, + ensure_table=False, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + read_columns=read_columns or adapter.get_columns, + render_add_columns=adapter.render_add_columns, + render_create_table_sql=adapter.render_create_fingerprint_table_sql, + ) + + +def append_contended_fingerprints( + *, adapter: Any, connection: Any, database_path: Path, fingerprint: Fingerprint +) -> None: + """Release a real SQLite write lock only after the first ALTER has failed.""" + with closing(sqlite3.connect(database_path)) as blocker: + _ = adapter.execute(connection=connection, sql="PRAGMA busy_timeout=1") + catalog_actions: Iterator[Callable[[], Any]] = iter( + (lambda: blocker.execute("BEGIN IMMEDIATE"), blocker.commit) + ) + + def read_columns(**scope: Any) -> Any: + columns: Any = adapter.get_columns(**scope) + next(catalog_actions, lambda: None)() + return columns + + try: + append_retried_fingerprints( + adapter=adapter, + connection=connection, + database_path=database_path, + fingerprint=fingerprint, + read_columns=read_columns, + ) + finally: + blocker.rollback() + + +def run_statistics_retention(directory: Path, arguments: tuple[str, ...]) -> str: + """Run one installed server until its first retention pass, returning its stderr.""" + return run_server_until( + directory, + ("--no-scheduler", *arguments), + lambda _: "sentinel_expired" not in statistics_relations(directory), + ) + + +def run_server_until( + directory: Path, + arguments: tuple[str, ...], + finished: Callable[[str], bool], + environment_overrides: dict[str, str] | None = None, +) -> str: + """Run one installed server until its stderr or store satisfies `finished`, returning stderr.""" + log: Path = directory / "server.log" + environment: dict[str, str] = os.environ.copy() + environment.pop("SQLBUILD_CONTROL_DATABASE_URL", None) + environment.update(environment_overrides or {}) + with log.open("wb") as stderr: + process: subprocess.Popen[bytes] = subprocess.Popen( + [str(Path(sys.executable).with_name("sqb")), "server", "--port", "0", *arguments], + cwd=directory, + env=environment, + stdout=subprocess.DEVNULL, + stderr=stderr, + ) + try: + _ = api_address(process, log) + deadline: float = time.monotonic() + 10 + while not finished(log.read_text()): + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, "server did not reach the expected state" + time.sleep(0.05) + finally: + process.terminate() + _ = process.wait(timeout=10) + return log.read_text() + + +def audit_actions(directory: Path) -> tuple[str, ...]: + """Read the synthetic retention fixture's audit actions directly, in storage order.""" + with sqlite3.connect(directory / ".sqlbuild/control.sqlite3") as connection: + rows: list[tuple[str]] = connection.execute( + "SELECT action FROM sqb_security_audit WHERE action LIKE 'orders.%' ORDER BY storage_id" + ).fetchall() + return tuple(action for (action,) in rows) + + +def statistics_relations(directory: Path) -> tuple[str, ...]: + """Read the retained sample relations directly, oldest first.""" + with sqlite3.connect(directory / ".sqlbuild/control.sqlite3") as connection: + rows: list[tuple[str]] = connection.execute( + "SELECT relation_name FROM sqb_warehouse_statistics ORDER BY observed_at" + ).fetchall() + return tuple(name for (name,) in rows) + + +def scheduler_crash_history(directory: Path, fixture: dict[str, Any]) -> dict[str, int]: + """SIGKILL real schedulers during catch-up; return breaches and the kill count.""" + project: Path = directory / "project" + start: datetime = datetime.now(UTC).replace(second=0, microsecond=0) - timedelta( + minutes=fixture["backlog_minutes"] + ) + declarations: str = "".join( + f'{name} = job("{name}", select="orders", description="Build {name}.", ' + f'schedule=schedule(cron="* * * * *", timezone="UTC"))\n' + for name in fixture["jobs"] + ) + write_files( + project_dir=project, + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n' + '[targets.development]\nschema = "main"\n' + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized table);\nSELECT 1 AS order_id\n" + ), + "orchestration/orders.py": "from sqlbuild.orchestration import job, schedule\n" + + declarations, + }, + ) + executable: str = str(Path(sys.executable).with_name("sqb")) + control_store: Path = directory / "control/.sqlbuild/control.sqlite3" + environment: dict[str, str] = os.environ | { + "SQLBUILD_SCHEDULER_TEST_STORE": f"sqlite://{control_store}" + } + for name in ("SQLBUILD_PLATFORM_WEBHOOK_URL", "SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL"): + _ = environment.pop(name, None) + with snapshot_api(directory / "control") as api: + published: subprocess.CompletedProcess[str] = subprocess.run( + [ + executable, + "location", + "publish", + "--project-dir", + str(project), + "--location-id", + api.location_id, + "--target", + "development", + "--server", + api.url, + "--generation", + "1", + ], + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert published.returncode == 0, published.stderr + # The triggers were published hours ago, so a backlog of minutes is due when schedulers start. + with closing(sqlite3.connect(control_store)) as control, control: + _ = control.execute( + "UPDATE sqb_trigger_state SET started_at=?, next_due_at=?, last_tick_for=NULL", + (start.strftime("%Y-%m-%dT%H:%M:%S.%fZ"),) * 2, + ) + choices: random.Random = random.Random(fixture["seed"]) + command: list[str] = [ + executable, + "scheduler", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + ] + kills: int = 0 + for round_index in range(fixture["rounds"]): + processes: list[subprocess.Popen[bytes]] = [ + subprocess.Popen( + command, + env=environment, + stdout=subprocess.DEVNULL, + stderr=(directory / f"scheduler-{round_index}-{index}.log").open("wb"), + ) + for index in range(2) + ] + time.sleep(choices.uniform(0.05, fixture["max_kill_delay_ms"] / 1000)) + for process in processes: + process.kill() + kills += int(process.wait(timeout=10) == -signal.SIGKILL) + # A killed holder's lease would lapse after its TTL; lapse it now instead of waiting. + with closing(sqlite3.connect(control_store, timeout=30)) as control, control: + _ = control.execute( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'" + ) + settled: datetime = datetime.now(UTC) + with closing(sqlite3.connect(control_store, timeout=30)) as control: + (interrupted,) = control.execute("SELECT count(*) FROM sqb_ticks").fetchone() + with location_process( + command, directory / "survivor.log", directory / "survivor.stdout", env=environment + ): + deadline: float = time.monotonic() + 60 + pending: int = 1 + while pending and time.monotonic() < deadline: + time.sleep(0.5) + with closing(sqlite3.connect(control_store, timeout=30)) as control: + (pending,) = control.execute( + "SELECT count(*) FROM sqb_trigger_state " + "WHERE last_tick_for IS NULL OR last_tick_for Iterator[tuple[str, Path, dict[str, str]]]: + """Generate disposable keys; forward HTTP unchanged except hop-by-hop transport headers.""" + certificate: Path = root / "localhost.pem" + key: Path = root / "localhost-key.pem" + subprocess.run( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "1", + "-subj", + "/CN=localhost", + "-addext", + "subjectAltName=DNS:localhost", + "-addext", + "basicConstraints=critical,CA:FALSE", + "-keyout", + str(key), + "-out", + str(certificate), + ], + check=True, + capture_output=True, + timeout=15, + ) + state: dict[str, str] = {} + identity_provider: Callable[[BaseHTTPRequestHandler, bytes], None] = provider(key) + + class Forward(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self.forward() + + def do_POST(self) -> None: + self.forward() + + def log_message(self, format: str, *args: Any) -> None: + """Do not log handoff or authentication request paths.""" + + def forward(self) -> None: + body: bytes = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + if self.path.startswith("/issuer/"): + identity_provider(self, body) + return + upstream: http.client.HTTPConnection = http.client.HTTPConnection( + state["backend"], timeout=10 + ) + try: + headers: dict[str, str] = dict(self.headers) + if "proxy-subject" in state: + headers = { + name: value + for name, value in headers.items() + if name.lower() != "x-sqb-user" + } + headers["X-SQB-User"] = state["proxy-subject"] + upstream.request(self.command, self.path, body, headers) + response: http.client.HTTPResponse = upstream.getresponse() + content: bytes = response.read() + self.send_response(response.status) + for name, value in response.getheaders(): + if name.lower() not in {"connection", "transfer-encoding", "content-length"}: + self.send_header(name, value) + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) + finally: + upstream.close() + + with ThreadingHTTPServer(("127.0.0.1", 0), Forward) as server: + context: ssl.SSLContext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(certificate, key) + server.socket = context.wrap_socket(server.socket, server_side=True) + thread: threading.Thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"https://localhost:{server.server_port}", certificate, state + finally: + server.shutdown() + thread.join(timeout=10) + + +def browser(origin: str, certificate: Path) -> tuple[Any, CookieJar]: + """Use a real TLS verifier and secure cookie jar, not an insecure browser bypass.""" + cookies: CookieJar = CookieJar() + return build_opener( + HTTPSHandler(context=ssl.create_default_context(cafile=str(certificate))), + HTTPCookieProcessor(cookies), + ), cookies + + +def request( + client: Any, endpoint: str, body: dict[str, Any] | None = None, *, csrf: str = "" +) -> tuple[int, bytes]: + """Return denials without printing authentication response payloads.""" + origin: str = endpoint.split("/", 3)[0] + "//" + endpoint.split("/", 3)[2] + call: Request = Request( + endpoint, + data=None if body is None else json.dumps(body).encode(), + headers={"Origin": origin, "Content-Type": "application/json", "x-sqb-csrf": csrf}, + ) + try: + response: Any = client.open(call, timeout=10) + except HTTPError as error: + response = error + with response: + status: int | None = response.status + assert status is not None + return status, response.read() + + +def wait_for_handoff(process: subprocess.Popen[bytes], log: Path) -> str: + """Observe the CLI's public URL while keeping all credential responses out of logs.""" + deadline: float = time.monotonic() + 20 + while "\nVerification code: " not in log.read_text(): + assert process.poll() is None, "CLI exited before offering browser confirmation" + assert time.monotonic() < deadline, "CLI did not offer browser confirmation" + time.sleep(0.02) + return log.read_text().split("Open ", 1)[1].splitlines()[0] + + +@contextmanager +def pending_login( + command: list[str], environment: dict[str, str], project: Path, log: Path +) -> Iterator[tuple[subprocess.Popen[bytes], str]]: + """Reap a real login process even if browser approval or an assertion fails.""" + with log.open("wb") as stderr: + process: subprocess.Popen[bytes] = subprocess.Popen( + command, + cwd=project, + env=environment, + stdout=subprocess.PIPE, + stderr=stderr, + ) + try: + yield process, wait_for_handoff(process, log) + finally: + process.kill() + process.wait(timeout=10) + + +def authentication_arguments( + mode: str, origin: str, certificate: Path, principal: str +) -> tuple[str, ...]: + """Use only the product's existing explicit authentication configuration.""" + if mode == "password": + return "--password-origin", origin + configuration: dict[str, Any] = {"origin": origin, "identities": {"orders-person": principal}} + if mode == "oidc": + configuration.update( + issuer=origin + "/issuer", + client_id="orders-control", + certificate_authority=str(certificate), + ) + flag = "--oidc-config" + else: + assert mode == "proxy" + configuration["trusted_peers"] = ["127.0.0.1"] + flag = "--trusted-proxy-config" + path: Path = certificate.parent / "authentication.json" + path.write_text(json.dumps(configuration)) + return flag, str(path) + + +def sign_in(client: Any, origin: str, mode: str, username: str, password: str) -> str: + """Password and PKCE provider sign-in use real routes; the proxy authenticates each request.""" + if mode == "proxy": + return "" + if mode == "password": + status, payload = request( + client, + origin + "/api/auth/password/login", + {"username": username, "password": password}, + ) + else: + status, payload = request(client, origin + "/api/auth/oidc/login") + assert status == 200, f"Installed browser authentication failed with HTTP {status}" + return json.loads(payload)["csrf_token"] + + +def assert_private_storage( + credential: Path, command: list[str], environment: dict[str, str], project: Path +) -> None: + """Real CLI reload must refuse an unsafe file mode and a symlinked credential.""" + original_mode: int = stat.S_IMODE(credential.stat().st_mode) + credential.chmod(0o644) + unsafe: subprocess.CompletedProcess[str] = subprocess.run( + command, + cwd=project, + env=environment, + text=True, + capture_output=True, + timeout=20, + check=False, + ) + assert unsafe.returncode != 0 and not unsafe.stdout + credential.chmod(original_mode) + backup: Path = credential.with_suffix(".backup") + credential.rename(backup) + credential.symlink_to(backup) + symlink: subprocess.CompletedProcess[str] = subprocess.run( + command, + cwd=project, + env=environment, + text=True, + capture_output=True, + timeout=20, + check=False, + ) + assert symlink.returncode != 0 and not symlink.stdout + credential.unlink() + backup.rename(credential) + + +def encoded(value: bytes) -> str: + """JOSE uses unpadded URL-safe base64.""" + return base64.urlsafe_b64encode(value).rstrip(b"=").decode() + + +def provider(key: Path) -> Callable[[BaseHTTPRequestHandler, bytes], None]: + """Consume codes once and validate real PKCE before signing the requested nonce.""" + modulus: str = ( + subprocess.run( + ["openssl", "rsa", "-in", str(key), "-modulus", "-noout"], + check=True, + capture_output=True, + text=True, + timeout=10, + ) + .stdout.strip() + .split("=", 1)[1] + ) + jwk: dict[str, str] = { + "kty": "RSA", + "use": "sig", + "alg": "RS256", + "kid": "orders", + "n": encoded(bytes.fromhex(modulus)), + "e": "AQAB", + } + codes: dict[str, dict[str, list[str]]] = {} + + def handle(handler: BaseHTTPRequestHandler, body: bytes) -> None: + assert isinstance(handler.server, ThreadingHTTPServer) + origin: str = f"https://localhost:{handler.server.server_port}" + issuer: str = origin + "/issuer" + path: SplitResult = urlsplit(handler.path) + payload: dict[str, object] + if path.path.endswith("/.well-known/openid-configuration"): + payload = { + "issuer": issuer, + "authorization_endpoint": issuer + "/authorize", + "token_endpoint": issuer + "/token", + "jwks_uri": issuer + "/jwks", + "response_types_supported": ["code"], + "subject_types_supported": ["public"], + "id_token_signing_alg_values_supported": ["RS256"], + "code_challenge_methods_supported": ["S256"], + } + elif path.path.endswith("/jwks"): + payload = {"keys": [jwk]} + elif path.path.endswith("/authorize"): + parameters: dict[str, list[str]] = parse_qs(path.query) + assert parameters["client_id"] == ["orders-control"] + assert parameters["code_challenge_method"] == ["S256"] + assert parameters["redirect_uri"] == [origin + "/api/auth/oidc/callback"] + code: str = str(uuid.uuid4()) + codes[code] = parameters + handler.send_response(303) + handler.send_header( + "Location", + parameters["redirect_uri"][0] + + "?" + + urlencode({"code": code, "state": parameters["state"][0]}), + ) + handler.send_header("Content-Length", "0") + handler.end_headers() + return + elif path.path.endswith("/token"): + form: dict[str, list[str]] = parse_qs(body.decode()) + parameters = codes.pop(form["code"][0]) + assert form["grant_type"] == ["authorization_code"] + assert form["redirect_uri"] == parameters["redirect_uri"] + assert ( + encoded(hashlib.sha256(form["code_verifier"][0].encode()).digest()) + == parameters["code_challenge"][0] + ) + header: str = encoded(json.dumps({"alg": "RS256", "kid": "orders"}).encode()) + claims: str = encoded( + json.dumps( + { + "iss": issuer, + "sub": "orders-person", + "aud": "orders-control", + "iat": int(time.time()), + "exp": int(time.time()) + 300, + "nonce": parameters["nonce"][0], + } + ).encode() + ) + signing_input: bytes = f"{header}.{claims}".encode() + signature: bytes = subprocess.run( + ["openssl", "dgst", "-sha256", "-sign", str(key)], + input=signing_input, + check=True, + capture_output=True, + timeout=10, + ).stdout + payload = { + "access_token": "synthetic-orders-access", + "token_type": "Bearer", + "expires_in": 300, + "id_token": f"{header}.{claims}.{encoded(signature)}", + } + else: + raise AssertionError("unexpected identity-provider endpoint") + content: bytes = json.dumps(payload).encode() + handler.send_response(200) + handler.send_header("Content-Type", "application/json") + handler.send_header("Content-Length", str(len(content))) + handler.end_headers() + handler.wfile.write(content) + + return handle diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py new file mode 100644 index 0000000000..f6b5ebe58d --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py @@ -0,0 +1,302 @@ +"""Exercise owner provisioning and browser transport without bypassing role authorization.""" + +import json +import os +import subprocess +import sys +import time +from http.cookies import SimpleCookie +from pathlib import Path +from subprocess import Popen +from typing import Any +from urllib.error import HTTPError +from urllib.parse import urlencode +from urllib.request import Request, urlopen +from uuid import uuid4 + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + SnapshotApi, +) + + +def provision_person(control: Path, command: str, name: str) -> dict[str, Any]: + environment: dict[str, str] = os.environ.copy() + environment.pop("SQLBUILD_CONTROL_DATABASE_URL", None) + executable: str = str(Path(sys.executable).with_name("sqb")) + created: subprocess.CompletedProcess[str] = subprocess.run( + [executable, "admin", command, name], + cwd=control, + env=environment, + capture_output=True, + text=True, + check=True, + timeout=15, + ) + person: dict[str, Any] = json.loads(created.stdout) + _ = subprocess.run( + [executable, "admin", "set-password", person["principal_id"]], + input="orders-person-password-2026\n", + cwd=control, + env=environment, + capture_output=True, + text=True, + check=True, + timeout=15, + ) + return person + + +def person_headers(session: tuple[str, str]) -> dict[str, str]: + return { + "Host": "control.example.com", + "Origin": "https://control.example.com", + "Content-Type": "application/json", + "Cookie": session[0], + "x-sqb-csrf": session[1], + } + + +def person_read(api: SnapshotApi, path: str, *, session: tuple[str, str]) -> Any: + with urlopen(Request(api.url + path, headers=person_headers(session)), timeout=5) as response: + return json.load(response) + + +def verify_compute_transport(api: SnapshotApi, session: tuple[str, str], run_id: str) -> None: + """Real retained SQL/output are readable by the location, never by a browser cookie alone.""" + metadata: list[dict[str, Any]] = person_read( + api, f"/api/control/locations/{api.location_id}/targets/development/runs", session=session + ) + assert metadata[0]["run_id"] == run_id + assert not ({"sql", "compiled_sql", "output", "rows", "evidence"} & metadata[0].keys()) + path: str = f"/api/locations/{api.location_id}/targets/development/runs/{run_id}" + headers: dict[str, str] = {"Authorization": "Bearer " + api.bearer} + with urlopen(Request(api.url + path + "/invocations", headers=headers), timeout=5) as response: + invocations: list[dict[str, Any]] = json.load(response) + assert any(row["invocation_id"].startswith("worker/") for row in invocations) + for view in ["artifacts", "output_tail"]: + evidence: list[dict[str, Any]] = [] + for invocation in invocations: + endpoint: str = ( + path + + "/recall/" + + view + + "?" + + urlencode({"invocation": invocation["invocation_id"]}) + ) + with urlopen(Request(api.url + endpoint, headers=headers), timeout=5) as response: + evidence.extend(json.load(response)) + with pytest.raises(HTTPError) as rejected: + person_read(api, endpoint, session=session) + assert rejected.value.code == 401 + assert evidence, view + assert any( + row["invocation_id"].startswith("worker/") and row["text"] for row in evidence + ), view + + +def wait_for_person_catalog( + api: SnapshotApi, + session: tuple[str, str], + process: Popen[bytes], + log: Path, + *, + expected_definition_hash: str, +) -> None: + """Wait for an actual worker-built relation through the scoped browser API, not store SQL.""" + path: str = f"/api/control/locations/{api.location_id}/targets/development/catalog?schema=main" + deadline: float = time.monotonic() + 45 + page: dict[str, Any] = person_read(api, path, session=session) + records: dict[str, Any] = {row["record"]["name"]: row for row in page["records"]} + while "orders" not in records: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, (page, log.read_text()) + time.sleep(0.05) + page = person_read(api, path, session=session) + records = {row["record"]["name"]: row for row in page["records"]} + assert records["orders"]["event_id"] and records["orders"]["observed_at"] + assert records["orders"]["record"] == { + "state_database": None, + "state_schema": "main", + "name": "orders", + "relation_type": "BASE TABLE", + "columns": [{"name": "amount", "data_type": "INTEGER"}], + } + assert page["next_after"] is None + assert person_read(api, path + "&database=unlisted", session=session) == { + "records": [], + "next_after": None, + } + fingerprint_path: str = ( + f"/api/control/locations/{api.location_id}/targets/development/fingerprint" + "?node_type=model&node_name=orders&schema=main&name=orders" + ) + evidence: dict[str, Any] | None = person_read(api, fingerprint_path, session=session) + while evidence is None or evidence["definition_hash"] != expected_definition_hash: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, (evidence, log.read_text()) + time.sleep(0.05) + evidence = person_read(api, fingerprint_path, session=session) + assert set(evidence) == { + "event_id", + "observed_at", + "source_event_id", + "run_id", + "definition_hash", + "version_hash", + "schema_fingerprint", + } + assert all(evidence.values()), evidence + assert person_read(api, fingerprint_path + "&database=unlisted", session=session) is None + status: dict[str, Any] = person_read( + api, + f"/api/control/locations/{api.location_id}/targets/development/node-status?node_key=model:orders", + session=session, + ) + assert status["physical"]["status"] == "observed" + assert status["physical"]["observation"] == records["orders"] + assert status["definition"]["status"] == "matched", status + assert status["build"] == evidence + assert status["source_data"] == { + "status": "unknown", + "reason": "consumed-input provenance is not recorded", + } + statistics_path: str = ( + f"/api/control/locations/{api.location_id}/targets/development/statistics" + "?schema=main&name=orders" + ) + statistics: dict[str, Any] = person_read(api, statistics_path, session=session) + while not statistics["records"]: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, (statistics, log.read_text()) + time.sleep(0.05) + statistics = person_read(api, statistics_path, session=session) + sample: dict[str, Any] = statistics["records"][0] + assert sample["event_id"] and sample["record"]["observed_at"] + assert sample["record"] | {"observed_at": None} == { + "state_database": None, + "state_schema": "main", + "name": "orders", + "observed_at": None, + "row_count": 1, + "detail": None, + "byte_count": None, + "byte_count_basis": None, + } + assert statistics["next_after"] is None + assert person_read(api, statistics_path + "&database=unlisted", session=session) == { + "records": [], + "next_after": None, + } + + +def verify_security_audit( + api: SnapshotApi, administrator: tuple[str, str], ordinary: tuple[str, str] +) -> None: + audit: list[dict[str, Any]] = person_read( + api, "/api/control/audit?limit=3", session=administrator + ) + assert len(audit) == 3 + cursor: int = audit[-1]["storage_id"] + following: list[dict[str, Any]] = person_read( + api, f"/api/control/audit?after={cursor}&limit=3", session=administrator + ) + assert len(following) == 3 + assert all(event["storage_id"] > cursor for event in following) + assert len({event["event_id"] for event in audit + following}) == 6 + assert all(event["actor"] and event["granting_role"] for event in audit + following) + with pytest.raises(HTTPError) as audit_rejected: + person_read(api, "/api/control/audit", session=ordinary) + assert audit_rejected.value.code == 403 + + +def verify_sql_inspection( + api: SnapshotApi, + sessions: tuple[tuple[str, str], tuple[str, str]], + assignment: dict[str, Any], + model: Path, + snapshot_id: str, +) -> None: + """Inspect actual published SQL under a live scoped role, then revoke that role.""" + administrator, session = sessions + path: str = ( + f"/api/control/locations/{api.location_id}/targets/development/nodes/model:orders/sql" + ) + assignment = assignment | {"role": "developer", "revoke": False} + assert person_request(api, "/api/admin/roles", body=assignment, session=administrator) == { + "applied": True + } + sql: dict[str, Any] = person_read(api, path, session=session) + assert set(sql) == {"snapshot_id", "node_key", "raw_sql", "compiled_sql"} + assert sql["snapshot_id"] == snapshot_id + assert sql["node_key"] == "model:orders" + assert sql["raw_sql"] == model.read_text() + with duckdb.connect() as inspection: + assert inspection.sql(sql["compiled_sql"]).fetchall() == [(41,)] + for wrong_scope in [ + path.replace("development", "preview"), + path.replace(api.location_id, str(uuid4())), + ]: + with pytest.raises(HTTPError) as denied: + person_read(api, wrong_scope, session=session) + assert denied.value.code == 403 + assert person_request( + api, "/api/admin/roles", body=assignment | {"revoke": True}, session=administrator + ) == {"applied": True} + with pytest.raises(HTTPError) as revoked: + person_read(api, path, session=session) + assert revoked.value.code == 403 + + +def person_request( + api: SnapshotApi, path: str, *, body: Any, session: tuple[str, str] = ("", "") +) -> Any: + request: Request = Request( + api.url + path, + data=json.dumps(body).encode(), + headers=person_headers(session), + ) + with urlopen(request, timeout=5) as response: + return json.load(response) + + +def person_login(api: SnapshotApi, name: str) -> tuple[str, str]: + request: Request = Request( + api.url + "/api/auth/password/login", + data=json.dumps({"username": name, "password": "orders-person-password-2026"}).encode(), + headers={ + "Host": "control.example.com", + "Origin": "https://control.example.com", + "Content-Type": "application/json", + }, + ) + with urlopen(request, timeout=5) as response: + cookies: SimpleCookie[str] = SimpleCookie("\n".join(response.headers.get_all("Set-Cookie"))) + cookie: str = "__Host-sqb-session=" + cookies["__Host-sqb-session"].value + return cookie, json.load(response)["csrf_token"] + + +def verify_connection_test_transport( + api: SnapshotApi, path: str, body: dict[str, Any], administrator: tuple[str, str] +) -> None: + """Machine credentials, CSRF-less writes and caller-supplied work never cross the person API.""" + for credentials in [("", ""), (administrator[0], "wrong-csrf")]: + with pytest.raises(HTTPError) as denied: + person_request(api, path, body=body, session=credentials) + assert denied.value.code == 401 + with pytest.raises(HTTPError) as denied: + urlopen( + Request( + api.url + path, + data=json.dumps(body).encode(), + headers=person_headers(("", "")) | {"Authorization": "Bearer " + api.bearer}, + ), + timeout=5, + ) + assert denied.value.code == 401 + for extra in [{"actor": "forged"}, {"work": {"kind": "evaluate_sensor"}}]: + with pytest.raises(HTTPError) as denied: + person_request(api, path, body=body | extra, session=administrator) + assert denied.value.code == 422 diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py new file mode 100644 index 0000000000..9d54256fc6 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py @@ -0,0 +1,164 @@ +"""Explicit backfills and their reruns execute the same pinned window through real workers.""" + +import json +import sqlite3 +import subprocess +import sys +from dataclasses import replace +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + BackfillExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + scheduled_worker, + wait_for_run, +) + + +@pytest.mark.parametrize( + "test_case", + [ + BackfillExecutionTestCase( + description="local integer window overrides authored bounds and survives group rerun", + model_sql="SELECT 1 AS amount", + warehouse_query="SELECT amount FROM orders ORDER BY amount", + expected_status="success", + expected_rows=[(23,), (37,)], + files={ + "models/orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/backfill_integer.sql" + ).read_text() + }, + window_arguments=("--start-cursor-int", "3", "--end-cursor-int", "6"), + expected_window={"kind": "integer", "start": 3, "end": 6}, + ), + BackfillExecutionTestCase( + description="Docker timestamp window normalizes offsets and survives group rerun", + model_sql="SELECT 1 AS amount", + warehouse_query="SELECT amount FROM orders ORDER BY amount", + expected_status="success", + expected_rows=[(23,), (37,)], + launcher="docker", + files={ + "models/orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/backfill_timestamp.sql" + ).read_text() + }, + window_arguments=( + "--start-cursor-ts", + "2026-07-03T05:30:00+05:30", + "--end-cursor-ts", + "2026-07-04T05:30:00+05:30", + ), + expected_window={ + "kind": "timestamp", + "start": "2026-07-03T00:00:00Z", + "end": "2026-07-04T00:00:00Z", + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_explicit_backfill_bounds_when_running_and_rerunning_a_group_then_only_its_frozen_window_is_written( + test_case: BackfillExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + test_case = replace( + test_case, + files=test_case.files + | { + "sources/raw_orders.yml": "sources:\n - name: raw_orders\n description: Synthetic order windows.\n schema: main\n table: raw_orders\n", + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, job_group\n" + "orders = job('orders', description='Windowed orders.', select='orders', " + "settings={'start_cursor': '0', 'end_cursor': '1'})\n" + "annual_orders = job_group('annual_orders', description='Windowed group.', jobs=[orders])\n" + ), + }, + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + with duckdb.connect(str(worker.project / "warehouse.duckdb")) as warehouse: + warehouse.execute( + "CREATE TABLE raw_orders AS SELECT * FROM (VALUES (11, 2, TIMESTAMP '2026-07-02'), (23, 3, TIMESTAMP '2026-07-03'), (37, 6, TIMESTAMP '2026-07-04'), (41, 7, TIMESTAMP '2026-07-05')) t(amount, batch_id, ordered_at)" + ) + executable: str = str(Path(sys.executable).with_name("sqb")) + run_id: str = str(uuid4()) + arguments: list[str] = [ + executable, + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "request-run", + worker.api.location_id, + "annual_orders", + "--target", + "development", + "--run-id", + run_id, + "--reason", + "Backfill an explicit order window", + *test_case.window_arguments, + ] + rejected: subprocess.CompletedProcess[str] = subprocess.run( + arguments[:-2], capture_output=True, text=True, timeout=10, check=False + ) + assert rejected.returncode != 0 + assert "both start and end of exactly one cursor type" in rejected.stderr + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_runs").fetchone() == (0,) + for expected in [True, False]: + requested: subprocess.CompletedProcess[str] = subprocess.run( + arguments, capture_output=True, text=True, timeout=10, check=False + ) + assert requested.returncode == 0, requested.stderr + assert json.loads(requested.stdout) == {"run_id": run_id, "applied": expected} + completed: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=run_id + ) + assert completed["status"] == test_case.expected_status, worker.log.read_text() + with duckdb.connect(str(worker.project / "warehouse.duckdb")) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + warehouse.execute("UPDATE raw_orders SET amount=amount+100") + retry_id: str = str(uuid4()) + retried: subprocess.CompletedProcess[str] = subprocess.run( + [ + executable, + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "rerun-run", + run_id, + "--run-id", + retry_id, + "--reason", + "Repeat the same explicit order window", + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert retried.returncode == 0, retried.stderr + retry: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=retry_id + ) + assert retry["status"] == test_case.expected_status, worker.log.read_text() + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == [(123,), (137,)] + with sqlite3.connect(worker.control_store) as control: + requests: list[dict[str, Any]] = [ + json.loads(row[0]) + for row in control.execute("SELECT request_json FROM sqb_runs").fetchall() + ] + assert len(requests) == 4 + assert all(request["window"] == test_case.expected_window for request in requests) + assert {request["origin"] for request in requests} == {"backfill", "rerun"} + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute( + "SELECT count(*) FROM sqb_security_audit WHERE action='job.run_requested'" + ).fetchone() == (1,) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py new file mode 100644 index 0000000000..9f3a98f241 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py @@ -0,0 +1,292 @@ +"""Calendar callbacks remain in locations; native publication and refresh preserve good state.""" + +import json +import sqlite3 +import subprocess +import sys +from dataclasses import replace +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any +from urllib.error import HTTPError + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + CalendarRefreshTestCase, + CalendarRejectionTestCase, + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + current_snapshot, + location_process, + scheduled_worker, + snapshot_api, + wait_for_generation, + wait_for_location_health, + wait_for_run, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + CalendarRejectionTestCase( + "naive time", "[datetime(2026, 1, 1)]", "timezone-aware datetimes" + ), + CalendarRejectionTestCase("exclusive end", "[window.end]", "half-open evaluation window"), + CalendarRejectionTestCase( + "before start", + "[window.start - timedelta(microseconds=1)]", + "half-open evaluation window", + ), + CalendarRejectionTestCase( + "non-datetime", "['2026-01-01T00:00:00Z']", "timezone-aware datetimes" + ), + CalendarRejectionTestCase( + "unbounded generator", "repeat(window.start)", "exceeds 10000 upcoming times" + ), + CalendarRejectionTestCase("zero horizon", "[]", "positive fixed duration", horizon="0s"), + CalendarRejectionTestCase( + "overflowing horizon", + "[]", + "positive fixed duration", + horizon="999999999999999999999999999d", + ), + CalendarRejectionTestCase("unknown zone", "[]", "IANA timezone", timezone="Unknown/Orders"), + ], + ids=lambda case: case.description, +) +def test_given_invalid_calendar_results_when_publishing_through_the_cli_then_validation_is_bounded_and_no_snapshot_is_committed( + test_case: CalendarRejectionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + '\n[targets.development]\nschema = "main"\n', + "orchestration/orders.py": ( + "from datetime import datetime, timedelta\nfrom itertools import repeat\n" + "from sqlbuild.orchestration import calendar_schedule, job\n" + "orders = job('orders', description='Order reporting.', select='orders')\n" + f"@calendar_schedule(job=orders, description='Delivery times.', horizon={test_case.horizon!r}, timezone={test_case.timezone!r})\n" + f"def deliveries(window):\n return {test_case.expression}\n" + ), + }, + ) + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "publish", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + "--generation", + "1", + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert result.returncode != 0 + assert test_case.expected_message in result.stderr + with pytest.raises(HTTPError) as failure: + current_snapshot(api, "development") + assert failure.value.code == 404 + + +@pytest.mark.parametrize( + "test_case", + [ + CalendarRefreshTestCase("offset-aware calendar refresh", "America/New_York"), + CalendarRefreshTestCase("UTC calendar refresh", "UTC"), + ], + ids=lambda case: case.description, +) +def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refresh_reuses_the_interpreter_and_keeps_last_good( + test_case: CalendarRefreshTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + data: Path = tmp_path / "fulfillment.json" + evaluations: Path = tmp_path / "evaluations.jsonl" + data.write_text(json.dumps({"offsets": [3, 0, 3]})) + monkeypatch.setenv("SQLBUILD_FULFILLMENT_DATA", str(data)) + monkeypatch.setenv("SQLBUILD_CALENDAR_EVALUATIONS", str(evaluations)) + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + '\n[targets.development]\nschema = "main"\n', + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py" + ) + .read_text() + .replace("America/New_York", test_case.expected_timezone) + + ( + "\n@calendar_schedule(job=orders, description='Other target.', horizon='8s', timezone='UTC', targets=('production',))\n" + "def excluded_calendar(window):\n raise ValueError('excluded calendar must not evaluate')\n" + ), + "orchestration/platform_orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ).read_text(), + "providers/fulfillment.py": ( + "import json, os\nfrom pathlib import Path\nfrom datetime import timedelta, timezone\n" + "from sqlbuild.provider.classes.provider import Provider\n" + "class Fulfillment(Provider):\n" + ' """Upcoming synthetic fulfillment times."""\n' + " def setup(self, ctx):\n self.record({'phase': 'setup'})\n" + " def teardown(self):\n self.record({'phase': 'teardown'})\n" + " def record(self, value):\n" + " with Path(os.environ['SQLBUILD_CALENDAR_EVALUATIONS']).open('a') as out:\n" + " out.write(json.dumps(value | {'pid': os.getpid()}) + '\\n')\n" + " def upcoming(self, window):\n" + " self.record({'phase': 'evaluate', 'start': window.start.isoformat(), 'end': window.end.isoformat(), 'zone': window.start.tzinfo.key})\n" + " data = json.loads(Path(os.environ['SQLBUILD_FULFILLMENT_DATA']).read_text())\n" + " if data.get('fail'):\n raise ValueError('synthetic calendar unavailable')\n" + " start = window.start.astimezone(timezone.utc)\n" + " return [start + timedelta(seconds=offset) for offset in data['offsets']]\n" + ), + }, + ) + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + log: Path = tmp_path / "location.log" + publications: Path = tmp_path / "publications.jsonl" + arguments: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ] + with location_process(arguments, log, publications) as process: + first: dict[str, Any] = wait_for_generation(api, "development", 1, process, log) + second: dict[str, Any] = wait_for_generation(api, "development", 2, process, log) + assert first["nodes"] == second["nodes"] + assert second["changes"] == [] + assert first["documents"]["triggers"].keys() == { + "fulfillment_calendar", + "overdue_orders", + } + with sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3") as control: + stored, next_due = control.execute( + "SELECT definition_json, next_due_at FROM sqb_triggers JOIN sqb_trigger_state USING (trigger_id) WHERE kind = 'schedule'" + ).fetchone() + original_times: list[str] = first["documents"]["triggers"]["fulfillment_calendar"][ + "schedule" + ]["times"] + assert json.loads(stored)["schedule"]["times"] == sorted( + original_times + + second["documents"]["triggers"]["fulfillment_calendar"]["schedule"]["times"] + ) + assert datetime.fromisoformat(next_due) == datetime.fromisoformat(original_times[0]) + observations: list[dict[str, Any]] = [ + json.loads(line) for line in evaluations.read_text().splitlines() + ] + assert [row["phase"] for row in observations] == ["setup", "evaluate", "teardown"] * 2 + successful: list[dict[str, Any]] = observations[1::3] + assert len(successful) == 2 + assert len({row["pid"] for row in observations}) == 1 + assert all(row["zone"] == test_case.expected_timezone for row in successful) + for snapshot, evaluation in zip([first, second], successful, strict=True): + start: datetime = datetime.fromisoformat(evaluation["start"]) + end: datetime = datetime.fromisoformat(evaluation["end"]) + assert end - start == timedelta(seconds=8) + schedule: dict[str, Any] = snapshot["documents"]["triggers"][ + "fulfillment_calendar" + ]["schedule"] + assert schedule["kind"] == "calendar" + assert [datetime.fromisoformat(value) for value in schedule["times"]] == [ + start + timedelta(seconds=offset) for offset in [0, 3] + ] + assert datetime.fromisoformat(schedule["refresh_at"]) == start + timedelta( + seconds=4 + ) + data.write_text(json.dumps({"fail": True})) + _ = wait_for_location_health( + api, "development", {"kind": "compile_failed", "generation": 3}, process, log + ) + assert current_snapshot(api, "development")["record"] == second["record"] + observations = [json.loads(line) for line in evaluations.read_text().splitlines()] + assert [row["phase"] for row in observations] == ["setup", "evaluate", "teardown"] * 3 + data.write_text(json.dumps({"offsets": [1]})) + (project / "models/stg_orders.sql").write_text( + TYPE_PROOF_FILES["models/stg_orders.sql"].replace( + "CAST(5 AS INTEGER)", "CAST(19 AS INTEGER)" + ) + ) + third: dict[str, Any] = wait_for_generation(api, "development", 3, process, log) + assert "CAST(19 AS INTEGER)" in third["nodes"]["model:stg_orders"]["raw_sql"] + assert ( + len(third["documents"]["triggers"]["fulfillment_calendar"]["schedule"]["times"]) + == 1 + ) + assert [ + json.loads(line)["generation"] for line in publications.read_text().splitlines() + ] == [1, 2, 3] + assert api.bearer not in log.read_text() + publications.read_text() + assert not (project / ".sqlbuild/control.sqlite3").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + description="calendar evaluation belongs only to the location", + model_sql="SELECT 29 AS amount", + warehouse_query="SELECT amount FROM orders", + expected_status="success", + expected_rows=[(29,)], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_published_calendar_when_a_real_worker_compiles_then_it_does_not_evaluate_the_calendar_again( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + evaluated: Path = tmp_path / "evaluated" + test_case = replace( + test_case, + files=test_case.files + | { + "orchestration/orders.py": ( + "from pathlib import Path\nfrom sqlbuild.orchestration import calendar_schedule, job, schedule\n" + "orders = job('orders', description='Order reporting.', select='orders', schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "@calendar_schedule(job=orders, description='Upcoming deliveries.', horizon='1d', timezone='UTC')\n" + "def deliveries(window):\n" + f" marker = Path({str(evaluated)!r})\n" + " if marker.exists():\n raise ValueError('calendar evaluated inside a run worker')\n" + " marker.write_text('evaluated')\n return []\n" + ) + }, + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + assert evaluated.read_text() == "evaluated" + completed: dict[str, Any] = wait_for_run(worker.api, worker.process, worker.log) + assert completed["status"] == test_case.expected_status, worker.log.read_text() + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_cli_login.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_cli_login.py new file mode 100644 index 0000000000..1cce5df4b1 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_cli_login.py @@ -0,0 +1,294 @@ +"""Installed login persists a bounded person session, reloads it, and observes live revocation.""" + +import json +import shutil +import stat +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + CliCleanupTestCase, + CliLoginTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.login.helpers import ( + assert_private_storage, + authentication_arguments, + browser, + gateway, + pending_login, + request, + sign_in, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.cli.helpers import ( + administer, + environment, + serve, + successful, +) + + +@pytest.mark.skipif( + sys.platform != "linux", reason="Linux owner-only fallback acceptance requires a Linux runtime" +) +@pytest.mark.parametrize( + "test_case", + [ + CliLoginTestCase( + "installed HTTPS password handoff and credential lifecycle", "password", "orders-reader" + ), + CliLoginTestCase( + "installed HTTPS OIDC handoff and credential lifecycle", "oidc", "orders-reader" + ), + CliLoginTestCase( + "installed HTTPS proxy handoff and credential lifecycle", "proxy", "orders-reader" + ), + ], + ids=lambda case: case.description, +) +def test_given_installed_browser_login_when_reloaded_revoked_and_logged_out_then_person_session_and_storage_boundaries_hold( + test_case: CliLoginTestCase, + tmp_path: Path, +) -> None: + binary: str | None = shutil.which("sqb") + assert binary is not None + installation: tuple[Path, Path] = Path(binary), tmp_path + home: Path = tmp_path / "person-home" + home.mkdir() + project: Path = tmp_path / "empty-project" + project.mkdir() + fixture: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/cli_login.json").read_text() + ) + successful(administer(installation, ["create-user", "orders-admin"])) + person: dict[str, Any] = successful( + administer(installation, ["create-principal", test_case.expected_person_name]) + ) + successful( + administer(installation, ["set-password", person["principal_id"]], fixture["password"]) + ) + client_environment: dict[str, str] = environment({"HOME": str(home)}) + client_environment.pop("DBUS_SESSION_BUS_ADDRESS", None) + client_environment.pop("XDG_RUNTIME_DIR", None) + with gateway(tmp_path) as (origin, certificate, transport): + transport.update({"proxy-subject": "orders-person"}) + arguments: tuple[str, ...] = authentication_arguments( + test_case.authentication, origin, certificate, person["principal_id"] + ) + with serve(installation, authentication_arguments=arguments) as backend: + transport["backend"] = backend.removeprefix("http://") + client, _ = browser(origin, certificate) + csrf: str = sign_in( + client, origin, test_case.authentication, person["name"], fixture["password"] + ) + assert request(client, origin + "/api/control/audit")[0] == 403 + common: list[str] = ["--server", origin, "--ca-cert", str(certificate)] + log: Path = tmp_path / "cli-login.log" + with pending_login( + [binary, "login", *common, "--no-browser"], client_environment, project, log + ) as (process, url): + assert url.startswith(f"{origin}/auth/cli#") + status, payload = request(client, url.split("#")[0]) + assert status == 200 and b"Authorize CLI" in payload + status, payload = request( + client, + f"{origin}/api/auth/cli/approve", + { + "browser_code": url.split("#", 1)[1], + "approve": True, + }, + csrf=csrf, + ) + assert status == 204 and payload == b"" + stdout, _ = process.communicate(timeout=20) + assert process.returncode == 0, "CLI handoff did not complete" + assert not stdout and "signed in successfully" in log.read_text() + assert not list(project.iterdir()) + credential_files: list[Path] = list( + (home / ".config/sqlbuild/credentials").glob("*.json") + ) + assert len(credential_files) == 1 + assert stat.S_IMODE(credential_files[0].stat().st_mode) == 0o600 + assert stat.S_IMODE(credential_files[0].parent.stat().st_mode) == 0o700 + saved: dict[str, Any] = json.loads(credential_files[0].read_text()) + assert saved["bearer"] not in log.read_text() + assert "csrf_token" not in saved + assert ( + saved["identity"]["session_id"] + not in request(client, f"{origin}/api/auth/session")[1].decode() + ) + checked: subprocess.CompletedProcess[str] = subprocess.run( + [binary, "login", "status", *common], + cwd=project, + env=client_environment, + text=True, + capture_output=True, + timeout=20, + check=False, + ) + assert checked.returncode == 0 + assert json.loads(checked.stdout)["principal_id"] == person["principal_id"] + assert "bearer" not in checked.stdout and "csrf_token" not in checked.stdout + assert_private_storage( + credential_files[0], + [binary, "login", "status", *common], + client_environment, + project, + ) + successful( + administer( + installation, ["set-password", person["principal_id"]], fixture["replacement"] + ) + ) + revoked: subprocess.CompletedProcess[str] = subprocess.run( + [binary, "login", "status", *common], + cwd=project, + env=client_environment, + text=True, + capture_output=True, + timeout=20, + check=False, + ) + assert revoked.returncode != 0 and not revoked.stdout and "revoked" in revoked.stderr + for logout_arguments, expected_exit in [ + (["--server", origin, "--ca-cert", str(tmp_path / "missing-ca.pem")], 1), + (common, 0), + ]: + logout: subprocess.CompletedProcess[str] = subprocess.run( + [binary, "logout", *logout_arguments], + cwd=project, + env=client_environment, + text=True, + capture_output=True, + timeout=20, + check=False, + ) + assert logout.returncode == expected_exit and not logout.stdout + assert not list(credential_files[0].parent.iterdir()) + assert not list(project.iterdir()) + + +@pytest.mark.skipif(sys.platform != "linux", reason="Linux fallback path regression") +@pytest.mark.parametrize( + "test_case", + [ + CliCleanupTestCase( + "absent fallback permits HTTPS logout", "https://control.example.com", 0 + ), + CliCleanupTestCase("plaintext origin is still refused", "http://control.example.com", 1), + ], + ids=lambda case: case.description, +) +def test_given_absent_fallback_when_configuration_is_symlinked_then_logout_does_not_create_or_validate_unused_storage( + test_case: CliCleanupTestCase, + tmp_path: Path, +) -> None: + binary: str | None = shutil.which("sqb") + assert binary is not None + home: Path = tmp_path / "home" + home.mkdir() + configuration: Path = tmp_path / "configuration" + configuration.mkdir() + (home / ".config").symlink_to(configuration, target_is_directory=True) + values: dict[str, str] = environment({"HOME": str(home)}) + values.pop("DBUS_SESSION_BUS_ADDRESS", None) + values.pop("XDG_RUNTIME_DIR", None) + logout: subprocess.CompletedProcess[str] = subprocess.run( + [binary, "logout", "--server", test_case.server], + cwd=tmp_path, + env=values, + text=True, + capture_output=True, + timeout=20, + check=False, + ) + assert logout.returncode == test_case.expected_exit and not logout.stdout + assert not list(configuration.iterdir()) + + +@pytest.mark.skipif(sys.platform != "linux", reason="Linux fallback permission failure") +@pytest.mark.parametrize( + "test_case", + [ + CliLoginTestCase( + "readable credential still revokes remotely when local unlink fails", + "password", + "orders-reader", + ) + ], + ids=lambda case: case.description, +) +def test_given_live_cli_session_when_local_unlink_fails_then_logout_still_revokes_remotely( + test_case: CliLoginTestCase, + tmp_path: Path, +) -> None: + binary: str | None = shutil.which("sqb") + assert binary is not None + installation: tuple[Path, Path] = Path(binary), tmp_path + home: Path = tmp_path / "home" + home.mkdir() + values: dict[str, str] = environment({"HOME": str(home)}) + values.pop("DBUS_SESSION_BUS_ADDRESS", None) + values.pop("XDG_RUNTIME_DIR", None) + successful(administer(installation, ["create-user", "orders-admin"])) + person: dict[str, Any] = successful( + administer(installation, ["create-principal", test_case.expected_person_name]) + ) + successful( + administer( + installation, ["set-password", person["principal_id"]], "orders-private-password" + ) + ) + with gateway(tmp_path) as (origin, certificate, transport): + arguments: tuple[str, ...] = authentication_arguments( + test_case.authentication, origin, certificate, person["principal_id"] + ) + with serve(installation, authentication_arguments=arguments) as backend: + transport["backend"] = backend.removeprefix("http://") + browser_client, _ = browser(origin, certificate) + csrf: str = sign_in( + browser_client, + origin, + test_case.authentication, + person["name"], + "orders-private-password", + ) + common: list[str] = ["--server", origin, "--ca-cert", str(certificate)] + with pending_login( + [binary, "login", *common, "--no-browser"], values, tmp_path, tmp_path / "cli.log" + ) as (process, url): + assert ( + request( + browser_client, + origin + "/api/auth/cli/approve", + {"browser_code": url.split("#", 1)[1], "approve": True}, + csrf=csrf, + )[0] + == 204 + ) + process.communicate(timeout=20) + assert process.returncode == 0 + credential: Path = next((home / ".config/sqlbuild/credentials").glob("*.json")) + saved: dict[str, Any] = json.loads(credential.read_text()) + cli_client, _ = browser(origin, certificate) + cli_client.addheaders = [("Authorization", f"Session {saved['bearer']}")] + assert request(cli_client, origin + "/api/auth/session")[0] == 200 + credential.parent.chmod(0o500) + try: + logout: subprocess.CompletedProcess[str] = subprocess.run( + [binary, "logout", *common], + cwd=tmp_path, + env=values, + text=True, + capture_output=True, + timeout=20, + check=False, + ) + assert logout.returncode != 0 and not logout.stdout + assert credential.exists() + assert request(cli_client, origin + "/api/auth/session")[0] == 401 + finally: + credential.parent.chmod(0o700) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py new file mode 100644 index 0000000000..30736d5b25 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py @@ -0,0 +1,308 @@ +"""Acceptance at the distributed package boundary, not the source-tree runtime mounts.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import time +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ComposeDeploymentTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.docker.compose.helpers import ( + ComposeStack, + platform_receiver, + wait_for_location_outage, +) + + +@pytest.mark.skipif( + os.environ.get("SQLBUILD_REQUIRE_COMPOSE_TESTS") != "1", + reason="Run make test-compose to build and test the packaged deployment", +) +@pytest.mark.parametrize( + "test_case", + [ + ComposeDeploymentTestCase( + "PostgreSQL control store and immutable DuckDB project images", [[41]] + ) + ], + ids=lambda case: case.description, +) +def test_given_packaged_services_when_redeployed_and_restarted_then_pinned_runs_and_platform_events_survive( + tmp_path: Path, + test_case: ComposeDeploymentTestCase, +) -> None: + stack: ComposeStack = ComposeStack(tmp_path) + with platform_receiver() as (port, events): + stack.environment["SQLBUILD_PLATFORM_WEBHOOK_URL"] = ( + f"http://host.docker.internal:{port}/events" + ) + try: + first_image, second_image, broken_image = stack.build_projects() + stack.register() + stack.compose("up", "-d", "--wait", "server") + stack.compose("up", "-d", "--no-deps", "location") + first: dict[str, Any] = stack.wait( + "SELECT * FROM sqb_runs WHERE status='running' AND origin='schedule'" + )[0] + assert first["image_id"] == first_image + worker = f"sqlbuild-{stack.location_id}-{first['run_id']}" + original: dict[str, Any] = json.loads( + stack.docker("inspect", "--format", "{{json .State}}", worker).stdout + ) + + # A real replacement removes the old network endpoint; merely restarting is insufficient. + stack.compose("stop", "--timeout", "1", "location") + stack.redeploy("second") + after: dict[str, Any] = json.loads( + stack.docker("inspect", "--format", "{{json .State}}", worker).stdout + ) + assert after["Running"] is True + assert after["Pid"] == original["Pid"] + assert ( + stack.docker("inspect", "--format", "{{.Image}}", worker).stdout.strip() + == first_image + ) + second: dict[str, Any] = stack.wait( + f"SELECT snapshot_id,image_id FROM sqb_snapshots WHERE image_id='{second_image}'" + )[0] + assert second["snapshot_id"] != first["snapshot_id"] + stack.wait( + f"SELECT run_id,status FROM sqb_runs WHERE run_id='{first['run_id']}' AND status='success'" + ) + + # These are real Docker/credential/mount boundaries, without exposing their values. + names: list[str] = json.loads( + stack.docker("inspect", "--format", "{{json .Config.Env}}", worker).stdout + ) + names = [entry.split("=", 1)[0] for entry in names] + assert "SQLBUILD_RUN_TOKEN" in names + assert not { + "SQLBUILD_LOCATION_TOKEN", + "SQLBUILD_CONTROL_DATABASE_URL", + "SQLBUILD_PLATFORM_WEBHOOK_URL", + }.intersection(names) + mounts: list[dict[str, Any]] = json.loads( + stack.docker("inspect", "--format", "{{json .Mounts}}", worker).stdout + ) + assert {mount["Destination"] for mount in mounts} == {"/warehouse"} + assert ( + stack.docker( + "inspect", "--format", "{{.HostConfig.NetworkMode}}", worker + ).stdout.strip() + == f"{stack.name}_workers" + ) + assert ( + stack.compose( + "exec", + "-T", + "server", + "sh", + "-c", + "! command -v python && ! command -v python3", + ) + == "" + ) + + # A location cannot substitute an operator-provided equivalent hash for physical evidence. + mismatch: subprocess.CompletedProcess[str] = stack.docker( + "exec", + f"{stack.name}-location", + "sqb", + "location", + "publish", + "--server", + "http://server:8000", + "--location-id", + stack.location_id, + "--target", + "development", + "--generation", + "99", + "--image-id", + "sha256:" + "0" * 64, + check=False, + ) + assert mismatch.returncode != 0 + assert "409" in mismatch.stderr + assert ( + stack.query("SELECT MAX(generation) AS generation FROM sqb_snapshots")[0][ + "generation" + ] + == 2 + ) + + stack.redeploy("broken") + failure: dict[str, Any] = stack.wait( + "SELECT occurred_at FROM recall_events WHERE event_type='location_health' AND payload->'report'->'health'->>'kind'='compile_failed' ORDER BY storage_id DESC" + )[0] + current: dict[str, Any] = stack.query( + "SELECT image_id,snapshot_id FROM sqb_snapshots JOIN sqb_current_snapshots USING(snapshot_id,location_id,target)" + )[0] + assert current == second + assert current["image_id"] != broken_image + + # Queue the next scheduled occurrence from the surviving declaration, not a manual run. + fallback: dict[str, Any] = stack.wait( + f"SELECT run_id,status,image_id,snapshot_id FROM sqb_runs WHERE image_id='{second_image}' AND origin='schedule' AND queued_at>'{failure['occurred_at']}' AND status='success'" + )[0] + assert fallback["snapshot_id"] == second["snapshot_id"] + warehouse: str = stack.compose( + "exec", + "-T", + "location", + "python", + "-c", + "import duckdb,json; print(json.dumps(duckdb.connect('/warehouse/orders.duckdb',read_only=True).sql('SELECT order_id FROM orders').fetchall()))", + ) + assert json.loads(warehouse) == test_case.expected_warehouse_rows + + # Let a real minute boundary pass with no scheduler process, then catch up once. + stack.compose("stop", "--timeout", "5", "server") + before: int = stack.query("SELECT count(*) AS count FROM sqb_runs")[0]["count"] + due: str = stack.query("SELECT next_due_at FROM sqb_trigger_state")[0]["next_due_at"] + deadline: float = datetime.fromisoformat(due).timestamp() + 2 + while datetime.now(UTC).timestamp() < deadline: + time.sleep(0.5) + assert stack.query("SELECT count(*) AS count FROM sqb_runs")[0]["count"] == before + stack.compose("up", "-d", "--wait", "server") + stack.wait( + f"SELECT r.run_id,r.status,t.scheduled_for FROM sqb_runs r JOIN sqb_ticks t USING(tick_id) WHERE t.scheduled_for='{due}' AND r.status='success'" + ) + assert ( + stack.query( + "SELECT trigger_id,scheduled_for,count(*) FROM sqb_ticks GROUP BY trigger_id,scheduled_for HAVING count(*)>1" + ) + == [] + ) + + prior_events: set[str] = {event["event_id"] for event in events} + stack.compose("stop", "--timeout", "1", "location") + stopped_at: datetime = datetime.now(UTC) + broken: dict[str, Any] = wait_for_location_outage(events, prior_events) + assert broken["payload"]["platform"]["location_id"] == stack.location_id + assert broken["payload"]["platform"]["target"] == "development" + assert ( + datetime.fromisoformat(broken["payload"]["platform"]["last_seen_at"]) <= stopped_at + ) + + # Even a valid location bearer cannot publish without physical image evidence. + stack.compose("rm", "--force", "location") + unavailable: subprocess.CompletedProcess[str] = stack.docker( + "run", + "--rm", + "--network", + f"{stack.name}_workers", + "--env", + "SQLBUILD_LOCATION_TOKEN", + f"{stack.name}:second", + "location", + "publish", + "--server", + "http://server:8000", + "--location-id", + stack.location_id, + "--target", + "development", + "--generation", + "99", + check=False, + ) + assert unavailable.returncode != 0 + assert "503" in unavailable.stderr + assert ( + stack.query( + "SELECT image_id,snapshot_id FROM sqb_snapshots JOIN sqb_current_snapshots USING(snapshot_id,location_id,target)" + )[0] + == second + ) + + stack.verify_control_store_restore(second) + finally: + (tmp_path / "platform-events.json").write_text(json.dumps(events, indent=2)) + stack.close() + + +@pytest.mark.skipif( + os.environ.get("SQLBUILD_REQUIRE_COMPOSE_TESTS") != "1", + reason="Run make test-compose to build and test the packaged deployment", +) +@pytest.mark.parametrize( + "test_case", + [ + ComposeDeploymentTestCase( + "Host-owned development source with isolated runtime writes", [[41]] + ) + ], + ids=lambda case: case.description, +) +def test_given_host_owned_source_when_using_the_development_overlay_then_reloaded_workers_can_write_runtime_outputs( + tmp_path: Path, + test_case: ComposeDeploymentTestCase, +) -> None: + stack: ComposeStack = ComposeStack(tmp_path) + stack.environment["SQLBUILD_LOCATION_IMAGE"] = os.environ.get( + "SQLBUILD_TEST_LOCATION_IMAGE", "sqlbuild-location:test" + ) + project: Path = tmp_path / "source" + shutil.copytree(Path("deploy/project"), project) + assert project.stat().st_uid != 10001 + override: dict[str, Any] = json.loads(stack.override.read_text()) + override["services"]["location"] = { + "volumes": [{"type": "bind", "source": str(project), "target": "/project"}] + } + stack.override.write_text(json.dumps(override)) + stack.compose_files.insert(1, Path("deploy/compose.dev.yml").resolve()) + try: + stack.register() + stack.compose("up", "-d", "--wait", "server") + stack.compose("up", "-d", "--no-deps", "location") + first: dict[str, Any] = stack.wait( + "SELECT snapshot_id FROM sqb_snapshots WHERE generation=1", timeout=60 + )[0] + cases: dict[str, str] = json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/container_deployment.json" + ).read_text() + ) + (project / "models/orders.sql").write_text( + "MODEL (description 'Example orders.', materialized table);\n" + cases["second"] + "\n" + ) + second: dict[str, Any] = stack.wait( + "SELECT snapshot_id FROM sqb_snapshots WHERE generation=2", timeout=45 + )[0] + assert second != first + stack.wait( + f"SELECT run_id FROM sqb_runs WHERE snapshot_id='{second['snapshot_id']}' " + "AND origin='schedule' AND status='success'", + timeout=100, + ) + warehouse: str = stack.compose( + "exec", + "-T", + "location", + "python", + "-c", + "import duckdb,json; print(json.dumps(duckdb.connect('/warehouse/orders.duckdb',read_only=True).sql('SELECT order_id FROM orders').fetchall()))", + ) + assert json.loads(warehouse) == test_case.expected_warehouse_rows + (project / "models/orders.sql").write_text(cases["broken"] + "\n") + stack.wait( + "SELECT event_id FROM recall_events WHERE event_type='location_health' " + "AND payload->'report'->'health'->>'kind'='compile_failed'", + timeout=45, + ) + assert stack.query( + "SELECT snapshot_id FROM sqb_snapshots " + "JOIN sqb_current_snapshots USING(snapshot_id,location_id,target)" + ) == [second] + finally: + stack.close() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py new file mode 100644 index 0000000000..e6da7b8e1b --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py @@ -0,0 +1,205 @@ +"""A real location callback times out, restarts and queues a real DuckDB worker.""" + +import ast +import json +import os +import sqlite3 +import sys +from pathlib import Path +from typing import Any, cast + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + location_process, + scheduled_worker, + sensor_calls, + wait_for_generation, + wait_for_location_work, + wait_for_run, + wait_for_sensor_phase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.person.helpers import ( + person_login, + person_read, + person_request, + provision_person, + verify_compute_transport, +) + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "polling fulfillment sensor", + "SELECT 83 AS amount", + "SELECT amount FROM orders", + "success", + [(83,)], + expected_origin="sensor", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/custom_orders.py" + ).read_text(), + "providers/fulfillment.py": ( + "import json, os, time\nfrom pathlib import Path\n" + "from sqlbuild.orchestration import RunRequest, SkipReason\n" + "from sqlbuild.provider.classes.provider import Provider\n" + "class Fulfillment(Provider):\n" + ' """Synthetic fulfillment readiness."""\n' + " def record(self, value):\n" + " with Path(os.environ['SQLBUILD_SENSOR_CALLS']).open('a') as out:\n" + " out.write(json.dumps(value | {'pid': os.getpid()}) + '\\n')\n" + " def setup(self, ctx):\n self.record({'phase': 'setup'})\n" + " def teardown(self):\n self.record({'phase': 'teardown'})\n" + " def poll(self, ctx):\n" + " mode = json.loads(Path(os.environ['SQLBUILD_FULFILLMENT_DATA']).read_text())['mode']\n" + " self.record({'phase': 'evaluate', 'cursor': ctx.cursor, 'evaluation_id': ctx.evaluation_id, 'target': ctx.target, 'mode': mode})\n" + " if mode == 'failed':\n" + " ctx.cursor = 'uncommitted'\n" + " raise ValueError('synthetic source unavailable')\n" + " if mode == 'timeout':\n time.sleep(90)\n" + " if ctx.cursor:\n return SkipReason('No new orders')\n" + " ctx.target = 'not-an-authority-input'\n" + " ctx.evaluation_id = 'not-an-authority-input'\n" + " ctx.cursor = 'orders/page-1'\n" + " return [RunRequest(run_key='orders-1'), RunRequest(run_key='orders-1')]\n" + ), + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_custom_polling_sensor_when_a_callback_fails_or_times_out_then_a_fresh_process_retries_without_advancing_its_cursor( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + context_module: ast.Module = ast.parse( + Path("src/sqlbuild/orchestration_declarations/models/sensor_context.py").read_text() + ) + assert [type(node).__name__ for node in context_module.body] == [ + "Expr", + "ImportFrom", + "ClassDef", + ] + declaration: ast.ClassDef = cast(ast.ClassDef, context_module.body[-1]) + assert declaration.name == "SensorContext" + assert [type(node).__name__ for node in declaration.body] == [ + "Expr", + "AnnAssign", + "AnnAssign", + "AnnAssign", + ] + data: Path = tmp_path / "fulfillment.json" + calls: Path = tmp_path / "sensor-calls.jsonl" + data.write_text(json.dumps({"mode": "failed"})) + monkeypatch.setenv("SQLBUILD_FULFILLMENT_DATA", str(data)) + monkeypatch.setenv("SQLBUILD_SENSOR_CALLS", str(calls)) + with scheduled_worker( + tmp_path, + monkeypatch, + test_case, + server_arguments=("--password-origin", "https://control.example.com"), + ) as worker: + administrator: dict[str, Any] = provision_person( + tmp_path / "control", "create-user", "orders-admin" + ) + person: dict[str, Any] = provision_person( + tmp_path / "control", "create-principal", "orders-developer" + ) + admin: tuple[str, str] = person_login(worker.api, administrator["name"]) + session: tuple[str, str] = person_login(worker.api, person["name"]) + assert person_request( + worker.api, + "/api/admin/roles", + body={ + "principal_id": person["principal_id"], + "role": "developer", + "location_id": worker.api.location_id, + "target": "development", + }, + session=admin, + ) == {"applied": True} + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", worker.api.bearer) + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + log: Path = tmp_path / "location.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(worker.project), + "location", + "serve", + "--server", + worker.api.url, + "--location-id", + worker.api.location_id, + "--target", + "development", + ], + log, + tmp_path / "publications.jsonl", + ) as location: + _ = wait_for_generation(worker.api, "development", 2, location, log) + failed: dict[str, Any] = wait_for_sensor_phase(worker.control_store, "failed") + assert failed["request"]["cursor"] == {"event_storage_id": 0} + work: dict[str, Any] = wait_for_location_work( + worker.api, failed["request"]["evaluation_id"], location, log + ) + assert work["request_json"]["work"]["kind"] == "evaluate_sensor" + assert work["request_json"]["request_id"] == failed["request"]["evaluation_id"] + path: str = ( + f"/api/control/locations/{worker.api.location_id}/targets/development/connection-tests/" + + failed["request"]["evaluation_id"] + ) + for browser in [admin, session]: + assert person_read(worker.api, path, session=browser) is None + data.write_text(json.dumps({"mode": "timeout"})) + timed_out: dict[str, Any] = wait_for_sensor_phase(worker.control_store, "timed_out") + assert timed_out["request"]["cursor"] == failed["request"]["cursor"] + timed_call: dict[str, Any] = sensor_calls( + calls, {"evaluation_id": timed_out["request"]["evaluation_id"]} + )[0] + with pytest.raises(ProcessLookupError): + os.kill(timed_call["pid"], 0) + data.write_text(json.dumps({"mode": "ready"})) + destination: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert destination["status"] == test_case.expected_status, log.read_text() + for browser in [admin, session]: + verify_compute_transport(worker.api, browser, destination["run_id"]) + # The next successful evaluation skips but retains its committed cursor. + skipped: dict[str, Any] = wait_for_sensor_phase( + worker.control_store, "completed", empty_runs=True + ) + assert skipped["outcome"]["cursor"]["custom"] == "orders/page-1" + evaluated: list[dict[str, Any]] = sensor_calls(calls, {"phase": "evaluate"}) + assert {record["target"] for record in evaluated} == {"development"} + assert len({record["pid"] for record in evaluated}) == len(evaluated) + assert { + record["cursor"] + for record in sensor_calls(calls, {"phase": "evaluate", "mode": "failed"}) + } == {None} + assert { + record["cursor"] + for record in sensor_calls(calls, {"phase": "evaluate", "mode": "timeout"}) + } == {None} + assert location.poll() is None and worker.process.poll() is None + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT run_key,run_id FROM sqb_run_requests" + ).fetchall() == [("orders-1", destination["run_id"])] + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + with duckdb.connect( + str(worker.project / "warehouse.duckdb"), read_only=True + ) as warehouse: + assert ( + warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + ) + assert not (worker.project / ".sqlbuild/control.sqlite3").exists() + assert worker.api.bearer not in log.read_text() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_denial.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_denial.py new file mode 100644 index 0000000000..2c1f786811 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_denial.py @@ -0,0 +1,140 @@ +"""Denial metadata identifies durable decisions without granting access to their scope.""" + +import json +import sqlite3 +from pathlib import Path +from typing import Any +from urllib.error import HTTPError +from uuid import UUID, uuid4 + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + DenialMetadataTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + snapshot_api, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.person.helpers import ( + person_login, + person_read, + person_request, + provision_person, +) + + +@pytest.mark.parametrize( + "test_case", + [ + DenialMetadataTestCase( + description="durable ID without scope disclosure", expected_denial_count=6 + ) + ], + ids=lambda case: case.description, +) +def test_given_denied_browser_requests_when_returned_then_only_durable_decision_metadata_is_exposed( + test_case: DenialMetadataTestCase, + tmp_path: Path, +) -> None: + with snapshot_api( + tmp_path / "control", + server_arguments=("--password-origin", "https://control.example.com"), + ) as api: + administrator: dict[str, Any] = provision_person( + tmp_path / "control", "create-user", "orders-admin" + ) + person: dict[str, Any] = provision_person( + tmp_path / "control", "create-principal", "orders-reader" + ) + admin_session: tuple[str, str] = person_login(api, administrator["name"]) + session: tuple[str, str] = person_login(api, person["name"]) + catalog: str = ( + f"/api/control/locations/{api.location_id}/targets/development/catalog?schema=main" + ) + unlisted: str = catalog.replace(api.location_id, str(uuid4())) + decisions: dict[str, str] = {} + for path in [catalog, unlisted, "/api/control/audit"]: + with pytest.raises(HTTPError) as rejected: + person_read(api, path, session=session) + assert rejected.value.code == 403 + assert rejected.value.headers["Cache-Control"] == "no-store" + body: dict[str, Any] = json.load(rejected.value) + assert set(body) == {"error", "decision_id"} + decision: str = str(UUID(body["decision_id"])) + assert body["error"] == ( + "missing permission in this scope; all requested nodes denied; decision " + decision + ) + decisions[decision] = "authorization.denied" + request: dict[str, Any] = { + "run_id": str(uuid4()), + "location_id": api.location_id, + "target": "development", + "job_key": "unlisted_orders_job", + "reason": "Refresh order totals", + "priority": 0, + } + for credentials, expected_status, action in [ + (session, 403, "authorization.denied"), + ((session[0], "wrong-csrf"), 401, "authentication.denied"), + (("", ""), 401, "authentication.denied"), + ]: + with pytest.raises(HTTPError) as rejected: + person_request(api, "/api/control/runs", body=request, session=credentials) + assert rejected.value.code == expected_status + assert rejected.value.headers["Cache-Control"] == "no-store" + body = json.load(rejected.value) + identifier: str = { + "authorization.denied": "decision_id", + "authentication.denied": "request_id", + }[action] + assert set(body) == {"error", identifier} + decision = str(UUID(body[identifier])) + assert ( + body["error"] + == { + "authorization.denied": ( + "missing permission in this scope; all requested nodes denied; decision " + + decision + ), + "authentication.denied": "authentication rejected", + }[action] + ) + assert decision not in decisions + decisions[decision] = action + assignment: dict[str, Any] = { + "principal_id": person["principal_id"], + "role": "viewer", + "location_id": api.location_id, + "target": "development", + } + assert person_request(api, "/api/admin/roles", body=assignment, session=admin_session) == { + "applied": True + } + assert person_read(api, catalog, session=session) == {"records": [], "next_after": None} + with pytest.raises(HTTPError) as rejected: + person_read(api, unlisted, session=session) + assert rejected.value.code == 403 + assert person_request( + api, "/api/admin/roles", body=assignment | {"revoke": True}, session=admin_session + ) == {"applied": True} + with pytest.raises(HTTPError) as rejected: + person_read(api, catalog, session=session) + assert rejected.value.code == 403 + events: list[dict[str, Any]] = person_read( + api, "/api/control/audit?limit=100", session=admin_session + ) + by_id: dict[str, dict[str, Any]] = {event["event_id"]: event for event in events} + assert len(decisions) == test_case.expected_denial_count + recorded: dict[str, str] = { + "authorization.denied": "authorization.denied", + "authentication.denied": "not audited", + } + assert { + decision: by_id.get(decision, {"action": "not audited"})["action"] + for decision in decisions + } == {decision: recorded[action] for decision, action in decisions.items()}, ( + "only authenticated denials reach the durable audit" + ) + with sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3") as store: + assert store.execute("SELECT count(*) FROM sqb_runs").fetchone() == (0,) + assert store.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py new file mode 100644 index 0000000000..5c85fca197 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py @@ -0,0 +1,208 @@ +"""The real development process tree retains identity, reloads, streams and executes DuckDB work.""" + +import json +import os +import sqlite3 +import subprocess +import sys +import time +from itertools import islice +from pathlib import Path +from typing import Any +from urllib.error import HTTPError +from urllib.parse import quote +from urllib.request import Request, urlopen +from uuid import uuid4 + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + DevelopmentTestCase, + SnapshotApi, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + current_snapshot, + location_process, + stop_location, + wait_for_generation, + wait_for_log, + wait_for_run, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.skipif(sys.platform == "win32", reason="Unix signal and process ownership contract") +@pytest.mark.parametrize( + "test_case", + [ + DevelopmentTestCase( + "default target and store", + "development", + (), + "project/.sqlbuild/control.sqlite3", + [(1, 5)], + ), + DevelopmentTestCase( + "encoded target and custom store", + "preview #1", + ("--target", "preview #1", "--control-store-url", "{control}"), + "project/custom-control.sqlite3", + [(1, 5)], + '[targets."preview #1"]\nschema = "main"\n', + ), + DevelopmentTestCase( + "local target takes precedence over project default", + "preview #1", + (), + "project/.sqlbuild/control.sqlite3", + [(1, 5)], + 'target = "preview #1"\n[targets."preview #1"]\nschema = "main"\n', + ), + ], + ids=lambda case: case.description, +) +def test_given_development_when_reloaded_and_restarted_then_local_scope_and_paused_defaults_survive( + test_case: DevelopmentTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + pidfile: Path = tmp_path / "interpreter.pid" + declaration: str = ( + "import os\nfrom pathlib import Path\n" + "assert 'SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL' not in os.environ\n" + f"Path({str(pidfile)!r}).write_text(str(os.getpid()))\n" + + Path("benchmarks/duckshop/edge_cases/orchestration/development_orders.py").read_text() + ) + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": 'default_target = "development"\n' + + TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + '[targets.development]\nschema = "main"\n', + "sqlbuild_local.toml": test_case.local_config, + "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( + "SELECT order_id, amount", "SELECT order_id, CAST(amount AS BIGINT) AS amount" + ), + "orchestration/orders.py": declaration, + }, + ) + executable: str = str(Path(sys.executable).with_name("sqb")) + control: Path = tmp_path / test_case.control_path + arguments: list[str] = [executable, "dev", "--project-dir", str(project), "--port", "0"] + [ + argument.format(control=control) for argument in test_case.arguments + ] + environment: dict[str, str] = os.environ | { + "SQLBUILD_CONTROL_DATABASE_URL": "unsupported://not-development", + "SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL": ( + "http://127.0.0.1:1/audit?token=synthetic-audit-credential" + ), + } + _ = environment.pop("SQLBUILD_PLATFORM_WEBHOOK_URL", None) + log: Path = tmp_path / "development.log" + output: Path = tmp_path / "publications.jsonl" + with location_process(arguments, log, output, env=environment) as process: + wait_for_log(process, log, "Development API ready: ") + url: str = ( + "http://" + log.read_text().partition("Development API ready: ")[2].splitlines()[0] + ) + identity: str = log.read_text().partition("Development location: ")[2].splitlines()[0] + api: SnapshotApi = SnapshotApi(url, identity, "local-development") + scope: str = f"{url}/api/locations/{identity}/targets/{quote(test_case.target, safe='')}" + _ = wait_for_generation(api, test_case.target, 1, process, log) + with urlopen(f"{scope}/triggers", timeout=5) as response: + triggers: list[dict[str, Any]] = json.load(response) + assert len(triggers) == 1 + assert triggers[0]["status"] == "paused" + assert triggers[0]["definition_json"]["default_status"] == "running" + first_interpreter: int = int(pidfile.read_text()) + for headers in [{"Host": "outside.example.com"}, {"Origin": "https://outside.example.com"}]: + with pytest.raises(HTTPError) as rejected: + urlopen(Request(f"{scope}/snapshots", headers=headers), timeout=5) + assert rejected.value.code == 403 + with pytest.raises(HTTPError) as foreign: + urlopen( + f"{url}/api/locations/{uuid4()}/targets/{quote(test_case.target, safe='')}/snapshots", + timeout=5, + ) + assert foreign.value.code == 403 + run_id: str = str(uuid4()) + requested: subprocess.CompletedProcess[str] = subprocess.run( + [ + executable, + "--project-dir", + str(project), + "admin", + "request-run", + identity, + "annual_orders", + "--target", + test_case.target, + "--run-id", + run_id, + "--reason", + "Development execution contract", + ], + env=environment | {"SQLBUILD_CONTROL_DATABASE_URL": str(control)}, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert requested.returncode == 0, requested.stderr + run: dict[str, Any] = wait_for_run( + api, process, log, run_id=run_id, target=test_case.target + ) + assert run["status"] == "success", log.read_text() + time.sleep(1) + assert current_snapshot(api, test_case.target)["record"]["generation"] == 1 + with duckdb.connect(str(project / "warehouse.duckdb"), read_only=True) as warehouse: + assert ( + warehouse.sql("SELECT order_id, amount FROM orders").fetchall() + == test_case.expected_rows + ) + stop_location(process, first_interpreter, log) + _ = log.write_text("") + with location_process( + arguments + ["--no-scheduler", "--triggers"], log, output, env=environment + ) as restarted: + wait_for_log(restarted, log, "Development API ready: ") + url = "http://" + log.read_text().partition("Development API ready: ")[2].splitlines()[0] + assert log.read_text().partition("Development location: ")[2].splitlines()[0] == identity + assert "Starting development scheduler" not in log.read_text() + api = SnapshotApi(url, identity, "local-development") + scope = f"{url}/api/locations/{identity}/targets/{quote(test_case.target, safe='')}" + _ = wait_for_generation(api, test_case.target, 2, restarted, log) + assert int(pidfile.read_text()) != first_interpreter + with urlopen(f"{scope}/triggers", timeout=5) as response: + triggers = json.load(response) + assert triggers[0]["status"] == "paused" + with urlopen(f"{scope}/changes", timeout=5) as stream: + assert stream.readline() == b"event: changed\n" + assert json.loads(stream.readline().removeprefix(b"data: ")) == {"topic": "all"} + _ = (project / "orchestration/orders.py").write_text( + declaration + + '\nadditional_orders = job("additional_orders", description="Additional orders.", select="orders", schedule=annual)\n' + ) + assert f"snapshot:{identity}" in islice( + ( + json.loads(line[6:])["topic"] + for line in filter(lambda line: line.startswith(b"data: "), stream) + ), + 100, + ) + _ = wait_for_generation(api, test_case.target, 3, restarted, log) + with urlopen(f"{scope}/triggers", timeout=5) as response: + statuses: dict[str, str] = { + entry["job_key"]: entry["status"] for entry in json.load(response) + } + assert statuses == {"annual_orders": "paused", "additional_orders": "running"} + stop_location(restarted, int(pidfile.read_text()), log) + with sqlite3.connect(control) as connection: + assert connection.execute("SELECT count(*) FROM sqb_development_locations").fetchone() == ( + 1, + ) + for line in output.read_text().splitlines(): + assert json.loads(line)["location_id"] == identity diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py new file mode 100644 index 0000000000..acd4cde4b9 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py @@ -0,0 +1,336 @@ +"""A published schedule executes in a real pinned container, not a mocked launcher.""" + +import signal +import sqlite3 +import sys +import time +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from threading import Event +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication import helpers +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ScheduledWorker, + WorkerDeployment, + WorkerExecutionTestCase, + WorkerRestartTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.docker.helpers import ( + REQUIRES_DOCKER, + docker_command, + docker_deployment, + lost_start_response, + worker_container, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + cancel_running_statement, + location_process, + running_statement, + scheduled_worker, + wait_for_log, + wait_for_run, + worker_result, +) + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "pinned container inherits project context but not privileged credentials", + "WITH source(value) AS (VALUES ('37')) SELECT CAST(value AS BIGINT) AS amount FROM source", + "SELECT amount FROM orders", + "success", + [(37,)], + launcher="docker", + files={ + "orchestration/orders.py": ( + "import os\nfrom sqlbuild.orchestration import job, schedule\n" + "if 'SQLBUILD_RUN_TOKEN' in os.environ:\n" + " assert os.getenv('SQLBUILD_WORKER_TEST_VALUE') == 'orders-context'\n" + " for name in ('SQLBUILD_LOCATION_TOKEN', 'SQLBUILD_CONTROL_DATABASE_URL', " + "'SQLBUILD_SCHEDULER_TEST_STORE', 'SQLBUILD_PLATFORM_WEBHOOK_URL', " + "'SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL'):\n" + " assert name not in os.environ, name\n" + 'annual_orders = job("annual_orders", select="orders", ' + 'description="Build orders.", ' + 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ) + }, + ), + WorkerExecutionTestCase( + "container warehouse error retains failed node evidence", + "WITH source(value) AS (VALUES ('invalid')) SELECT CAST(value AS BIGINT) AS amount FROM source", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "failed", + [(0,)], + launcher="docker", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_scheduled_docker_job_when_published_then_a_pinned_worker_executes_and_reports( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + assert worker.docker is not None + deadline: float = time.monotonic() + 10 + state: dict[str, object] = worker_container( + worker.docker, worker.api.location_id, run["run_id"] + ) + while state["Running"]: + assert time.monotonic() < deadline, "Terminal worker did not physically exit" + time.sleep(0.02) + state = worker_container(worker.docker, worker.api.location_id, run["run_id"]) + assert state["ExitCode"] == {"success": 0, "failed": 1}[test_case.expected_status] + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "container refuses changed source before writing", + "SELECT 23 AS amount", + "", + "failed", + [], + launcher="docker", + files_after_publication={ + "models/orders.sql": "MODEL (description 'Orders.', materialized table);\nSELECT 41 AS amount\n" + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_pinned_snapshot_when_container_source_drifts_then_no_warehouse_write_is_permitted( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = wait_for_run(worker.api, worker.process, worker.log) + assert run["status"] == test_case.expected_status, worker.log.read_text() + assert run["node_counts_json"] == {"skipped": 1} + assert not (worker.project / "warehouse.duckdb").exists() + assert worker.docker is not None + name: str = f"sqlbuild-{worker.api.location_id}-{run['run_id']}" + assert "node content differs" in docker_command(worker.docker.socket, "logs", name) + assert ( + worker_container(worker.docker, worker.api.location_id, run["run_id"])["ExitCode"] == 1 + ) + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "Docker cancellation stops an active warehouse statement", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "cancelled", + [(0,)], + launcher="docker", + ) + ], + ids=lambda case: case.description, +) +def test_given_an_executing_container_when_cancelled_then_the_statement_stops_before_claim_release( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + cancel_running_statement(worker) + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "killed Docker worker preserves its physical exit code", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "failed", + [(0,)], + launcher="docker", + ) + ], + ids=lambda case: case.description, +) +def test_given_an_executing_container_when_killed_then_the_monitor_records_exit_and_releases_claims( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + active: dict[str, Any] = running_statement(worker) + assert worker.docker is not None + name: str = f"sqlbuild-{worker.api.location_id}-{active['run_id']}" + _ = docker_command(worker.docker.socket, "kill", "--signal", "KILL", name) + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + assert ( + worker_container(worker.docker, worker.api.location_id, run["run_id"])["ExitCode"] + == 137 + ) + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT json_extract(payload, '$.change.change.exit_code') FROM recall_events " + "WHERE event_type='run_terminated'" + ).fetchone() == (137,) + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerRestartTestCase( + "graceful scheduler restart retains the executing container", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "cancelled", + [(0,)], + launcher="docker", + shutdown_signal=signal.SIGTERM, + expected_scheduler_exit=0, + ), + WorkerRestartTestCase( + "abrupt scheduler death retains the executing container", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "cancelled", + [(0,)], + launcher="docker", + shutdown_signal=signal.SIGKILL, + expected_scheduler_exit=-signal.SIGKILL, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_executing_container_when_scheduler_restarts_then_the_same_worker_is_adopted_without_relaunch( + test_case: WorkerRestartTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + active: dict[str, Any] = running_statement(worker) + assert worker.docker is not None + before: dict[str, object] = worker_container( + worker.docker, worker.api.location_id, active["run_id"] + ) + worker.process.send_signal(test_case.shutdown_signal) + assert worker.process.wait(timeout=10) == test_case.expected_scheduler_exit, ( + worker.log.read_text() + ) + assert worker_container(worker.docker, worker.api.location_id, active["run_id"])["Running"] + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (1,) + log: Path = tmp_path / "replacement-scheduler.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "scheduler", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "--launchers", + str(tmp_path / "launchers.json"), + ], + log, + tmp_path / "replacement-scheduler.stdout", + ) as process: + replacement: ScheduledWorker = worker._replace(process=process, log=log) + wait_for_log(process, log, "Worker supervision recovered for run", timeout=45) + assert ( + worker_container(worker.docker, worker.api.location_id, active["run_id"])["Pid"] + == before["Pid"] + ) + cancel_running_statement(replacement) + run: dict[str, Any] = worker_result(replacement, test_case) + assert run["status"] == test_case.expected_status, log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_claimed'" + ).fetchone() == (1,) + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "worker completes before the lost launch response can be durably observed", + "SELECT 37 AS amount", + "SELECT amount FROM orders", + "success", + [(37,)], + launcher="docker", + store_setup=( + "CREATE TRIGGER reject_launch_observation BEFORE INSERT ON recall_events " + "WHEN NEW.event_type='platform_launch_failed' " + "BEGIN SELECT RAISE(ABORT,'synthetic observation outage'); END;" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_a_lost_docker_start_response_when_the_worker_finishes_then_late_observation_retries_do_not_revive_it( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + inspection: Event = Event() + + @contextmanager + def deployment(directory: Path, executable: str) -> Iterator[WorkerDeployment]: + with docker_deployment(directory, executable) as actual: + assert actual.docker is not None + with lost_start_response(directory, actual.docker.socket, inspection) as proxy: + yield actual._replace(configuration=actual.configuration | {"socket": str(proxy)}) + + monkeypatch.setattr(helpers, "docker_deployment", deployment) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + wait_for_log(worker.process, worker.log, "Retrying worker shutdown publication") + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + before: list[tuple[Any, ...]] = control.execute("SELECT * FROM sqb_runs").fetchall() + assert control.execute( + "SELECT COUNT(*) FROM recall_events WHERE event_type IN ('run_launch_failed','platform_launch_failed')" + ).fetchone() == (0,) + _ = control.execute("DROP TRIGGER reject_launch_observation") + deadline: float = time.monotonic() + 10 + while control.execute( + "SELECT COUNT(*) FROM recall_events WHERE event_type='platform_launch_failed'" + ).fetchone() != (1,): + assert worker.process.poll() is None, worker.log.read_text() + assert time.monotonic() < deadline, worker.log.read_text() + time.sleep(0.02) + # Terminal state alone cannot bypass the still-unavailable physical observation. + time.sleep(1.2) + assert worker.process.poll() is None, worker.log.read_text() + assert control.execute("SELECT * FROM sqb_runs").fetchall() == before + assert control.execute( + "SELECT event_type FROM recall_events WHERE event_type IN " + "('run_terminated','run_launch_failed','platform_launch_failed') ORDER BY storage_id" + ).fetchall() == [ + ("run_terminated",), + ("run_launch_failed",), + ("platform_launch_failed",), + ] + assert control.execute("SELECT COUNT(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute( + "SELECT COUNT(*) FROM sqb_runs WHERE worker_instance IS NOT NULL" + ).fetchone() == (0,) + inspection.set() + assert worker.process.wait(timeout=5) != 0, "Original response failure remains visible" + assert worker.docker is not None + state: dict[str, object] = worker_container( + worker.docker, worker.api.location_id, run["run_id"] + ) + assert state["Running"] is False + assert state["ExitCode"] == 0 diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_installed_recovery.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_installed_recovery.py new file mode 100644 index 0000000000..75a91f61d9 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_installed_recovery.py @@ -0,0 +1,160 @@ +"""An installed native executable restores lost administrator access with grant-admin.""" + +import json +import shutil +import subprocess +from pathlib import Path +from typing import Any + +import psycopg +import pytest +from testcontainers.postgres import PostgresContainer + +from scripts.native_tests.constants import POSTGRES_IMAGE +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + InstalledRecoveryTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.api.helpers import ( + login, + request, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.cli.helpers import ( + administer, + serve, + successful, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.helpers import ( + postgres_recovery_state, + verify_recovery, +) + + +@pytest.mark.parametrize( + "test_case", + [ + InstalledRecoveryTestCase( + description="installed native server recovers authority without rebootstrap", + executable="sqb", + expected_rejection="cannot remove or disable the final installation administrator", + ), + ], + ids=lambda case: case.description, +) +def test_given_an_installed_server_when_admin_is_granted_then_authority_is_restored_without_rebootstrap( + test_case: InstalledRecoveryTestCase, +) -> None: + binary: str | None = shutil.which(test_case.executable) + assert binary is not None + assert test_case.expected_rejection in verify_recovery(Path(binary)) + + +@pytest.mark.parametrize( + "test_case", + [ + InstalledRecoveryTestCase( + description="installed PostgreSQL grant-admin restores authority while replicas run", + executable="sqb", + expected_rejection="cannot remove or disable the final installation administrator", + ), + ], + ids=lambda case: case.description, +) +def test_given_postgres_replicas_when_grant_admin_runs_then_only_the_granted_persons_sessions_end( + test_case: InstalledRecoveryTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + binary: str | None = shutil.which(test_case.executable) + assert binary is not None + installation: tuple[Path, Path] = Path(binary), tmp_path + replica: Path = tmp_path / "replica" + replica.mkdir() + fixture: dict[str, Any] = json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json" + ).read_text() + ) + state_sql: str = ( + "SELECT (SELECT COUNT(*) FROM sqb_principals WHERE disabled_at IS NOT NULL)," + "(SELECT COUNT(*) FROM sqb_role_assignments)," + "(SELECT COUNT(*) FROM sqb_sessions WHERE revoked_at IS NULL)," + "(SELECT COUNT(*) FROM sqb_security_audit WHERE action='admin.granted')" + ) + with PostgresContainer(POSTGRES_IMAGE, driver=None) as postgres: + url: str = postgres.get_connection_url() + extra: dict[str, str] = {"SQLBUILD_CONTROL_DATABASE_URL": url} + people: list[dict[str, Any]] = [] + for command, name in [ + ("create-user", fixture["first_person"]), + ("create-principal", fixture["second_person"]), + ]: + person: dict[str, Any] = successful( + administer(installation, [command, name], extra_environment=extra) + ) + successful( + administer( + installation, + ["set-password", person["principal_id"]], + fixture["password"], + extra_environment=extra, + ) + ) + people.append(person) + principal: str = people[0]["principal_id"] + recovery: list[str] = ["grant-admin", principal] + with ( + serve(installation, extra_environment=extra) as first, + serve((Path(binary), replica), extra_environment=extra) as second, + ): + previous: tuple[str, str] = login(first, people[0]["name"], fixture["password"]) + other: tuple[str, str] = login(second, people[1]["name"], fixture["password"]) + for endpoint in [first, second]: + assert request(endpoint + "/api/auth/session", session=previous)[0] == 200 + assert request(endpoint + "/api/auth/session", session=other)[0] == 200 + with psycopg.connect(url) as control: + control.execute( + "UPDATE sqb_principals SET disabled_at=now() WHERE principal_id=%s", + (principal,), + ) + control.execute( + "DELETE FROM sqb_role_assignments WHERE principal_id=%s", + (principal,), + ) + assert control.execute(state_sql).fetchone() == (1, 0, 2, 0) + retained_before: dict[str, str] = postgres_recovery_state(control) + assert successful(administer(installation, recovery, extra_environment=extra)) == { + "principal_id": principal, + "role": "admin", + } + with psycopg.connect(url) as control: + after: Any = control.execute(state_sql).fetchone() + assert after is not None + assert after == (0, 1, 1, 1) + retained_after: dict[str, str] = postgres_recovery_state(control) + assert retained_after["sqb_installation"] == retained_before["sqb_installation"] + audit: Any = control.execute( + "SELECT scope_json FROM sqb_security_audit WHERE action='admin.granted'" + ).fetchone() + assert audit == ({"principal_id": principal},) + current: tuple[str, str] = login(first, people[0]["name"], fixture["password"]) + for endpoint in [first, second]: + assert request(endpoint + "/api/auth/session", session=previous)[0] == 401 + assert request(endpoint + "/api/auth/session", session=other)[0] == 200 + assert ( + request( + endpoint + "/api/admin/people/" + principal, + b'"revoke"', + (current[0], "wrong-csrf"), + )[0] + == 401 + ) + assert request(endpoint + "/api/auth/session", session=current)[0] == 200 + status, _, body = request( + endpoint + "/api/admin/people/" + principal, b'"revoke"', current + ) + assert status == 403 + assert test_case.expected_rejection in body["error"] + rejected: subprocess.CompletedProcess[str] = administer( + installation, ["create-user", "replacement-admin"], extra_environment=extra + ) + assert rejected.returncode != 0 and not rejected.stdout + assert "bootstrap is closed" in rejected.stderr diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py new file mode 100644 index 0000000000..c4c773829f --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py @@ -0,0 +1,254 @@ +"""A replacement Linux dev scheduler retains physical ownership of surviving local workers.""" + +import os +import signal +import sqlite3 +import sys +import time +from contextlib import suppress +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ScheduledWorker, + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + cancel_running_statement, + location_process, + running_statement, + scheduled_worker, + wait_for_interpreter, + wait_for_log, + wait_for_run, + worker_result, +) + +fcntl: ModuleType = pytest.importorskip("fcntl") + + +@pytest.mark.skipif(sys.platform != "linux", reason="Linux inherited-lock and pidfd recovery") +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "scheduler SIGKILL does not orphan a running local warehouse statement", + Path( + "benchmarks/duckshop/edge_cases/orchestration/local_worker_recovery.sql" + ).read_text(), + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "cancelled", + [(0,)], + ) + ], + ids=lambda case: case.description, +) +def test_given_a_running_local_worker_when_scheduler_dies_then_restart_adopts_and_cancels_without_relaunch( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + active: dict[str, Any] = running_statement(worker) + with sqlite3.connect(worker.control_store) as control: + claim_id: str = control.execute( + "SELECT worker_instance FROM sqb_runs WHERE run_id=?", (active["run_id"],) + ).fetchone()[0] + lock: Path = worker.project / ".sqlbuild/workers" / f"{claim_id}.lock" + with lock.open("rb") as ownership: + with pytest.raises(BlockingIOError): + fcntl.flock(ownership, fcntl.LOCK_EX | fcntl.LOCK_NB) + worker.process.kill() + assert worker.process.wait(timeout=10) == -signal.SIGKILL + with pytest.raises(BlockingIOError): + fcntl.flock(ownership, fcntl.LOCK_EX | fcntl.LOCK_NB) + log: Path = tmp_path / "replacement-scheduler.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "scheduler", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "--launchers", + str(tmp_path / "launchers.json"), + ], + log, + tmp_path / "replacement-scheduler.stdout", + ) as process: + replacement: ScheduledWorker = worker._replace(process=process, log=log) + wait_for_log(process, log, "Worker supervision recovered for run", timeout=45) + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == ( + 1, + ) + cancel_running_statement(replacement) + run: dict[str, Any] = worker_result(replacement, test_case) + assert run["status"] == test_case.expected_status, log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_claimed'" + ).fetchone() == (1,) + deadline: float = time.monotonic() + 10 + while True: + try: + fcntl.flock(ownership, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.02) + + +@pytest.mark.skipif(sys.platform != "linux", reason="Linux inherited-lock and pidfd recovery") +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "successive supervisor crashes retain a stopped child's claim until publication commits", + "SELECT 23 AS amount", + "", + "failed", + [], + launcher="uncooperative", + concurrency_limits={"orders_limit": 1}, + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py" + ).read_text(), + }, + store_setup=Path( + "benchmarks/duckshop/edge_cases/orchestration/local_recovery_publication_fault.sql" + ).read_text(), + ) + ], + ids=lambda case: case.description, +) +def test_given_an_unstarted_orphan_when_recovery_publication_fails_then_claims_wait_for_physical_exit_and_commit( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + pid: int = wait_for_interpreter(worker.process, worker.project / "worker.pid", worker.log) + active: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, pending=frozenset({"queued"}) + ) + assert active["status"] == "launched" + claims: tuple[str, ...] = ( + "SELECT * FROM sqb_write_claims", + "SELECT * FROM sqb_concurrency_slots", + "SELECT run_id, worker_instance, container_id, claimed_at, last_heartbeat_at " + "FROM sqb_runs WHERE worker_instance IS NOT NULL", + ) + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT group_key, run_id FROM sqb_concurrency_slots" + ).fetchall() == [("orders_limit", active["run_id"])] + retained: dict[str, list[tuple[Any, ...]]] = { + claim: control.execute(claim).fetchall() for claim in claims + } + request: tuple[Any, ...] = control.execute( + "SELECT request_json FROM sqb_runs" + ).fetchone() + claim_id: str = control.execute( + "SELECT worker_instance FROM sqb_runs WHERE worker_instance IS NOT NULL" + ).fetchone()[0] + arguments: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "scheduler", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "--launchers", + str(tmp_path / "launchers.json"), + ] + handle: int = os.pidfd_open(pid) + try: + worker.process.kill() + assert worker.process.wait(timeout=10) == -signal.SIGKILL + log: Path = tmp_path / "replacement-scheduler.log" + with location_process( + arguments, + log, + tmp_path / "replacement-scheduler.stdout", + ) as process: + wait_for_log(process, log, "Worker supervision recovered for run", timeout=45) + signal.pidfd_send_signal(handle, 0) + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == ( + 1, + ) + assert control.execute("SELECT status FROM sqb_runs").fetchone() == ( + "launched", + ) + signal.pidfd_send_signal(handle, signal.SIGKILL) + wait_for_log(process, log, "Retrying worker shutdown publication") + assert process.poll() is None, log.read_text() + assert "Worker launched for run" not in log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == ( + 1, + ) + assert { + claim: control.execute(claim).fetchall() for claim in claims + } == retained + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_terminated'" + ).fetchone() == (0,) + process.kill() + assert process.wait(timeout=10) == -signal.SIGKILL + log = tmp_path / "second-replacement-scheduler.log" + with location_process( + arguments, log, tmp_path / "second-replacement-scheduler.stdout" + ) as process: + wait_for_log(process, log, "Worker supervision recovered for run", timeout=45) + wait_for_log(process, log, "Retrying worker shutdown publication") + assert process.poll() is None, log.read_text() + assert "Worker launched for run" not in log.read_text() + assert int((worker.project / "worker.pid").read_text()) == pid + with sqlite3.connect(worker.control_store) as control: + assert { + claim: control.execute(claim).fetchall() for claim in claims + } == retained + assert ( + control.execute("SELECT request_json FROM sqb_runs").fetchone() == request + ) + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_terminated'" + ).fetchone() == (0,) + lock: Path = worker.project / ".sqlbuild/workers" / f"{claim_id}.lock" + with lock.open("rb") as ownership: + with pytest.raises(BlockingIOError): + fcntl.flock(ownership, fcntl.LOCK_EX | fcntl.LOCK_NB) + with sqlite3.connect(worker.control_store) as control: + _ = control.execute("DROP TRIGGER reject_terminal") + assert process.wait(timeout=10) != 0, log.read_text() + fcntl.flock(ownership, fcntl.LOCK_EX | fcntl.LOCK_NB) + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT status FROM sqb_runs").fetchone() == ( + test_case.expected_status, + ) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == ( + 0, + ) + assert control.execute( + "SELECT count(*) FROM sqb_concurrency_slots" + ).fetchone() == (0,) + assert control.execute( + "SELECT count(*) FROM sqb_runs WHERE worker_instance IS NOT NULL" + ).fetchone() == (0,) + assert ( + control.execute("SELECT request_json FROM sqb_runs").fetchone() == request + ) + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_claimed'" + ).fetchone() == (1,) + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_terminated'" + ).fetchone() == (1,) + assert control.execute( + "SELECT json_extract(payload, '$.change.change.exit_code') FROM recall_events " + "WHERE event_type='run_terminated'" + ).fetchone() == (None,) + assert not (worker.project / "warehouse.duckdb").exists() + finally: + with suppress(ProcessLookupError): + signal.pidfd_send_signal(handle, signal.SIGKILL) + os.close(handle) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py new file mode 100644 index 0000000000..780abc474f --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py @@ -0,0 +1,419 @@ +"""A real owner, browser, scheduler, location and DuckDB share one permission-enforced journey.""" + +import json +import sys +from contextlib import nullcontext +from functools import partial +from pathlib import Path +from typing import Any +from urllib.error import HTTPError +from uuid import uuid4 + +import duckdb +import pytest + +from sqlbuild._native import compute_query_hash +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + PersonJourneyTestCase, + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + current_snapshot, + location_process, + scheduled_worker, + wait_for_generation, + wait_for_location_health, + wait_for_run, + worker_result, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.person.helpers import ( + person_login, + person_read, + person_request, + provision_person, + verify_compute_transport, + verify_security_audit, + verify_sql_inspection, + wait_for_person_catalog, +) + + +@pytest.mark.parametrize( + "test_case", + [ + PersonJourneyTestCase( + description="no-role default, scoped browser operations, reload and last-good recovery", + model_sql="SELECT 23 AS amount", + warehouse_query="SELECT amount FROM orders", + expected_status="success", + expected_rows=[(23,)], + default_role="none", + expected_metadata_access=partial(pytest.raises, HTTPError, match="HTTP Error 403"), + ), + PersonJourneyTestCase( + description="configured viewer default, scoped execution, reload and last-good recovery", + model_sql="SELECT 23 AS amount", + warehouse_query="SELECT amount FROM orders", + expected_status="success", + expected_rows=[(23,)], + default_role="viewer", + expected_metadata_access=nullcontext, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_initialized_installation_when_people_run_and_reload_then_permissions_and_recovery_hold( + test_case: PersonJourneyTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + with scheduled_worker( + tmp_path, + monkeypatch, + test_case, + server_arguments=( + "--password-origin", + "https://control.example.com", + "--default-role", + test_case.default_role, + ), + ) as worker: + scheduled: dict[str, Any] = worker_result(worker, test_case) + admin: dict[str, Any] = provision_person( + tmp_path / "control", "create-user", "orders-admin" + ) + person: dict[str, Any] = provision_person( + tmp_path / "control", "create-principal", "orders-person" + ) + admin_session: tuple[str, str] = person_login(worker.api, admin["name"]) + session: tuple[str, str] = person_login(worker.api, person["name"]) + verify_security_audit(worker.api, admin_session, session) + runs_path: str = f"/api/control/locations/{worker.api.location_id}/targets/development/runs" + request: dict[str, Any] = { + "run_id": str(uuid4()), + "location_id": worker.api.location_id, + "target": "development", + "job_key": "annual_orders", + "reason": "Refresh order totals", + "priority": 0, + } + assignment: dict[str, Any] = { + "principal_id": person["principal_id"], + "role": "viewer", + "location_id": worker.api.location_id, + "target": "development", + } + rerun: dict[str, Any] = { + "run_id": str(uuid4()), + "source_run_id": scheduled["run_id"], + "from_failure": False, + "reason": "Repeat the pinned order build", + "priority": 0, + } + cancellation: dict[str, Any] = { + "run_id": scheduled["run_id"], + "event_id": str(uuid4()), + "reason": "Stop the order build", + } + with test_case.expected_metadata_access(): + assert ( + person_read(worker.api, runs_path, session=session)[0]["run_id"] + == scheduled["run_id"] + ) + for path, body in [ + ("/api/control/runs", request), + ("/api/admin/roles", assignment), + ("/api/control/runs/cancel", cancellation), + ("/api/control/runs/rerun", rerun), + ]: + with pytest.raises(HTTPError) as rejected: + person_request(worker.api, path, body=body, session=session) + assert rejected.value.code == 403 + assert "decision" in json.load(rejected.value)["error"] + assert person_request( + worker.api, "/api/admin/roles", body=assignment, session=admin_session + ) == {"applied": True} + verify_compute_transport(worker.api, session, scheduled["run_id"]) + for path, body in [ + ("/api/control/runs", request), + ("/api/control/runs/cancel", cancellation), + ("/api/control/runs/rerun", rerun), + ("/api/admin/people/" + admin["principal_id"], "disable"), + ]: + with pytest.raises(HTTPError) as rejected: + person_request(worker.api, path, body=body, session=session) + assert rejected.value.code == 403 + with test_case.expected_metadata_access(): + assert ( + person_read( + worker.api, runs_path.replace("development", "preview"), session=session + ) + == [] + ) + assignment["role"] = "operator" + assert person_request( + worker.api, "/api/admin/roles", body=assignment, session=admin_session + ) == {"applied": True} + with pytest.raises(HTTPError) as csrf_rejected: + person_request( + worker.api, "/api/control/runs", body=request, session=(session[0], "wrong-csrf") + ) + assert csrf_rejected.value.code == 401 + for sensitive in [request | {"window": {"kind": "integer", "start": 3, "end": 8}}]: + with pytest.raises(HTTPError) as rewrite_rejected: + person_request(worker.api, "/api/control/runs", body=sensitive, session=session) + assert rewrite_rejected.value.code == 403 + assert person_request(worker.api, "/api/control/runs", body=request, session=session) == { + "applied": True + } + assert ( + wait_for_run(worker.api, worker.process, worker.log, run_id=request["run_id"])["status"] + == test_case.expected_status + ) + assert person_request(worker.api, "/api/control/runs", body=request, session=session) == { + "applied": False + } + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", worker.api.bearer) + log: Path = tmp_path / "location.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(worker.project), + "location", + "serve", + "--server", + worker.api.url, + "--location-id", + worker.api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + ) as location: + _ = wait_for_generation(worker.api, "development", 2, location, log) + model: Path = worker.project / "models/orders.sql" + original: str = model.read_text() + _ = model.write_text(original.replace("23", "37")) + edited: dict[str, Any] = wait_for_generation( + worker.api, "development", 3, location, log + ) + request["run_id"] = str(uuid4()) + assert person_request( + worker.api, "/api/control/runs", body=request, session=session + ) == {"applied": True} + reloaded: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=request["run_id"] + ) + assert reloaded["status"] == test_case.expected_status + assert reloaded["snapshot_id"] == edited["record"]["snapshot_id"] + with duckdb.connect( + str(worker.project / "warehouse.duckdb"), read_only=True + ) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == [(37,)] + _ = model.write_text("SELECT FROM invalid SQL !\n") + _ = wait_for_location_health( + worker.api, + "development", + {"kind": "compile_failed", "generation": 4}, + location, + log, + ) + assert current_snapshot(worker.api, "development")["record"] == edited["record"] + _ = model.write_text(original.replace("23", "41")) + recovered: dict[str, Any] = wait_for_generation( + worker.api, "development", 4, location, log + ) + request["run_id"] = str(uuid4()) + assert person_request( + worker.api, "/api/control/runs", body=request, session=session + ) == {"applied": True} + recovery: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=request["run_id"] + ) + assert (recovery["status"], recovery["snapshot_id"]) == ( + test_case.expected_status, + recovered["record"]["snapshot_id"], + ) + with duckdb.connect( + str(worker.project / "warehouse.duckdb"), read_only=True + ) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == [(41,)] + wait_for_person_catalog( + worker.api, + session, + location, + log, + expected_definition_hash=compute_query_hash( + query_sql=test_case.model_sql.replace("23", "41"), dialect="duckdb" + ), + ) + assignment["revoke"] = True + assert person_request( + worker.api, "/api/admin/roles", body=assignment, session=admin_session + ) == {"applied": True} + for path, body in [ + ("/api/control/runs", request | {"run_id": str(uuid4())}), + ("/api/control/runs/cancel", cancellation), + ("/api/control/runs/rerun", rerun), + ]: + with pytest.raises(HTTPError) as revoked: + person_request(worker.api, path, body=body, session=session) + assert revoked.value.code == 403 + verify_sql_inspection( + worker.api, + (admin_session, session), + assignment, + model, + recovered["record"]["snapshot_id"], + ) + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + description="scoped operator frozen rerun, cancellation and revoked-state privacy", + model_sql="SELECT 23 AS amount", + warehouse_query="SELECT amount FROM orders", + expected_status="success", + expected_rows=[(23,)], + ) + ], + ids=lambda case: case.description, +) +def test_given_person_run_controls_when_scopes_change_then_frozen_execution_and_state_privacy_hold( + test_case: WorkerExecutionTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + with scheduled_worker( + tmp_path, + monkeypatch, + test_case, + server_arguments=("--password-origin", "https://control.example.com"), + ) as worker: + scheduled: dict[str, Any] = worker_result(worker, test_case) + admin: dict[str, Any] = provision_person( + tmp_path / "control", "create-user", "orders-admin" + ) + person: dict[str, Any] = provision_person( + tmp_path / "control", "create-principal", "orders-person" + ) + admin_session: tuple[str, str] = person_login(worker.api, admin["name"]) + session: tuple[str, str] = person_login(worker.api, person["name"]) + assignment: dict[str, Any] = { + "principal_id": person["principal_id"], + "role": "operator", + "location_id": worker.api.location_id, + "target": "preview", + } + assert person_request( + worker.api, "/api/admin/roles", body=assignment, session=admin_session + ) == {"applied": True} + rerun: dict[str, Any] = { + "run_id": str(uuid4()), + "source_run_id": scheduled["run_id"], + "from_failure": False, + "reason": "Repeat pinned order build", + "priority": 0, + } + for from_failure in [False, True]: + with pytest.raises(HTTPError) as out_of_scope: + person_request( + worker.api, + "/api/control/runs/rerun", + body=rerun | {"from_failure": from_failure}, + session=session, + ) + assert out_of_scope.value.code == 403 + assignment["target"] = "development" + assert person_request( + worker.api, "/api/admin/roles", body=assignment, session=admin_session + ) == {"applied": True} + for applied in [True, False]: + assert person_request( + worker.api, "/api/control/runs/rerun", body=rerun, session=session + ) == {"applied": applied} + repeated: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=rerun["run_id"] + ) + assert repeated["status"] == test_case.expected_status + assert repeated["snapshot_id"] == scheduled["snapshot_id"] + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + worker.process.terminate() + assert worker.process.wait(timeout=5) == 0 + request: dict[str, Any] = { + "run_id": str(uuid4()), + "location_id": worker.api.location_id, + "target": "development", + "job_key": "annual_orders", + "reason": "Build order totals", + "priority": 0, + } + assert person_request(worker.api, "/api/control/runs", body=request, session=session) == { + "applied": True + } + cancellation: dict[str, Any] = { + "run_id": request["run_id"], + "event_id": str(uuid4()), + "reason": "Stop before launch", + } + with pytest.raises(HTTPError) as forged: + person_request( + worker.api, + "/api/control/runs/cancel", + body=cancellation | {"actor": "local-control-store-owner"}, + session=session, + ) + assert forged.value.code == 422 + with pytest.raises(HTTPError) as csrf: + person_request( + worker.api, + "/api/control/runs/cancel", + body=cancellation, + session=(session[0], "wrong-csrf"), + ) + assert csrf.value.code == 401 + for applied in [True, False]: + assert person_request( + worker.api, "/api/control/runs/cancel", body=cancellation, session=session + ) == {"applied": applied} + runs_path: str = f"/api/control/locations/{worker.api.location_id}/targets/development/runs" + assert person_read(worker.api, runs_path, session=session)[0]["status"] == "cancelled" + assignment["revoke"] = True + assert person_request( + worker.api, "/api/admin/roles", body=assignment, session=admin_session + ) == {"applied": True} + queued: dict[str, Any] = request | {"run_id": str(uuid4())} + assert person_request( + worker.api, "/api/control/runs", body=queued, session=admin_session + ) == {"applied": True} + with pytest.raises(HTTPError) as metadata: + person_read(worker.api, runs_path, session=session) + assert metadata.value.code == 403 + for source_id in [scheduled["run_id"], request["run_id"], queued["run_id"], str(uuid4())]: + for from_failure in [False, True]: + with pytest.raises(HTTPError) as private: + person_request( + worker.api, + "/api/control/runs/rerun", + body=rerun + | { + "run_id": str(uuid4()), + "source_run_id": source_id, + "from_failure": from_failure, + }, + session=session, + ) + assert private.value.code == 403 + with pytest.raises(HTTPError) as cancel_denied: + person_request( + worker.api, + "/api/control/runs/cancel", + body=cancellation | {"run_id": source_id}, + session=session, + ) + assert cancel_denied.value.code == 403 diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py new file mode 100644 index 0000000000..d7e12791d8 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_sql.py @@ -0,0 +1,201 @@ +"""An installed CLI inspects actual published SQL through a bounded live person session.""" + +import json +import subprocess +import sys +from functools import partial +from pathlib import Path +from typing import Any + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + current_snapshot, + scheduled_worker, + worker_result, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.login.helpers import ( + browser, + gateway, + pending_login, + request, + sign_in, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.person.helpers import ( + provision_person, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.cli.helpers import ( + administer, + environment, +) + + +@pytest.mark.skipif(sys.platform != "linux", reason="Headless Linux credential-store acceptance") +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + description="saved person session, actual SQL, exact key and live revocation", + model_sql="SELECT 23 AS amount, '" + "orders " * 400 + "' AS description", + warehouse_query="SELECT amount FROM orders", + expected_status="success", + expected_rows=[(23,)], + ) + ], + ids=lambda case: case.description, +) +def test_given_published_orders_when_cli_reads_sql_then_only_live_scoped_authority_discloses_sources( + test_case: WorkerExecutionTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + home: Path = tmp_path / "home" + home.mkdir() + empty: Path = tmp_path / "empty" + empty.mkdir() + client_environment: dict[str, str] = environment({"HOME": str(home)}) + client_environment.pop("DBUS_SESSION_BUS_ADDRESS", None) + client_environment.pop("XDG_RUNTIME_DIR", None) + binary: str = str(Path(sys.executable).with_name("sqb")) + invoke: partial[subprocess.CompletedProcess[Any]] = partial( + subprocess.run, + cwd=empty, + env=client_environment, + capture_output=True, + text=True, + check=False, + timeout=20, + ) + with ( + gateway(tmp_path) as (origin, certificate, transport), + scheduled_worker( + tmp_path, + monkeypatch, + test_case, + server_arguments=("--password-origin", origin), + ) as worker, + ): + transport["backend"] = worker.api.url.removeprefix("http://") + _ = worker_result(worker, test_case) + admin: dict[str, Any] = provision_person( + tmp_path / "control", "create-user", "orders-admin" + ) + person: dict[str, Any] = provision_person( + tmp_path / "control", "create-principal", "orders-person" + ) + client, _ = browser(origin, certificate) + csrf: str = sign_in( + client, origin, "password", person["name"], "orders-person-password-2026" + ) + common: list[str] = ["--server", origin, "--ca-cert", str(certificate)] + command: list[str] = [ + binary, + "control", + "sql", + "model:orders", + *common, + "--location-id", + worker.api.location_id, + "--target", + "development", + ] + absent: subprocess.CompletedProcess[str] = invoke(command) + assert absent.returncode != 0 and not absent.stdout and "not signed in" in absent.stderr + with pending_login( + [binary, "login", *common, "--no-browser"], + client_environment, + empty, + tmp_path / "cli-login.log", + ) as (process, url): + assert ( + request( + client, + origin + "/api/auth/cli/approve", + {"browser_code": url.split("#", 1)[1], "approve": True}, + csrf=csrf, + )[0] + == 204 + ) + stdout, _ = process.communicate(timeout=20) + assert process.returncode == 0 and not stdout + denied: subprocess.CompletedProcess[str] = invoke(command) + assert ( + denied.returncode != 0 and not denied.stdout and "HTTP 403; decision " in denied.stderr + ) + administrator, _ = browser(origin, certificate) + admin_csrf: str = sign_in( + administrator, origin, "password", admin["name"], "orders-person-password-2026" + ) + assignment: dict[str, Any] = { + "principal_id": person["principal_id"], + "role": "developer", + "location_id": worker.api.location_id, + "target": "development", + } + assert ( + request(administrator, origin + "/api/admin/roles", assignment, csrf=admin_csrf)[0] + == 200 + ) + inspected: subprocess.CompletedProcess[str] = invoke(command) + assert inspected.returncode == 0 and "Published SQL read successfully" in inspected.stderr + sql: dict[str, Any] = json.loads(inspected.stdout) + assert set(sql) == {"snapshot_id", "node_key", "raw_sql", "compiled_sql"} + assert ( + sql["snapshot_id"] + == current_snapshot(worker.api, "development")["record"]["snapshot_id"] + ) + assert sql["node_key"] == "model:orders" + assert sql["raw_sql"] == (worker.project / "models/orders.sql").read_text() + with duckdb.connect() as verification: + assert verification.sql(sql["compiled_sql"]).fetchall() == [ + (*row, "orders " * 400) for row in test_case.expected_rows + ] + assert 4097 <= len(inspected.stdout.encode()) < 8192 + with Path("/dev/full").open("w") as unavailable: + undelivered: subprocess.CompletedProcess[str] = invoke( + command, + capture_output=False, + stdout=unavailable, + stderr=subprocess.PIPE, + ) + assert undelivered.returncode != 0 and "SQL inspection failed" in undelivered.stderr + assert "Published SQL read successfully" not in undelivered.stderr + for node in ["model:orders+", "model:orders?query=1/#café"]: + missing: subprocess.CompletedProcess[str] = invoke(command[:3] + [node] + command[4:]) + assert missing.returncode == 0 and json.loads(missing.stdout) is None + assert ( + request( + administrator, + origin + "/api/admin/roles", + assignment | {"revoke": True}, + csrf=admin_csrf, + )[0] + == 200 + ) + revoked: subprocess.CompletedProcess[str] = invoke(command) + assert ( + revoked.returncode != 0 + and not revoked.stdout + and "HTTP 403; decision " in revoked.stderr + ) + changed: subprocess.CompletedProcess[str] = administer( + (Path(binary), tmp_path / "control"), + ["set-password", person["principal_id"]], + "orders-replacement-password-2026", + ) + assert changed.returncode == 0 + expired: subprocess.CompletedProcess[str] = invoke(command) + assert expired.returncode != 0 and not expired.stdout and "revoked" in expired.stderr + logout: subprocess.CompletedProcess[str] = invoke([binary, "logout", *common]) + assert logout.returncode == 0 and not logout.stdout + signed_out: subprocess.CompletedProcess[str] = invoke(command) + assert ( + signed_out.returncode != 0 + and not signed_out.stdout + and "not signed in" in signed_out.stderr + ) + assert not list(empty.iterdir()) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py new file mode 100644 index 0000000000..a0a60c70c7 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -0,0 +1,299 @@ +"""Real compile, native transport, authenticated server and durable publication.""" + +import json +import socket +import sqlite3 +import subprocess +from pathlib import Path +from typing import Any +from urllib.parse import quote +from urllib.request import Request, urlopen + +import pytest +from _pytest.capture import CaptureResult + +from sqlbuild._native import ( + NativeOrchestrationSnapshot, + build_compiled_project, + build_orchestration_snapshot, + discover_project_inputs, + run_cli, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + METADATA, + TYPE_PROOF_FILES, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + SnapshotPublicationTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + RejectedPublicationOutput, + current_snapshot, + snapshot_api, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotPublicationTestCase( + description="ordinary target", + target="development", + expected_generation=2, + expected_nodes=frozenset({"model:orders", "model:stg_orders"}), + ), + SnapshotPublicationTestCase( + description="target requiring URL path encoding", + target="preview #1", + expected_generation=2, + expected_nodes=frozenset({"model:orders", "model:stg_orders"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_retained_compiler_data_when_publishing_then_http_updates_atomically_and_rejections_keep_last_good( + test_case: SnapshotPublicationTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "project" + files: dict[str, str] = TYPE_PROOF_FILES | { + "orchestration/edge_cases/definitions.py": Path( + "benchmarks/duckshop/project/orchestration/edge_cases/definitions.py" + ).read_text(), + "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( + "MODEL (", "MODEL (tags ['edge_case'],", 1 + ), + } + write_files(project_dir=project_dir, files=files) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + assert not any(item.severity == "error" for item in project.diagnostics) + with snapshot_api(tmp_path / "control") as api: + metadata: dict[str, Any] = METADATA | { + "location_id": api.location_id, + "target": test_case.target, + } + first: NativeOrchestrationSnapshot = build_orchestration_snapshot( + project, json.dumps(metadata), project_dir + ) + record: dict[str, Any] = json.loads(first.publish(api.url, api.bearer)) + assert record["generation"] == 1 + assert record["location_id"] == api.location_id + initial: dict[str, Any] = current_snapshot(api, test_case.target) + assert frozenset(initial["nodes"]) == test_case.expected_nodes + assert initial["nodes"]["model:orders"]["definition"]["deps"] == ["model:stg_orders"] + edited: dict[str, str] = files | { + "models/stg_orders.sql": TYPE_PROOF_FILES["models/stg_orders.sql"].replace( + "CAST(5 AS INTEGER)", "CAST(9 AS INTEGER)" + ) + } + write_files(project_dir=project_dir, files=edited) + changed: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + next_snapshot: NativeOrchestrationSnapshot = build_orchestration_snapshot( + changed, json.dumps(metadata | {"generation": 2}), project_dir + ) + updated: dict[str, Any] = json.loads(next_snapshot.publish(api.url, api.bearer)) + assert updated["generation"] == test_case.expected_generation + assert updated["snapshot_id"] != record["snapshot_id"] + with pytest.raises(RuntimeError, match="HTTP 409"): + _ = first.publish(api.url, api.bearer) + with pytest.raises(RuntimeError, match="HTTP 401"): + _ = next_snapshot.publish(api.url, "invalid-credential") + current: dict[str, Any] = current_snapshot(api, test_case.target) + assert current["record"] == updated + assert "CAST(9 AS INTEGER)" in current["nodes"]["model:stg_orders"]["raw_sql"] + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/targets/" + f"{quote(test_case.target, safe='')}/triggers", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + triggers: list[dict[str, Any]] = json.load(response) + by_name: dict[str, dict[str, Any]] = {trigger["name"]: trigger for trigger in triggers} + assert len(triggers) == 4 + assert set(by_name) == { + "edge_snapshot_orders", + "edge_platform_orders", + "edge_job_outcomes", + "edge_group_outcomes", + } + scheduled: dict[str, Any] = by_name["edge_snapshot_orders"] + assert scheduled["status"] == "paused" + assert scheduled["definition_json"]["starts_at"] == "2026-07-02T19:47:23.456789Z" + assert scheduled["definition_json"]["default_status"] == "paused" + assert by_name["edge_platform_orders"]["status"] == "paused" + assert by_name["edge_platform_orders"]["definition_json"]["default_status"] == "paused" + assert by_name["edge_platform_orders"]["definition_json"]["when"] == { + "kind": "platform_event", + "kinds": ["tick_overdue", "location_broken", "launch_failed"], + } + assert by_name["edge_job_outcomes"]["definition_json"]["when"] == { + "kind": "run_event", + "job_key": "edge_snapshot_orders", + "kinds": ["warn", "partial"], + } + assert by_name["edge_group_outcomes"]["definition_json"]["when"] == { + "kind": "run_event", + "job_key": "edge_snapshot_group", + "kinds": ["completed"], + "executed_successfully": ["model:orders"], + } + assert by_name["edge_job_outcomes"]["status"] == "paused" + assert by_name["edge_group_outcomes"]["status"] == "paused" + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotPublicationTestCase( + description="configured development target", + target="development", + expected_generation=2, + expected_nodes=frozenset({"model:orders", "model:stg_orders"}), + ), + SnapshotPublicationTestCase( + description="configured target with encoded path characters", + target="preview #1", + expected_generation=2, + expected_nodes=frozenset({"model:orders", "model:stg_orders"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_code_location_when_publishing_from_cli_then_compilation_isolated_from_store_and_failed_edits_keep_last_good( + test_case: SnapshotPublicationTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir: Path = tmp_path / "project" + files: dict[str, str] = TYPE_PROOF_FILES | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + f'\n[targets.{json.dumps(test_case.target)}]\nschema = "main"\n', + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job\n" + 'daily_orders = job("daily_orders", select="orders", description="Build orders.")\n' + ), + } + write_files(project_dir=project_dir, files=files) + _ = subprocess.run(["git", "init", "-q", str(project_dir)], check=True, capture_output=True) + _ = subprocess.run( + [ + "git", + "-C", + str(project_dir), + "-c", + "user.name=Orders Contributor", + "-c", + "user.email=orders@example.com", + "commit", + "--allow-empty", + "-qm", + "Initial orders", + ], + check=True, + capture_output=True, + ) + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + arguments: list[str] = [ + "--no-color", + "--project-dir", + str(project_dir), + "location", + "publish", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + test_case.target, + "--image-id", + "sha256:synthetic-orders-image", + ] + assert run_cli([*arguments, "--generation", "1"]) == 0 + first_output: CaptureResult[str] = capsys.readouterr() + first: dict[str, Any] = json.loads(first_output.out) + assert first["generation"] == 1 + assert "Location snapshot published." in first_output.err + initial: dict[str, Any] = current_snapshot(api, test_case.target) + assert frozenset(initial["nodes"]) == test_case.expected_nodes + assert initial["documents"]["jobs"]["daily_orders"]["selection"] == ["model:orders"] + assert not (project_dir / ".sqlbuild/control.sqlite3").exists() + assert not (project_dir / "warehouse.duckdb").exists() + with sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3") as connection: + metadata: tuple[str, int, str, str] | None = connection.execute( + "SELECT git_revision, git_dirty, git_author, image_id FROM sqb_snapshots WHERE snapshot_id=?", + (first["snapshot_id"],), + ).fetchone() + assert metadata is not None + assert len(metadata[0]) == 40 + assert metadata[1:] == (1, "Orders Contributor", "sha256:synthetic-orders-image") + + staging: Path = project_dir / "models/stg_orders.sql" + _ = staging.write_text( + files["models/stg_orders.sql"].replace("CAST(5 AS INTEGER)", "CAST(11 AS INTEGER)") + ) + assert run_cli([*arguments, "--generation", "2"]) == 0 + second_output: CaptureResult[str] = capsys.readouterr() + second: dict[str, Any] = json.loads(second_output.out) + assert second["generation"] == test_case.expected_generation + updated: dict[str, Any] = current_snapshot(api, test_case.target) + assert {change["subject_key"] for change in updated["changes"]} == {"model:stg_orders"} + assert "CAST(11 AS INTEGER)" in updated["nodes"]["model:stg_orders"]["raw_sql"] + + _ = staging.write_text("SELECT FROM\n") + assert run_cli([*arguments, "--generation", "3"]) != 0 + failed_output: CaptureResult[str] = capsys.readouterr() + assert not failed_output.out + assert "Location command failed." in failed_output.err + assert current_snapshot(api, test_case.target)["record"] == second + + _ = staging.write_text(files["models/stg_orders.sql"]) + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", "invalid-credential") + assert run_cli([*arguments, "--generation", "3"]) != 0 + rejected_output: CaptureResult[str] = capsys.readouterr() + assert not rejected_output.out + assert "HTTP 401" in rejected_output.err + assert api.bearer not in "".join( + ( + first_output.out, + first_output.err, + second_output.out, + second_output.err, + failed_output.err, + rejected_output.err, + ) + ) + assert current_snapshot(api, test_case.target)["record"] == second + + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + with socket.socket() as unavailable: + unavailable.bind(("127.0.0.1", 0)) + disconnected: list[str] = list(arguments) + disconnected[arguments.index("--server") + 1] = ( + f"http://127.0.0.1:{unavailable.getsockname()[1]}" + ) + assert run_cli([*disconnected, "--generation", "3"]) != 0 + transport_output: CaptureResult[str] = capsys.readouterr() + assert not transport_output.out + assert "generation 3 may already be published" in transport_output.err + assert current_snapshot(api, test_case.target)["record"] == second + + with monkeypatch.context() as result_scope: + result_scope.setattr("sys.stdout", RejectedPublicationOutput()) + assert run_cli([*arguments, "--generation", "3"]) != 0 + delivery_output: CaptureResult[str] = capsys.readouterr() + assert "was published, but result delivery failed" in delivery_output.err + assert current_snapshot(api, test_case.target)["record"]["generation"] == 3 diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py new file mode 100644 index 0000000000..b8e7303b67 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py @@ -0,0 +1,356 @@ +"""A native supervisor reloads real project interpreters through the authenticated HTTP API.""" + +import json +import os +import signal +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + LocationDatabaseTestCase, + LocationReloadTestCase, + LocationShutdownTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + current_snapshot, + location_process, + snapshot_api, + wait_for_generation, + wait_for_interpreter, + wait_for_location_health, + wait_for_log, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + LocationReloadTestCase( + "healthy connection", + "development", + 7, + int(sys.platform == "win32"), + "warehouse.duckdb", + True, + ), + LocationReloadTestCase( + "failed connection with encoded target", + "preview #1", + 7, + int(sys.platform == "win32"), + "missing/warehouse.duckdb", + False, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_served_location_when_inputs_change_or_restart_then_generations_reload_without_losing_last_good( + test_case: LocationReloadTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + compiler_pid: Path = tmp_path / "compiler.pid" + declaration: str = ( + "from sqlbuild.orchestration import job\n" + "import os\nfrom pathlib import Path\n" + f"Path({str(compiler_pid)!r}).write_text(str(os.getpid()))\n" + 'print("Orders declaration loaded.")\n' + 'daily_orders = job("daily_orders", select="orders", description="Build orders.")\n' + ) + files: dict[str, str] = TYPE_PROOF_FILES | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + f"\n[connections.local]\ndatabase = {json.dumps(test_case.database)}\n" + + f'\n[targets.{json.dumps(test_case.target)}]\nschema = "main"\n', + "orchestration/orders.py": declaration, + "sinks/orders.py": 'print("Loading orders sink", flush=True)\n', + } + write_files(project_dir=project, files=files) + _ = subprocess.run(["git", "init", "-q", str(project)], check=True, capture_output=True) + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + # A presentation-only width fallback must not bypass the native-only supervisor. + # Keep the adversarial width out of pytest's own terminal reporter. + environment: dict[str, str] = os.environ | {"COLUMNS": "999999999999999999999999999999"} + arguments: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + test_case.target, + ] + log: Path = project / "location.log" + publications: Path = project / "publications.jsonl" + with location_process(arguments, log, publications, env=environment) as process: + first: dict[str, Any] = wait_for_generation(api, test_case.target, 1, process, log) + first_interpreter: int = int(compiler_pid.read_text()) + heartbeat: dict[str, Any] = wait_for_location_health( + api, test_case.target, {"kind": "heartbeat"}, process, log + ) + initial_health: dict[str, Any] = wait_for_location_health( + api, + test_case.target, + {"kind": "connection", "generation": 1, "healthy": test_case.expected_connection}, + process, + log, + ) + assert initial_health["report"]["session_id"] == heartbeat["report"]["session_id"] + staging: Path = project / "models/stg_orders.sql" + for value in (11, 12, 13): + _ = staging.write_text( + files["models/stg_orders.sql"].replace( + "CAST(5 AS INTEGER)", f"CAST({value} AS INTEGER)" + ) + ) + second: dict[str, Any] = wait_for_generation(api, test_case.target, 2, process, log) + assert second["record"]["snapshot_id"] != first["record"]["snapshot_id"] + assert int(compiler_pid.read_text()) == first_interpreter + assert "CAST(13 AS INTEGER)" in second["nodes"]["model:stg_orders"]["raw_sql"] + assert {change["subject_key"] for change in second["changes"]} == {"model:stg_orders"} + + _ = staging.write_text("SELECT FROM invalid SQL !\n") + _ = wait_for_location_health( + api, test_case.target, {"kind": "compile_failed", "generation": 3}, process, log + ) + assert current_snapshot(api, test_case.target)["record"] == second["record"] + _ = staging.write_text( + files["models/stg_orders.sql"].replace("CAST(5 AS INTEGER)", "CAST(14 AS INTEGER)") + ) + recovered: dict[str, Any] = wait_for_generation(api, test_case.target, 3, process, log) + assert int(compiler_pid.read_text()) == first_interpreter + assert "CAST(14 AS INTEGER)" in recovered["nodes"]["model:stg_orders"]["raw_sql"] + + _ = (project / "orchestration/orders.py").write_text("invalid Python declaration !\n") + wait_for_log(process, log, "Location reload failed; retaining last good snapshot.") + _ = wait_for_location_health( + api, test_case.target, {"kind": "compile_failed", "generation": 4}, process, log + ) + assert current_snapshot(api, test_case.target)["record"] == recovered["record"] + _ = (project / "orchestration/orders.py").write_text( + declaration.replace("Build orders.", "Refresh order totals.") + ) + third: dict[str, Any] = wait_for_generation(api, test_case.target, 4, process, log) + replacement_interpreter: int = int(compiler_pid.read_text()) + assert replacement_interpreter != first_interpreter + assert ( + third["documents"]["jobs"]["daily_orders"]["description"] == "Refresh order totals." + ) + _ = subprocess.run( + [ + "git", + "-C", + str(project), + "-c", + "user.name=Orders Contributor", + "-c", + "user.email=orders@example.com", + "commit", + "--allow-empty", + "-qm", + "Record orders", + ], + check=True, + capture_output=True, + ) + committed: dict[str, Any] = wait_for_generation(api, test_case.target, 5, process, log) + assert int(compiler_pid.read_text()) == replacement_interpreter + assert committed["nodes"] == third["nodes"] + _ = (project / "sqlbuild_project.toml").write_text( + files["sqlbuild_project.toml"].replace('schema = "main"', 'schema = "reporting"') + ) + configured: dict[str, Any] = wait_for_generation(api, test_case.target, 6, process, log) + assert int(compiler_pid.read_text()) == replacement_interpreter + assert ( + configured["nodes"]["model:orders"]["definition"]["destination"]["schema"] + == "reporting" + ) + assert ( + configured["nodes"]["model:orders"]["raw_sql"] + == committed["nodes"]["model:orders"]["raw_sql"] + ) + process.terminate() + assert process.wait(timeout=5) == test_case.expected_stop_code + with location_process(arguments, log, publications, env=environment) as process: + restarted: dict[str, Any] = wait_for_generation( + api, test_case.target, test_case.expected_generation, process, log + ) + assert int(compiler_pid.read_text()) != replacement_interpreter + assert restarted["nodes"] == configured["nodes"] + restarted_health: dict[str, Any] = wait_for_location_health( + api, + test_case.target, + {"kind": "connection", "generation": 7, "healthy": test_case.expected_connection}, + process, + log, + ) + assert ( + restarted_health["report"]["session_id"] != initial_health["report"]["session_id"] + ) + assert restarted_health["received_at"] >= initial_health["received_at"] + # Redirected output and compiler cache writes must not feed the input watcher. + time.sleep(1) + assert current_snapshot(api, test_case.target)["record"] == restarted["record"] + process.terminate() + assert process.wait(timeout=5) == test_case.expected_stop_code + records: list[dict[str, Any]] = [ + json.loads(line) for line in publications.read_text().splitlines() + ] + assert [record["generation"] for record in records] == [1, 2, 3, 4, 5, 6, 7] + assert api.bearer not in log.read_text() + publications.read_text() + assert not (project / ".sqlbuild/control.sqlite3").exists() + assert (project / "warehouse.duckdb").exists() == test_case.expected_connection + + +@pytest.mark.skipif(sys.platform == "win32", reason="Unix signal and process ownership contract") +@pytest.mark.parametrize( + "test_case", + [ + LocationShutdownTestCase( + "terminate compiling interpreter after repeated heartbeats", + signal.SIGTERM, + 0, + 2, + "runtime", + ), + LocationShutdownTestCase("interrupt compiling interpreter", signal.SIGINT, 0, 1, "runtime"), + LocationShutdownTestCase( + "terminate connection checking interpreter", signal.SIGTERM, 0, 1, "debug" + ), + ], + ids=lambda case: case.description, +) +def test_given_a_busy_location_when_stopped_then_its_interpreter_is_reaped( + test_case: LocationShutdownTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + child_pid: Path = tmp_path / "interpreter.pid" + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + '\n[targets.development]\nschema = "main"\n', + "adapters/slow.py": ( + "import os, sys, time\nfrom pathlib import Path\n" + f"if {test_case.blocked_command!r} in sys.argv:\n" + f" Path({str(child_pid)!r}).write_text(str(os.getpid()))\n" + " time.sleep(60)\n" + ), + }, + ) + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + log: Path = tmp_path / "location.log" + arguments: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ] + with location_process(arguments, log, tmp_path / "publications.jsonl") as process: + interpreter: int = wait_for_interpreter(process, child_pid, log) + _ = wait_for_location_health( + api, + "development", + {"kind": "heartbeat"}, + process, + log, + minimum_reports=test_case.minimum_heartbeats, + ) + process.send_signal(test_case.signal) + assert process.wait(timeout=5) == test_case.expected_exit_code + with pytest.raises(ProcessLookupError): + os.kill(interpreter, 0) + + +@pytest.mark.parametrize( + "test_case", + [ + LocationDatabaseTestCase( + "custom database extension", "warehouse.db", 1, 'database = "warehouse.db"' + ), + LocationDatabaseTestCase( + "extensionless database", "warehouse", 1, 'database = "warehouse"' + ), + LocationDatabaseTestCase( + "custom attached database", + "inventory.db", + 1, + 'database = ":memory:"\nattach = [{path = "inventory.db", alias = "inventory"}]', + ), + ], + ids=lambda case: case.description, +) +def test_given_a_location_using_a_local_database_when_it_is_created_and_written_then_only_source_edits_reload( + test_case: LocationDatabaseTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + f"\n[connections.local]\n{test_case.connection_toml}\n" + + '\n[targets.development]\nschema = "main"\n', + }, + ) + database: Path = project / test_case.database + assert not database.exists() + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + log: Path = tmp_path / "location.log" + arguments: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ] + with location_process(arguments, log, tmp_path / "publications.jsonl") as process: + baseline: dict[str, Any] = wait_for_generation( + api, "development", test_case.expected_generation, process, log + ) + _ = wait_for_location_health( + api, + "development", + {"kind": "connection", "generation": 1, "healthy": True}, + process, + log, + ) + assert database.is_file() + with duckdb.connect(str(database)) as connection: + _ = connection.execute( + "CREATE TABLE orders (order_id INTEGER); INSERT INTO orders VALUES (17), (23)" + ) + assert connection.execute("SELECT sum(order_id) FROM orders").fetchone() == (40,) + time.sleep(1) + assert current_snapshot(api, "development")["record"] == baseline["record"] + time.sleep(1) + assert current_snapshot(api, "development")["record"] == baseline["record"] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py new file mode 100644 index 0000000000..1d1193a3a9 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py @@ -0,0 +1,124 @@ +"""Pinned group retries rebuild failed and skipped nodes without repeating successful writes.""" + +import json +import sqlite3 +import subprocess +import sys +from dataclasses import replace +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + scheduled_worker, + wait_for_run, +) + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + description="local group retries only failed and skipped nodes", + model_sql="SELECT 11 AS amount", + warehouse_query="SELECT amount FROM orders UNION ALL SELECT amount FROM totals ORDER BY amount", + expected_status="success", + expected_rows=[(17,), (99,)], + ), + WorkerExecutionTestCase( + description="Docker group retries only failed and skipped nodes", + model_sql="SELECT 11 AS amount", + warehouse_query="SELECT amount FROM orders UNION ALL SELECT amount FROM totals ORDER BY amount", + expected_status="success", + expected_rows=[(17,), (99,)], + launcher="docker", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_partial_group_when_retrying_from_failure_then_successful_nodes_are_not_rewritten( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + test_case = replace( + test_case, + files={ + "models/broken_orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql" + ).read_text(), + "models/totals.sql": ( + "MODEL (description 'Downstream totals.', materialized table);\n" + 'SELECT amount FROM __ref("broken_orders")\n' + ), + "models/inventory.sql": ( + "MODEL (description 'Independent inventory.', materialized table);\n" + "SELECT 29 AS amount\n" + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, job_group, schedule\n" + "staging = job('staging', description='Stage orders.', select='orders broken_orders')\n" + "marts = job('marts', description='Build totals.', select='totals')\n" + "inventory = job('inventory', description='Build inventory.', select='inventory')\n" + "annual_orders = job_group('annual_orders', description='Retry boundaries.', " + "jobs=[inventory, marts, staging], schedule=schedule(cron='0 0 * * *', " + "timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + ), + }, + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + source: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert source["status"] == "partial", worker.log.read_text() + with duckdb.connect(str(worker.project / "warehouse.duckdb")) as warehouse: + warehouse.execute("CREATE TABLE retry_ready AS SELECT 1 AS id") + warehouse.execute("UPDATE orders SET amount=99") + warehouse.execute("UPDATE inventory SET amount=71") + run_id: str = str(uuid4()) + command: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "rerun-run", + source["run_id"], + "--run-id", + run_id, + "--from-failure", + "--reason", + "Retry failed and skipped nodes after fixing warehouse input.", + ] + for expected_applied in [True, False]: + requested: subprocess.CompletedProcess[str] = subprocess.run( + command, capture_output=True, text=True, timeout=10, check=False + ) + assert requested.returncode == 0, requested.stderr + assert json.loads(requested.stdout) == {"run_id": run_id, "applied": expected_applied} + completed: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=run_id + ) + assert completed["status"] == test_case.expected_status, worker.log.read_text() + assert completed["snapshot_id"] == source["snapshot_id"] + assert completed["selection_json"] == ["model:broken_orders", "model:totals"] + assert completed["node_counts_json"] == {"success": 2} + with sqlite3.connect(worker.control_store) as control: + children: list[tuple[str, str]] = control.execute( + "SELECT job_key, selection_json FROM sqb_runs WHERE parent_run_id=? ORDER BY job_key", + (run_id,), + ).fetchall() + assert [(name, json.loads(selection)) for name, selection in children] == [ + ("marts", ["model:totals"]), + ("staging", ["model:broken_orders"]), + ] + assert control.execute( + "SELECT count(*) FROM sqb_security_audit WHERE action='run.rerun_requested'" + ).fetchone() == (1,) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + assert warehouse.sql("SELECT amount FROM inventory").fetchall() == [(71,)] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_retention.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_retention.py new file mode 100644 index 0000000000..cd62a882c6 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_retention.py @@ -0,0 +1,125 @@ +"""The installed server validates retention settings and expires audit rows only once delivered.""" + +import hashlib +import json +import os +import sqlite3 +import subprocess +import sys +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + AuditHorizonTestCase, + RetentionRejectionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + audit_actions, + run_server_until, +) + +UNREACHABLE_SINK: str = "http://127.0.0.1:9/orders-audit" + + +@pytest.mark.parametrize( + "test_case", + [ + AuditHorizonTestCase( + "Configured sink keeps the row it has not received", "undelivered_past_horizon", 1 + ) + ], + ids=lambda case: case.description, +) +def test_given_expired_audit_rows_when_server_runs_then_only_delivered_rows_are_deleted( + test_case: AuditHorizonTestCase, tmp_path: Path +) -> None: + fixture: dict[str, Any] = json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/security_audit_retention.json" + ).read_text() + )[test_case.fixture] + _ = run_server_until(tmp_path, ("--no-scheduler",), lambda log: "Control API ready" in log) + now: datetime = datetime.now(UTC) + sink_id: str = hashlib.sha256(UNREACHABLE_SINK.encode()).hexdigest() + with sqlite3.connect(tmp_path / ".sqlbuild/control.sqlite3") as connection: + (last,) = connection.execute( + "SELECT COALESCE(MAX(storage_id), 0) FROM sqb_security_audit" + ).fetchone() + _ = connection.executemany( + "INSERT INTO sqb_security_audit (event_id, occurred_at, actor, action, granting_role, " + "request_id, ip_address, scope_json, before_json, after_json, storage_id) " + "VALUES (?, ?, 'orders-admin', ?, 'admin', ?, NULL, '{}', 'null', '{}', ?)", + [ + ( + str(uuid4()), + (now - timedelta(days=row["age_days"])).strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + row["action"], + str(uuid4()), + last + offset, + ) + for offset, row in enumerate(fixture["rows"], start=1) + ], + ) + _ = connection.execute( + "INSERT INTO sqb_security_audit_deliveries (sink_id, storage_id, event_id, delivered_at) " + "SELECT ?, storage_id, event_id, occurred_at FROM sqb_security_audit " + "WHERE action IN (SELECT value FROM json_each(?))", + (sink_id, json.dumps(fixture["delivered_actions"])), + ) + log: str = run_server_until( + tmp_path, + ("--no-scheduler", *fixture["server_arguments"]), + lambda _: audit_actions(tmp_path) == tuple(fixture["expected_actions"]), + {"SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL": UNREACHABLE_SINK}, + ) + assert f"Security audit retention: retained past horizon: {test_case.expected_retained}" in log + with sqlite3.connect(tmp_path / ".sqlbuild/control.sqlite3") as connection: + deliveries: list[tuple[int]] = connection.execute( + "SELECT storage_id FROM sqb_security_audit_deliveries" + ).fetchall() + assert deliveries == [] + + +@pytest.mark.parametrize( + "test_case", + [ + RetentionRejectionTestCase( + "Zero audit days", + ("--audit-retention-days", "0"), + "--audit-retention-days must be between 1 and 65535", + ), + RetentionRejectionTestCase( + "Audit days beyond the supported range", + ("--audit-retention-days", "65536"), + "--audit-retention-days must be between 1 and 65535", + ), + RetentionRejectionTestCase( + "Zero snapshots", + ("--snapshot-retention-count", "0"), + "--snapshot-retention-count must be between 1 and 65535", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_retention_when_server_starts_then_it_refuses_before_storage( + test_case: RetentionRejectionTestCase, tmp_path: Path +) -> None: + environment: dict[str, str] = os.environ.copy() + environment.pop("SQLBUILD_CONTROL_DATABASE_URL", None) + result: subprocess.CompletedProcess[str] = subprocess.run( + [str(Path(sys.executable).with_name("sqb")), "server", "--port", "0", *test_case.arguments], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert result.returncode != 0 + assert test_case.expected_error in result.stderr + assert result.stdout == "" + assert not (tmp_path / ".sqlbuild/control.sqlite3").exists() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_scheduler_recovery.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_scheduler_recovery.py new file mode 100644 index 0000000000..d2c6eb0eaf --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_scheduler_recovery.py @@ -0,0 +1,50 @@ +"""Installed schedulers killed at random moments still tick each trigger's latest due minute once.""" + +import json +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + SchedulerCrashTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + scheduler_crash_history, +) + +NO_ANOMALIES: dict[str, int] = { + "triggers_without_ticks": 0, + "duplicate_ticks": 0, + "missing_latest_ticks": 0, + "off_schedule_ticks": 0, + "runs_without_exactly_one_tick": 0, + "request_event_mismatch": 0, + "accepted_duplicate_ticks": 0, +} + + +@pytest.mark.parametrize( + "test_case", + [ + SchedulerCrashTestCase( + description="two installed SQLite schedulers SIGKILLed while recording a backlog", + fixture="benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json", + expected_anomalies=NO_ANOMALIES, + expected_minimum_kills=32, + ) + ], + ids=lambda case: case.description, +) +def test_given_schedulers_killed_mid_backlog_when_a_survivor_recovers_then_the_latest_due_minute_ticks_once( + test_case: SchedulerCrashTestCase, tmp_path: Path +) -> None: + fixture: dict[str, Any] = json.loads(Path(test_case.fixture).read_text()) + observed: dict[str, int] = scheduler_crash_history(tmp_path, fixture) + kills: int = observed.pop("kills") + interrupted: int = observed.pop("ticks_before_survivor") + ticks: int = observed.pop("ticks") + print(f"{test_case.description}: {kills} kills, {interrupted} of {ticks} ticks before recovery") + assert observed == test_case.expected_anomalies + assert kills >= test_case.expected_minimum_kills + assert 0 < interrupted <= ticks diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py new file mode 100644 index 0000000000..5490cd0a8b --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py @@ -0,0 +1,185 @@ +"""An external warehouse writer drives a real sensor, queue and isolated DuckDB worker.""" + +import json +import os +import sqlite3 +import sys +import time +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any + +import duckdb +import pytest + +from sqlbuild._native import ( + SourceFreshnessRecord, + SourceFreshnessRenderers, + write_source_freshness_records, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + location_process, + scheduled_worker, + wait_for_generation, + wait_for_run, +) + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "changed external source versions", + "SELECT 73 AS amount", + "SELECT amount FROM orders", + "success", + [(73,)], + expected_origin="sensor", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.py" + ).read_text(), + "sources/orders.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" + ).read_text(), + }, + ), + WorkerExecutionTestCase( + "stale external timestamp versions", + "SELECT 73 AS amount", + "SELECT amount FROM orders", + "success", + [(73,)], + expected_origin="sensor", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" + ).read_text(), + "sources/orders.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/managed_orders.yml" + ).read_text(), + "python/loaders/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/orders_loader.py" + ).read_text(), + "source_observations.json": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_stale_journey.json" + ).read_text(), + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_external_source_versions_when_a_location_resyncs_then_only_changed_data_starts_real_work( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + data: dict[str, Any] = json.loads( + test_case.files.get( + "source_observations.json", + Path( + "benchmarks/duckshop/edge_cases/orchestration/source_data_sensor.json" + ).read_text(), + ) + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + for phase, (source, count) in enumerate( + ( + (data["record"], data["source_rows"]), + (data["record"] | data["late_record"], 1), + ) + ): + values: dict[str, Any] = source.copy() + for key in ("event_id", "state_database", "state_schema", "observed_at"): + values.pop(key) + records: tuple[SourceFreshnessRecord, ...] = tuple( + SourceFreshnessRecord( + **( + values + | { + "run_id": f"{source['run_id']}-{phase}-{index}", + "observed_at": datetime.fromisoformat(source["observed_at"]) + + timedelta(microseconds=index), + } + ) + ) + for index in range(count) + ) + adapter: Any = DuckDbAdapter() + connection: Any = adapter.connect( + {"database": str(worker.project / "warehouse.duckdb")} + ) + renderers: SourceFreshnessRenderers = SourceFreshnessRenderers( + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + render_insert_records_sql=adapter.render_insert_source_freshness_records_sql, + render_create_table_sql=adapter.render_create_source_freshness_table_sql, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, + ) + try: + for batch in (records, records[:1] * 3): + write_source_freshness_records( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + records=batch, + renderers=renderers, + ) + finally: + adapter.close(connection) + log: Path = tmp_path / f"location-{phase}.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(worker.project), + "location", + "serve", + "--server", + worker.api.url, + "--location-id", + worker.api.location_id, + "--target", + "development", + ], + log, + tmp_path / f"location-{phase}.stdout", + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": worker.api.bearer}, + ) as location: + _ = wait_for_generation(worker.api, "development", phase + 2, location, log) + deadline: float = time.monotonic() + 30 + observed: int = 0 + runs: list[tuple[str]] = [] + while observed != data["source_rows"] + phase or len(runs) != phase + 1: + assert location.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + observed = control.execute( + "SELECT COUNT(*) FROM sqb_warehouse_freshness" + ).fetchone()[0] + runs = control.execute( + "SELECT run_id FROM sqb_runs WHERE origin='sensor' ORDER BY queued_at,run_id" + ).fetchall() + assert len(runs) <= phase + 1, ( + "unchanged observations and exact retries must not create runs" + ) + time.sleep(0.05) + result: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=runs[-1][0] + ) + assert result["status"] == test_case.expected_status + assert result["job_key"] == "orders_refresh" + assert result["node_counts_json"] == {"success": 1} + with duckdb.connect( + str(worker.project / "warehouse.duckdb"), read_only=True + ) as warehouse: + assert ( + warehouse.execute(test_case.warehouse_query).fetchall() + == test_case.expected_rows + ) + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT COUNT(*) FROM sqb_run_requests").fetchone() == (2,) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py new file mode 100644 index 0000000000..a379eb2003 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py @@ -0,0 +1,205 @@ +"""Statistics sampling failures and expiration never disturb catalog reconciliation.""" + +import json +import os +import sqlite3 +import subprocess +import sys +import time +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + StatisticsHorizonRejectionTestCase, + StatisticsHorizonTestCase, + StatisticsRecoveryTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.catalog.helpers import ( + wait_for_catalog, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + location_process, + open_legacy_warehouse, + run_statistics_retention, + snapshot_api, + statistics_relations, + wait_for_generation, +) + + +@pytest.mark.parametrize( + "test_case", + [ + StatisticsRecoveryTestCase( + "Drop after discovery before sampling", "Warehouse history read failed" + ) + ], + ids=lambda case: case.description, +) +def test_given_relation_dropped_during_sampling_when_read_fails_then_catalog_can_reconcile( + test_case: StatisticsRecoveryTestCase, tmp_path: Path +) -> None: + fixture: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json").read_text() + )["drop_during_sampling"] + project: Path = tmp_path / "project" + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse(project, "sqlite", "path") + _ = adapter.execute(connection=connection, sql=fixture["create_sql"]) + adapter.close(connection) + (project / "orchestration/orders.py").write_text("") + (project / "models").mkdir() + (project / "models/orders.sql").write_text(fixture["model"]) + hook: Path = tmp_path / "sampling-hook" + hook.mkdir() + dropped: Path = hook / "dropped" + (hook / "sitecustomize.py").write_text( + "from pathlib import Path\n" + "import sqlite3\n" + "from sqlbuild.adapters.sqlite.classes.sqlite_adapter import SqliteAdapter\n" + "original = SqliteAdapter.estimate_relation_row_count\n" + "def sample(self, *, connection, database, schema, name, metadata_only=False):\n" + f" if name == {fixture['name']!r} and metadata_only:\n" + f" marker = Path({str(dropped)!r})\n" + " if not marker.exists():\n" + f" with sqlite3.connect({str(project / 'warehouse.db')!r}) as writer:\n" + f" writer.execute({fixture['drop_sql']!r})\n" + " marker.write_text('dropped after discovery')\n" + f" raise RuntimeError({fixture['failure']!r})\n" + " return original(self, connection=connection, database=database, schema=schema, " + "name=name, metadata_only=metadata_only)\n" + "SqliteAdapter.estimate_relation_row_count = sample\n" + ) + log: Path = tmp_path / "location.log" + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + with ( + snapshot_api(tmp_path / "control") as api, + location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer, "PYTHONPATH": str(hook)}, + ) as location, + ): + _ = wait_for_generation(api, "development", 1, location, log) + deadline: float = time.monotonic() + 30 + while not dropped.exists(): + assert location.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.05) + wait_for_catalog(control, [], location, log) + assert test_case.expected_log in log.read_text() + + +@pytest.mark.parametrize( + "test_case", + [ + StatisticsHorizonTestCase( + "Default horizon keeps a two-day-old sample", + (), + ("customers_two_days", "customers_recent"), + ), + StatisticsHorizonTestCase( + "One-day horizon deletes the two-day-old sample", + ("--statistics-retention-days", "1"), + ("customers_recent",), + ), + ], + ids=lambda case: case.description, +) +def test_given_configured_statistics_horizon_when_server_starts_then_older_samples_are_deleted( + test_case: StatisticsHorizonTestCase, tmp_path: Path +) -> None: + record: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/warehouse_statistics.json").read_text() + )["record"] + _ = run_statistics_retention(tmp_path, ()) + now: datetime = datetime.now(UTC) + location_id: str = str(uuid4()) + with sqlite3.connect(tmp_path / ".sqlbuild/control.sqlite3") as connection: + _ = connection.execute( + "INSERT INTO sqb_locations (location_id,name,created_by,created_at) " + "VALUES (?, 'orders', 'test-admin', ?)", + (location_id, now.isoformat()), + ) + _ = connection.executemany( + "INSERT INTO sqb_warehouse_statistics (event_id,location_id,target,state_database," + "state_schema,relation_name,observed_at,record_json,received_at) " + "VALUES (?,?,'development',NULL,'main',?,?,?,?)", + [ + ( + str(uuid4()), + location_id, + name, + (now - age).isoformat(), + json.dumps(record | {"name": name}), + now.isoformat(), + ) + for name, age in [ + ("sentinel_expired", timedelta(days=100)), + ("customers_two_days", timedelta(days=2)), + ("customers_recent", timedelta(hours=1)), + ] + ], + ) + log: str = run_statistics_retention(tmp_path, test_case.server_arguments) + assert "Control API ready" in log + assert statistics_relations(tmp_path) == test_case.expected_relations + + +@pytest.mark.parametrize( + "test_case", + [ + StatisticsHorizonRejectionTestCase( + "Zero days", "0", "--statistics-retention-days must be between 1 and 65535" + ), + StatisticsHorizonRejectionTestCase( + "Beyond the supported range", + "65536", + "--statistics-retention-days must be between 1 and 65535", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_statistics_horizon_when_server_starts_then_it_refuses_before_storage( + test_case: StatisticsHorizonRejectionTestCase, tmp_path: Path +) -> None: + environment: dict[str, str] = os.environ.copy() + environment.pop("SQLBUILD_CONTROL_DATABASE_URL", None) + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "server", + "--port", + "0", + "--statistics-retention-days", + test_case.days, + ], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert result.returncode != 0 + assert test_case.expected_error in result.stderr + assert result.stdout == "" + assert not (tmp_path / ".sqlbuild/control.sqlite3").exists() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py new file mode 100644 index 0000000000..429858e789 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py @@ -0,0 +1,269 @@ +"""Real adapter catalog inspection and automatic standalone location synchronization.""" + +import hashlib +import json +import os +import sqlite3 +import subprocess +import sys +from collections import Counter +from contextlib import closing +from pathlib import Path +from typing import Any + +import pytest +from testcontainers.postgres import PostgresContainer + +from scripts.native_tests.constants import POSTGRES_IMAGE +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + CatalogBoundaryTestCase, + CatalogInspectionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.catalog.helpers import ( + wait_for_catalog, + wait_for_statistics, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + location_process, + open_legacy_warehouse, + snapshot_api, + wait_for_generation, + warehouse_history_page, + warehouse_history_pages, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + CatalogInspectionTestCase("DuckDB catalog", "duckdb", "database", "BASE TABLE", "VIEW"), + CatalogInspectionTestCase("SQLite catalog", "sqlite", "path", "BASE TABLE", "VIEW"), + ], + ids=lambda case: case.description, +) +def test_given_real_catalog_when_reading_pages_then_columns_and_physical_types_are_read_without_mutation( + test_case: CatalogInspectionTestCase, tmp_path: Path +) -> None: + fixture: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json").read_text() + ) + project: Path = tmp_path / "project" + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + _ = adapter.execute(connection=connection, sql=fixture["create_sql"]) + for index in range(fixture["extra_relations"]): + _ = adapter.execute( + connection=connection, sql=f"CREATE TABLE catalog_orders_{index:03} (amount INTEGER)" + ) + adapter.close(connection) + before: str = hashlib.sha256((project / "warehouse.db").read_bytes()).hexdigest() + pages: list[dict[str, Any]] = warehouse_history_pages(project, "catalog") + assert tuple(len(page["records"]) for page in pages) == (100, 7) + records: list[dict[str, Any]] = [] + for page in pages: + records.extend(page["records"]) + assert [record["name"] for record in records] == sorted(record["name"] for record in records) + by_name: dict[str, Any] = {record["name"]: record for record in records} + expected: dict[str, Any] = fixture["record"] | {"relation_type": test_case.expected_table_type} + assert by_name[expected["name"]] == expected + assert by_name[expected["name"]]["relation_type"] == test_case.expected_table_type + assert hashlib.sha256((project / "warehouse.db").read_bytes()).hexdigest() == before + assert not (project / ".sqlbuild/control.sqlite3").exists() + + +@pytest.mark.parametrize( + "test_case", + [CatalogBoundaryTestCase("Case-distinct physical identities", True)], + ids=lambda case: case.description, +) +def test_given_case_distinct_postgres_relations_when_reading_catalog_then_columns_remain_exact( + test_case: CatalogBoundaryTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + fixture: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json").read_text() + ) + project: Path = tmp_path / "project" + write_files( + project_dir=project, + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "postgres"\n' + '[connections.local]\nconninfo = "${ENV:SQLBUILD_CATALOG_TEST_URL}"\n' + '[targets.development]\nschema = "main"\n' + ), + "orchestration/orders.py": 'raise AssertionError("unexpected declaration import")\n', + }, + ) + with PostgresContainer(POSTGRES_IMAGE, driver=None) as postgres: + url: str = postgres.get_connection_url() + monkeypatch.setenv("SQLBUILD_CATALOG_TEST_URL", url) + adapter: PostgresAdapter = PostgresAdapter() + connection: Any = adapter.connect({"conninfo": url}) + for sql in fixture["postgres_sql"]: + _ = adapter.execute(connection=connection, sql=sql) + adapter.close(connection) + page: Any = warehouse_history_page(project, "catalog") + assert (page is not None) is test_case.expected_page_delivered + assert page == {"records": fixture["postgres_records"], "next_after": None} + + +@pytest.mark.parametrize( + "test_case", + [CatalogBoundaryTestCase("Non-authoritative schema-only namespace", False)], + ids=lambda case: case.description, +) +def test_given_non_authoritative_adapter_scope_when_reading_catalog_then_no_page_can_authorize_removal( + test_case: CatalogBoundaryTestCase, + tmp_path: Path, +) -> None: + fixture: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json").read_text() + ) + project: Path = tmp_path / "project" + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse(project, "sqlite", "path") + adapter.close(connection) + # Exercise Databricks' real schema-only capability with a local read-only connection. + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + sys.executable, + "-c", + "from pathlib import Path\n" + "import sys\n" + "from sqlbuild._native import run_location_command\n" + "from sqlbuild.adapters.sqlite.classes.sqlite_adapter import SqliteAdapter\n" + "from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter\n" + "SqliteAdapter.lists_relations_authoritatively = DatabricksAdapter.lists_relations_authoritatively\n" + "raise SystemExit(run_location_command(project_dir=Path(sys.argv[1]), " + "location_id='00000000-0000-0000-0000-000000000001', selected_target='development', " + "generation=1, server_url='http://127.0.0.1:1', action='runtime'))\n", + str(project), + ], + input=json.dumps( + { + "catalog": { + "location": { + "database": fixture["non_authoritative_database"], + "schema": "main", + } + } + } + ) + + "\n", + text=True, + capture_output=True, + timeout=20, + check=False, + ) + assert result.returncode == 0, result.stderr + assert (json.loads(result.stdout) is not None) is test_case.expected_page_delivered + assert "No page delivered" in result.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + CatalogInspectionTestCase( + "DuckDB automatic catalog", "duckdb", "database", "BASE TABLE", "VIEW", 0 + ), + CatalogInspectionTestCase( + "SQLite automatic catalog", "sqlite", "path", "BASE TABLE", "VIEW" + ), + ], + ids=lambda case: case.description, +) +def test_given_served_location_when_catalog_changes_then_facts_and_current_projection_follow_the_warehouse( + test_case: CatalogInspectionTestCase, tmp_path: Path +) -> None: + fixture: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/warehouse_catalog.json").read_text() + ) + project: Path = tmp_path / "project" + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + _ = adapter.execute(connection=connection, sql=fixture["create_sql"]) + adapter.close(connection) + (project / "orchestration/orders.py").write_text("") + write_files(project_dir=project, files={"models/orders.sql": fixture["model"]}) + expected: dict[str, Any] = fixture["record"] | {"relation_type": test_case.expected_table_type} + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + log: Path = tmp_path / "location.log" + with ( + snapshot_api(tmp_path / "control") as api, + location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + ) as location, + ): + _ = wait_for_generation(api, "development", 1, location, log) + wait_for_catalog(control, [expected], location, log) + samples: list[dict[str, Any]] = wait_for_statistics( + control, expected["name"], location, log + ) + assert len(samples) == 1 + assert samples[0]["name"] == expected["name"] + assert samples[0]["byte_count"] is None + assert samples[0]["byte_count_basis"] is None + assert samples[0]["row_count"] == test_case.expected_metadata_rows + connection = adapter.connect({test_case.connection_key: str(project / "warehouse.db")}) + _ = adapter.execute(connection=connection, sql=fixture["alter_sql"]) + adapter.close(connection) + expected = expected | { + "columns": [*expected["columns"], {"name": "shipped", "data_type": "INTEGER"}] + } + wait_for_catalog(control, [expected], location, log) + connection = adapter.connect({test_case.connection_key: str(project / "warehouse.db")}) + for statement in fixture["replace_sql"]: + _ = adapter.execute(connection=connection, sql=statement) + adapter.close(connection) + expected = expected | { + "relation_type": test_case.expected_view_type, + "columns": [{"name": "order_id", "data_type": "INTEGER"}], + } + wait_for_catalog(control, [expected], location, log) + connection = adapter.connect({test_case.connection_key: str(project / "warehouse.db")}) + _ = adapter.execute(connection=connection, sql=fixture["drop_sql"]) + adapter.close(connection) + wait_for_catalog(control, [], location, log) + connection = adapter.connect({test_case.connection_key: str(project / "warehouse.db")}) + _ = adapter.execute(connection=connection, sql=fixture["create_sql"]) + adapter.close(connection) + expected = fixture["record"] | {"relation_type": test_case.expected_table_type} + wait_for_catalog(control, [expected], location, log) + with closing(sqlite3.connect(control)) as database: + _ = database.execute("DELETE FROM sqb_warehouse_catalog") + database.commit() + wait_for_catalog(control, [expected], location, log) + current: list[dict[str, Any]] = wait_for_statistics( + control, expected["name"], location, log + ) + assert samples[0] in current + assert ( + Counter(record["observed_at"][:13] for record in current)[ + samples[0]["observed_at"][:13] + ] + == 1 + ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py new file mode 100644 index 0000000000..1ebf21518f --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -0,0 +1,1171 @@ +"""Real warehouse writers, populated predecessors and the isolated read-only history protocol.""" + +import base64 +import gzip +import hashlib +import json +import os +import sqlite3 +import sys +import time +from contextlib import closing +from datetime import UTC, datetime, timedelta, timezone +from pathlib import Path +from typing import Any +from urllib.request import Request, urlopen +from uuid import uuid4 + +import pytest + +from sqlbuild._native import ( + Fingerprint, + SourceFreshnessRecord, + SourceFreshnessRenderers, + build_compiled_project, + build_orchestration_snapshot, + discover_project_inputs, + read_latest_fingerprints, + read_latest_source_freshness, + write_fingerprints, + write_source_freshness_records, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + METADATA, + TYPE_PROOF_FILES, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + AutomaticHistoryTestCase, + CatalogConnectionTestCase, + FingerprintHistoryTestCase, + StateEventsTestCase, + WarehouseHistoryTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + append_audit_history, + append_contended_fingerprints, + append_janitor_history, + append_legacy_history, + append_microbatch_history, + append_migration_history, + append_old_name_history, + append_retried_fingerprints, + location_process, + open_legacy_warehouse, + snapshot_api, + sync_warehouse_history, + wait_for_generation, + wait_for_warehouse_sync_health, + warehouse_history_page, + warehouse_history_pages, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + StateEventsTestCase( + f"{adapter_name} {fixture_name}", + adapter_name, + connection_key, + expected_run_id, + fixture_name, + append_history, + legacy_columns, + ) + for adapter_name, connection_key, fixture_name, append_history, expected_run_id, legacy_columns in [ + ( + "duckdb", + "database", + "audit_result_history", + append_audit_history, + "", + ("audit_description", "audit_meta_json"), + ), + ( + "duckdb", + "database", + "janitor_event_history", + append_janitor_history, + "cleanup-run", + (), + ), + ( + "sqlite", + "path", + "audit_result_history", + append_audit_history, + "", + ("audit_description", "audit_meta_json"), + ), + ("sqlite", "path", "janitor_event_history", append_janitor_history, "cleanup-run", ()), + # SQLite has no supported concurrent-microbatch coordination writer. + ("duckdb", "database", "microbatch_history", append_microbatch_history, "", ()), + ( + "duckdb", + "database", + "migration_history", + append_migration_history, + "external-move", + (), + ), + ( + "duckdb", + "database", + "old_name_history", + append_old_name_history, + "external-move", + (), + ), + ("sqlite", "path", "migration_history", append_migration_history, "external-move", ()), + ("sqlite", "path", "old_name_history", append_old_name_history, "external-move", ()), + ( + "duckdb", + "database", + "old_name_history", + append_old_name_history, + "external-move", + ("column_aliases",), + ), + ( + "sqlite", + "path", + "old_name_history", + append_old_name_history, + "external-move", + ("column_aliases",), + ), + ] + ], + ids=lambda case: case.description, +) +def test_given_external_state_events_when_reading_pages_then_identities_text_and_timestamps_survive( + test_case: StateEventsTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + fixture: dict[str, Any] = json.loads( + Path( + f"benchmarks/duckshop/edge_cases/orchestration/{test_case.fixture_name}.json" + ).read_text() + ) + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + test_case.append_history(adapter, connection, fixture) + table: str = f"_sqlbuild_{fixture['kind']}" + source_key: str = fixture["source_key"] + legacy_columns: tuple[str, ...] = test_case.legacy_columns + tuple( + fixture.get("legacy_columns", ()) + ) + for column in legacy_columns: + _ = adapter.execute(connection=connection, sql=f"ALTER TABLE {table} DROP COLUMN {column}") + adapter.close(connection) + pages: list[dict[str, Any]] = warehouse_history_pages(project, fixture["kind"]) + rows: list[dict[str, Any]] = [] + for page in pages: + rows.extend(page["records"]) + assert [len(page["records"]) for page in pages] == fixture["page_lengths"] + assert {row[source_key] for row in rows} == { + f"{fixture['id_prefix']}-{index:03}" for index in range(fixture["source_rows"]) + } + expected: dict[str, Any] = fixture["record"] | dict.fromkeys(legacy_columns) + assert rows[0] == expected + assert rows[0][fixture.get("run_key", "run_id")] == test_case.expected_run_id + assert rows[fixture["later_index"]] == ( + expected | fixture["later_record"] | dict.fromkeys(legacy_columns) + ) + assert rows[-1] == expected | fixture["last_record"] | dict.fromkeys(legacy_columns) + recent_pages: list[dict[str, Any]] = warehouse_history_pages( + project, fixture["kind"], fixture["later_time"] + ) + recent_rows: list[dict[str, Any]] = [] + for page in recent_pages: + recent_rows.extend(page["records"]) + assert recent_rows == [expected | fixture["later_record"] | dict.fromkeys(legacy_columns)] + connection = adapter.connect({test_case.connection_key: str(project / "warehouse.db")}) + columns: set[str] = { + column.name + for column in adapter.get_columns( + connection=connection, database=None, schema="main", name=table + ) + } + assert not columns.intersection(legacy_columns), "the reader never upgrades source storage" + assert adapter.query( + connection=connection, sql=f"SELECT COUNT(*) FROM {table}", limit=1 + ).rows == ((fixture["source_count"],),) + adapter.close(connection) + assert not (project / ".sqlbuild/control.sqlite3").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + StateEventsTestCase( + f"{adapter_name} {fixture_name}", + adapter_name, + connection_key, + expected_run_id, + fixture_name, + append_history, + legacy_columns, + ) + for adapter_name, connection_key, fixture_name, append_history, expected_run_id, legacy_columns in [ + ( + "duckdb", + "database", + "audit_result_history", + append_audit_history, + "", + ("audit_description", "audit_meta_json"), + ), + ( + "duckdb", + "database", + "janitor_event_history", + append_janitor_history, + "cleanup-run", + (), + ), + ( + "sqlite", + "path", + "audit_result_history", + append_audit_history, + "", + ("audit_description", "audit_meta_json"), + ), + ("sqlite", "path", "janitor_event_history", append_janitor_history, "cleanup-run", ()), + ("duckdb", "database", "microbatch_history", append_microbatch_history, "", ()), + ( + "duckdb", + "database", + "migration_history", + append_migration_history, + "external-move", + (), + ), + ( + "duckdb", + "database", + "old_name_history", + append_old_name_history, + "external-move", + (), + ), + ("sqlite", "path", "migration_history", append_migration_history, "external-move", ()), + ("sqlite", "path", "old_name_history", append_old_name_history, "external-move", ()), + ] + ], + ids=lambda case: case.description, +) +def test_given_a_served_location_when_state_event_history_is_rebuilt_and_retained_then_only_complete_scans_delete( + test_case: StateEventsTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + fixture: dict[str, Any] = json.loads( + Path( + f"benchmarks/duckshop/edge_cases/orchestration/{test_case.fixture_name}.json" + ).read_text() + ) + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + test_case.append_history(adapter, connection, fixture) + adapter.close(connection) + (project / "orchestration/orders.py").write_text("") + write_files(project_dir=project, files={"models/orders.sql": fixture["model"]}) + with snapshot_api(tmp_path / "control") as api: + log: Path = tmp_path / "location.log" + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ + | {"SQLBUILD_LOCATION_TOKEN": api.bearer, "SQLBUILD_METADATA_RECONCILE_SECONDS": "30"}, + ) as location: + _ = wait_for_generation(api, "development", 1, location, log) + identities: dict[str, str] = {} + source_key: str = fixture["source_key"] + mirror_table: str = f"sqb_warehouse_{fixture['kind']}" + all_ids: set[str] = { + f"{fixture['id_prefix']}-{index:03}" for index in range(fixture["source_rows"]) + } + for expected_ids, mirror_sql, warehouse_sql in ( + (all_ids, f"DELETE FROM {mirror_table}", "SELECT 1"), + ( + all_ids, + "SELECT 1", + f"DELETE FROM _sqlbuild_{fixture['kind']} " + f"WHERE {source_key} <> '{fixture['retained_result']}'", + ), + ({fixture["retained_result"]}, "SELECT 1", "SELECT 1"), + ): + deadline: float = time.monotonic() + 45 + mirrored: list[tuple[str, str]] = [] + while len(mirrored) != len(expected_ids): + assert location.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + with closing(sqlite3.connect(control)) as database: + mirrored = database.execute( + f"SELECT event_id,record_json FROM {mirror_table} ORDER BY event_id" + ).fetchall() + time.sleep(0.05) + assert {json.loads(row)[source_key] for _, row in mirrored} == expected_ids + for event_id, text in mirrored: + record: dict[str, Any] = json.loads(text) + assert record == ( + fixture["record"] + | {source_key: record[source_key]} + | fixture["record_overrides"].get(record[source_key], {}) + ) + assert record[fixture.get("run_key", "run_id")] == test_case.expected_run_id + assert identities.setdefault(record[source_key], event_id) == event_id + with closing(sqlite3.connect(control)) as database: + _ = database.execute(mirror_sql) + database.commit() + connection = adapter.connect( + {test_case.connection_key: str(project / "warehouse.db")} + ) + _ = adapter.execute(connection=connection, sql=warehouse_sql) + adapter.close(connection) + + +@pytest.mark.parametrize( + "test_case", + [ + CatalogConnectionTestCase("DuckDB independent reader", "duckdb", "database", "legacy-run"), + CatalogConnectionTestCase("SQLite independent reader", "sqlite", "path", "legacy-run"), + ], + ids=lambda case: case.description, +) +def test_given_independent_catalog_connection_when_reading_history_then_writer_secrets_and_files_are_untouched( + test_case: CatalogConnectionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + connection_key: str = test_case.connection_key + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse(project, test_case.adapter_name, connection_key) + adapter.close(connection) + config: Path = project / "sqlbuild_project.toml" + config.write_text( + Path("benchmarks/duckshop/edge_cases/orchestration/catalog_connection.toml") + .read_text() + .replace('adapter = "sqlite"', f'adapter = "{test_case.adapter_name}"') + .replace("\npath =", f"\n{connection_key} =") + ) + monkeypatch.delenv("SQLBUILD_MISSING_WRITER_PASSWORD", raising=False) + monkeypatch.setenv("SQLBUILD_CATALOG_DATABASE", "warehouse.db") + first: Any = warehouse_history_page(project, "source_freshness") + assert first is not None and len(first["records"]) == 1 + assert first["records"][0]["run_id"] == test_case.expected_run_id + # Local target selection replaces only the catalog name, not the writer connection. + (project / "warehouse.db").rename(project / "retained-warehouse.db") + (project / "sqlbuild_local.toml").write_text( + f'[connections.mirror]\n{connection_key} = "retained-warehouse.db"\n' + '[targets.development]\ncatalog_connection = "mirror"\n' + ) + assert warehouse_history_page(project, "source_freshness") == first + assert not (project / "must-not-open.db").exists() + assert not (project / "must-not-open-writer.db").exists() + assert not (project / ".sqlbuild/control.sqlite3").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + WarehouseHistoryTestCase( + "DuckDB typed timestamps", + "duckdb", + (100, 6), + "database", + (), + append_retried_fingerprints, + ), + WarehouseHistoryTestCase( + "SQLite timestamp text", + "sqlite", + (100, 6), + "path", + ("delete",), + append_contended_fingerprints, + ), + ], + ids=lambda case: case.description, +) +def test_given_populated_predecessor_tables_when_writers_add_ids_then_history_preserves_new_legacy_and_retried_observations( + test_case: WarehouseHistoryTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + connection_key: str = test_case.connection_key + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + adapter.close(connection) + for journal_mode in test_case.journal_modes: + with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: + assert predecessor.execute("PRAGMA journal_mode=DELETE").fetchone() == (journal_mode,) + + def page(after: list[str | None] | None = None, observed_since: str | None = None) -> Any: + return warehouse_history_page(project, "source_freshness", after, observed_since) + + old: dict[str, Any] = page() + assert len(old["records"]) == 1 and old["records"][0]["event_id"] is None + assert page(observed_since="2026-01-01T09:00:00Z") == old + assert page(observed_since="2026-01-01T10:00:00.000001Z") == { + "records": [], + "next_after": None, + } + for journal_mode in test_case.journal_modes: + with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: + assert predecessor.execute("PRAGMA journal_mode").fetchone() == (journal_mode,) + connection = adapter.connect({connection_key: str(project / "warehouse.db")}) + # Reading history never upgrades the warehouse or imports project declarations. + assert "event_id" not in { + column.name + for column in adapter.get_columns( + connection=connection, database=None, schema="main", name="_sqlbuild_source_freshness" + ) + } + latest: Any = read_latest_fingerprints( + connection=connection, + execute=adapter.execute, + table_exists=True, + database=None, + schema="main", + render_qualified_name=adapter.render_qualified_name, + render_read_latest_sql=adapter.render_read_latest_fingerprints_sql, + ) + assert latest.fingerprints["orders"].run_id == "legacy-run" + fingerprint: Fingerprint = Fingerprint( + node_type="model", + node_name="orders", + target_database=None, + target_schema="main", + target_name="orders", + run_id="external-run", + definition_hash="definition", + version_hash="version", + schema_fingerprint="schema", + definition="SELECT 2 AS order_id", + ts=datetime(2026, 2, 1, tzinfo=UTC), + ) + test_case.append_fingerprints( + adapter=adapter, + connection=connection, + database_path=project / "warehouse.db", + fingerprint=fingerprint, + ) + expected_fingerprint_id: str = hashlib.sha256( + json.dumps( + [ + "sqlbuild.fingerprint.event.v1", + "model", + "orders", + None, + "main", + "orders", + "external-run", + "version", + ], + ensure_ascii=False, + separators=(",", ":"), + ).encode() + ).hexdigest() + assert adapter.query( + connection=connection, + sql="SELECT event_id FROM main._sqlbuild_fingerprints ORDER BY ts", + limit=10, + ).rows == ( + (None,), + (expected_fingerprint_id,), + (expected_fingerprint_id,), + ) + records: tuple[SourceFreshnessRecord, ...] = tuple( + SourceFreshnessRecord( + source_name=("Orders", "orders", "orders ", "ordérs")[index % 4], + target_database=("", None)[index % 2], + target_schema="main", + target_name="orders", + run_id=f"external-{index}", + strategy="max_column", + value_kind="string", + data_version=("o'rders 🧾", None, None)[index % 3], + data_version_hash=f"value-{index}", + observed_at=datetime(2026, 2, 1, microsecond=index, tzinfo=UTC), + ) + for index in range(104) + ) + renderers: SourceFreshnessRenderers = SourceFreshnessRenderers( + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + render_insert_records_sql=adapter.render_insert_source_freshness_records_sql, + render_create_table_sql=adapter.render_create_source_freshness_table_sql, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, + ) + for batch in (records, records[:1]): + write_source_freshness_records( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + records=batch, + renderers=renderers, + ) + expected_freshness_id: str = hashlib.sha256( + json.dumps( + [ + "sqlbuild.source-freshness.event.v1", + "Orders", + "", + "main", + "orders", + "external-0", + "2026-02-01T00:00:00.000000+00:00", + "max_column", + "string", + "value-0", + ], + ensure_ascii=False, + separators=(",", ":"), + ).encode() + ).hexdigest() + assert adapter.query( + connection=connection, + sql="SELECT event_id FROM main._sqlbuild_source_freshness WHERE run_id='external-0'", + limit=10, + ).rows == ( + (expected_freshness_id,), + (expected_freshness_id,), + ) + # An equivalent offset denotes the same observation, not another event. + equivalent: SourceFreshnessRecord = SourceFreshnessRecord( + source_name="Orders", + target_database="", + target_schema="main", + target_name="orders", + run_id="external-0", + strategy="max_column", + value_kind="string", + data_version="o'rders 🧾", + data_version_hash="value-0", + observed_at=datetime(2026, 2, 1, 1, tzinfo=timezone(timedelta(hours=1))), + ) + assert expected_freshness_id in adapter.render_insert_source_freshness_records_sql( + database=None, schema="main", records=(equivalent,) + ) + current: Any = read_latest_source_freshness( + connection=connection, + execute=adapter.execute, + table_exists=True, + database=None, + schema="main", + render_qualified_name=adapter.render_qualified_name, + render_read_latest_sql=adapter.render_read_latest_source_freshness_sql, + ) + assert {row.run_id for row in current.records.values()} == { + "legacy-run", + "external-100", + "external-101", + "external-102", + "external-103", + } + adapter.close(connection) + pages: list[dict[str, Any]] = [page()] + while pages[-1]["next_after"] is not None: + pages.append(page(pages[-1]["next_after"])) + assert len(pages) < 4 + assert tuple(len(item["records"]) for item in pages) == test_case.expected_page_lengths + imported: list[dict[str, Any]] = [] + for item in pages: + imported.extend(item["records"]) + event_ids: set[str | None] = {row["event_id"] for row in imported} + assert len(event_ids - {None}) == 104 + assert sum(row["event_id"] is None for row in imported) == 1 + assert {row["run_id"] for row in imported} == {"legacy-run"} | { + f"external-{index}" for index in range(104) + } + assert {row["target_database"] for row in imported} == {None, ""} + # The time bound is inclusive and independent of source-name/keyset ordering. + # SQL may read a superset within the boundary day, so a short decoded page + # must retain its continuation if the raw query filled the page. + observed_since: str = "2026-02-01T01:00:00.000050+01:00" + recent: list[dict[str, Any]] = [page(observed_since=observed_since)] + while recent[-1]["next_after"] is not None: + recent.append(page(recent[-1]["next_after"], observed_since)) + assert len(recent) < 4 + recent_records: list[dict[str, Any]] = [] + for item in recent: + recent_records.extend(item["records"]) + assert len(recent_records) == 54 + assert {row["run_id"] for row in recent_records} == { + f"external-{index}" for index in range(50, 104) + } + zero: dict[str, Any] = page(observed_since="2026-02-01T00:00:00Z") + assert ( + len(zero["records"]) + len(page(zero["next_after"], "2026-02-01T00:00:00Z")["records"]) + == 105 + ) + assert sum(row["run_id"] == "external-0" for row in zero["records"]) == 2 + # An empty decoded page can still have another bounded SQL page to inspect. + empty_since: str = "2026-02-01T00:00:00.000104Z" + empty: dict[str, Any] = page(observed_since=empty_since) + assert empty["records"] == [] and empty["next_after"] is not None + assert page(empty["next_after"], empty_since) == {"records": [], "next_after": None} + assert page(observed_since="2030-01-01T00:00:00Z") == { + "records": [], + "next_after": None, + } + assert not (project / ".sqlbuild/control.sqlite3").exists() + (project / "warehouse.db").rename(project / "retained-warehouse.db") + assert page() is None + assert not (project / "warehouse.db").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + CatalogConnectionTestCase("DuckDB retained key set", "duckdb", "database", "retained-run"), + CatalogConnectionTestCase("SQLite retained key set", "sqlite", "path", "retained-run"), + ], + ids=lambda case: case.description, +) +def test_given_external_warehouse_retention_when_a_complete_scan_arrives_then_mirror_deletions_and_empty_resync_match_source( + test_case: CatalogConnectionTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "warehouse-project" + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + for table in ("_sqlbuild_source_freshness", "_sqlbuild_fingerprints"): + _ = adapter.execute(connection=connection, sql=f"INSERT INTO {table} SELECT * FROM {table}") + _ = adapter.execute( + connection=connection, + sql=f"UPDATE {table} SET run_id='retained-run' WHERE rowid=(SELECT MAX(rowid) FROM {table})", + ) + adapter.close(connection) + declarations: Path = tmp_path / "declarations" + write_files(project_dir=declarations, files=TYPE_PROOF_FILES) + compiled: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=declarations), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + with snapshot_api(tmp_path / "control") as api: + _ = build_orchestration_snapshot( + compiled, + json.dumps(METADATA | {"location_id": api.location_id, "target": "development"}), + declarations, + ).publish(api.url, api.bearer) + session_id: str = str(uuid4()) + heartbeat: Request = Request( + f"{api.url}/api/locations/{api.location_id}/health", + data=json.dumps( + { + "session_id": session_id, + "event_id": str(uuid4()), + "target": "development", + "health": {"kind": "heartbeat"}, + } + ).encode(), + headers={"Authorization": f"Bearer {api.bearer}", "Content-Type": "application/json"}, + ) + with urlopen(heartbeat, timeout=5) as response: + assert json.load(response) == {"applied": True} + for kind, mirror in [ + ("source_freshness", "sqb_warehouse_freshness"), + ("fingerprints", "sqb_warehouse_fingerprints"), + ]: + initial: dict[str, Any] = warehouse_history_page(project, kind) + assert len(initial["records"]) == 2 and initial["next_after"] is None + batch: dict[str, Any] = { + "kind": kind, + "session_id": session_id, + "generation": 1, + "records": initial["records"], + } + assert sync_warehouse_history(api, batch) == {"applied": True} + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + with closing(sqlite3.connect(control)) as database: + before: list[tuple[str, str]] = database.execute( + f"SELECT event_id,record_json FROM {mirror} ORDER BY event_id" + ).fetchall() + assert len(before) == 2 + connection = adapter.connect({test_case.connection_key: str(project / "warehouse.db")}) + _ = adapter.execute( + connection=connection, sql=f"DELETE FROM _sqlbuild_{kind} WHERE run_id='legacy-run'" + ) + adapter.close(connection) + remaining: dict[str, Any] = warehouse_history_page(project, kind) + assert [row["run_id"] for row in remaining["records"]] == [test_case.expected_run_id] + batch["records"] = remaining["records"] + batch["scan"] = { + "scan_id": str(uuid4()), + "location": {"database": None, "schema": "main"}, + "complete": False, + } + assert sync_warehouse_history(api, batch) == {"applied": False} + with closing(sqlite3.connect(control)) as database: + assert database.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone() == (2,) + batch["records"] = [] + batch["scan"]["complete"] = True + assert sync_warehouse_history(api, batch) == {"applied": True} + originals_by_run: dict[str, tuple[str, str]] = { + json.loads(row[1])["run_id"]: row for row in before + } + expected: list[tuple[str, str]] = [originals_by_run[test_case.expected_run_id]] + with closing(sqlite3.connect(control)) as database: + assert ( + database.execute(f"SELECT event_id,record_json FROM {mirror}").fetchall() + == expected + ) + _ = database.execute(f"DELETE FROM {mirror}") + database.commit() + assert sync_warehouse_history( + api, + { + "kind": kind, + "session_id": session_id, + "generation": 1, + "records": remaining["records"], + }, + ) == {"applied": True} + with closing(sqlite3.connect(control)) as database: + assert ( + database.execute(f"SELECT event_id,record_json FROM {mirror}").fetchall() + == expected + ) + assert warehouse_history_page(project, kind) == remaining, ( + "sync does not mutate warehouse state" + ) + assert not (project / ".sqlbuild/control.sqlite3").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + AutomaticHistoryTestCase("DuckDB automatic history", "duckdb", "database", "legacy-run"), + AutomaticHistoryTestCase("SQLite automatic history", "sqlite", "path", "legacy-run"), + AutomaticHistoryTestCase( + "SQLite wide history pages", + "sqlite", + "path", + "legacy-run", + wide=True, + initial_seconds=30, + expected_average_bytes=640_000, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_served_location_when_warehouse_history_changes_then_periodic_read_only_sync_reconciles_without_manual_requests( + test_case: AutomaticHistoryTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + fixture: dict[str, Any] = json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/automatic_warehouse_history.json" + ).read_text() + ) + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + expected_runs: set[str] = {test_case.expected_run_id} + definition: str = base64.b64encode( + ("SELECT 17 /*" + "x" * fixture["wide_definition_bytes"] + "*/").encode() + ).decode() + for index in range(fixture["wide_fingerprint_count"] * test_case.wide): + expected_runs.add(f"automatic-{index}") + _ = adapter.execute( + connection=connection, + sql=( + "INSERT INTO _sqlbuild_fingerprints SELECT node_type," + f"'orders-{index}',target_database,target_schema,target_name,'automatic-{index}'," + "definition_hash,version_hash,schema_fingerprint," + f"'{definition}',metadata_json_b64,ts FROM _sqlbuild_fingerprints WHERE run_id='legacy-run'" + ), + ) + assert adapter.query( + connection=connection, + sql="SELECT SUM(LENGTH(definition_b64))*3/4 FROM _sqlbuild_fingerprints", + limit=1, + ).rows[0][0] >= test_case.expected_average_bytes * len(expected_runs), ( + "wide fixture exceeds one API request before any synchronization" + ) + adapter.close(connection) + (project / "orchestration/orders.py").write_text("") + write_files( + project_dir=project, + files={"models/orders.sql": fixture["model"]}, + ) + with snapshot_api(tmp_path / "control") as api: + log: Path = tmp_path / "location.log" + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ + | {"SQLBUILD_LOCATION_TOKEN": api.bearer, "SQLBUILD_METADATA_RECONCILE_SECONDS": "30"}, + ) as location: + _ = wait_for_generation(api, "development", 1, location, log) + initial_tables: dict[str, set[str]] = { + "sqb_warehouse_fingerprints": expected_runs, + "sqb_warehouse_freshness": {test_case.expected_run_id}, + } + for expected_tables, seconds, mirror_sql, warehouse_sql in [ + ( + initial_tables, + test_case.initial_seconds, + "DELETE FROM {table}", + "SELECT 1", + ), + (initial_tables, 45, "SELECT 1", "DELETE FROM {table}"), + ({table: set() for table in initial_tables}, 45, "SELECT 1", "SELECT 1"), + ]: + deadline: float = time.monotonic() + seconds + observed: tuple[int, int] = (-1, -1) + expected: tuple[int, int] = ( + len(expected_tables["sqb_warehouse_fingerprints"]), + len(expected_tables["sqb_warehouse_freshness"]), + ) + while observed != expected: + assert location.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + with closing(sqlite3.connect(control)) as database: + observed = database.execute( + "SELECT (SELECT COUNT(*) FROM sqb_warehouse_fingerprints)," + "(SELECT COUNT(*) FROM sqb_warehouse_freshness)" + ).fetchone() + time.sleep(0.05) + with closing(sqlite3.connect(control)) as database: + bodies: list[tuple[bytes]] = database.execute( + "SELECT t.body FROM sqb_warehouse_fingerprints f " + "JOIN sqb_source_texts t ON t.text_hash=f.definition_text_hash" + ).fetchall() + assert sum(len(gzip.decompress(body)) for (body,) in bodies) >= ( + test_case.expected_average_bytes + * len(expected_tables["sqb_warehouse_fingerprints"]) + ) + for table, expected_runs in expected_tables.items(): + rows: list[tuple[str]] = database.execute( + f"SELECT record_json FROM {table}" + ).fetchall() + runs: set[str] = {json.loads(row[0])["run_id"] for row in rows} + assert runs == expected_runs + _ = database.execute(mirror_sql.format(table=table)) + database.commit() + connection = adapter.connect( + {test_case.connection_key: str(project / "warehouse.db")} + ) + for table in ("_sqlbuild_fingerprints", "_sqlbuild_source_freshness"): + _ = adapter.execute( + connection=connection, sql=warehouse_sql.format(table=table) + ) + adapter.close(connection) + + +@pytest.mark.parametrize( + "test_case", + [ + AutomaticHistoryTestCase("DuckDB incremental overlap", "duckdb", "database", "legacy-run"), + AutomaticHistoryTestCase("SQLite incremental overlap", "sqlite", "path", "legacy-run"), + ], + ids=lambda case: case.description, +) +def test_given_durable_mirror_when_late_rows_arrive_then_incremental_overlap_preserves_history_and_empty_resyncs( + test_case: AutomaticHistoryTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + fixture: dict[str, Any] = json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/incremental_warehouse_history.json" + ).read_text() + ) + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + append_legacy_history(adapter, connection, fixture["initial"], fixture["insert_sql"]) + adapter.close(connection) + (project / "orchestration/orders.py").write_text("") + write_files( + project_dir=project, + files={"models/orders.sql": "MODEL (description 'Orders.'); SELECT 17 AS amount"}, + ) + with snapshot_api(tmp_path / "control") as api: + control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" + log: Path = tmp_path / "location.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ + | {"SQLBUILD_LOCATION_TOKEN": api.bearer, "SQLBUILD_METADATA_RECONCILE_SECONDS": "300"}, + ) as location: + _ = wait_for_generation(api, "development", 1, location, log) + observed_health_at: datetime = datetime(1970, 1, 1, tzinfo=UTC) + for stage in fixture["stages"]: + connection = adapter.connect( + {test_case.connection_key: str(project / "warehouse.db")} + ) + arrivals: list[dict[str, str]] = stage["arrivals"] + [ + {"run_id": f"middle-{index:03}", "observed_at": "2026-02-01 11:58:00"} + for index in range(stage["overlap_count"]) + ] + append_legacy_history(adapter, connection, arrivals, fixture["insert_sql"]) + adapter.close(connection) + with closing(sqlite3.connect(control)) as database: + for table in ("sqb_warehouse_fingerprints", "sqb_warehouse_freshness"): + _ = database.execute(stage["mirror_sql"].format(table=table)) + database.commit() + expected: set[str] = set(stage["expected_runs"]) | { + f"middle-{index:03}" + for index in range(stage.get("expected_overlap_count", stage["overlap_count"])) + } + deadline: float = time.monotonic() + 45 + remaining: bool = True + while remaining: + assert location.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + remaining = False + with closing(sqlite3.connect(control)) as database: + for table in ("sqb_warehouse_fingerprints", "sqb_warehouse_freshness"): + observed: set[str] = { + json.loads(record)["run_id"] + for (record,) in database.execute( + f"SELECT record_json FROM {table}" + ) + } + assert observed <= expected, (table, stage, observed) + remaining |= observed != expected + remaining |= ( + log.read_text().count("Warehouse history synchronization completed.") + < stage["completed_cycles"] + ) + time.sleep(0.05) + assert test_case.expected_run_id in observed + reports: list[dict[str, Any]] = wait_for_warehouse_sync_health( + api, fixture["health"], location, log, after=observed_health_at + ) + sent_counts: dict[str, int] = dict.fromkeys(fixture["health"]["tables"], 0) + for report in reports: + sent_counts[report["report"]["health"]["table"]] += report["report"]["health"][ + "records_sent" + ] + assert sent_counts == dict.fromkeys( + fixture["health"]["tables"], stage["health_records_per_table"] + ) + observed_health_at = max( + datetime.fromisoformat(report["received_at"]) for report in reports + ) + for availability in fixture["availability"]: + _ = (project / availability["source"]).rename(project / availability["destination"]) + reports = wait_for_warehouse_sync_health( + api, availability, location, log, after=observed_health_at + ) + observed_health_at = max( + datetime.fromisoformat(report["received_at"]) for report in reports + ) + assert (project / "warehouse.db").exists() == availability["available"] + + +@pytest.mark.parametrize( + "test_case", + [ + FingerprintHistoryTestCase( + "DuckDB typed fingerprint timestamps", + "duckdb", + "database", + datetime(2026, 2, 1, tzinfo=UTC), + (), + (), + ("legacy-run",), + ), + FingerprintHistoryTestCase( + "SQLite offset fingerprint timestamps", + "sqlite", + "path", + datetime(2026, 1, 31, 23, tzinfo=timezone(timedelta(hours=-1))), + ("delete",), + ( + Path( + "benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_fingerprint_offsets.sql" + ), + ), + ("legacy-run", "offset-run"), + ), + ], + ids=lambda case: case.description, +) +def test_given_fingerprint_history_when_read_in_isolated_pages_then_legacy_retries_definitions_and_time_bounds_survive( + test_case: FingerprintHistoryTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + for fixture in test_case.fixture_paths: + _ = adapter.execute(connection=connection, sql=fixture.read_text()) + adapter.close(connection) + + def page(after: list[str | None] | None = None, observed_since: str | None = None) -> Any: + return warehouse_history_page(project, "fingerprints", after, observed_since) + + legacy: dict[str, Any] = page() + assert legacy["next_after"] is None + assert {row["run_id"] for row in legacy["records"]} == set(test_case.expected_legacy_runs) + assert legacy["records"][0]["event_id"] is None + assert legacy["records"][0]["definition"] == "SELECT 1 AS order_id" + assert legacy["records"][0]["metadata_json"] == "{}" + assert page(observed_since="2026-01-01T09:00:00Z") == legacy + assert page(observed_since="2026-01-01T10:00:00.000001Z") == { + "records": [], + "next_after": None, + } + connection = adapter.connect({test_case.connection_key: str(project / "warehouse.db")}) + assert "event_id" not in { + column.name + for column in adapter.get_columns( + connection=connection, database=None, schema="main", name="_sqlbuild_fingerprints" + ) + } + fingerprints: tuple[Fingerprint, ...] = tuple( + Fingerprint( + node_type=("model", "seed")[index % 2], + node_name=("Orders", "orders", "orders ", "ordérs", "o'rders 🧾")[index % 5], + target_database=(None, "")[index % 2], + target_schema="main", + target_name="orders", + run_id=f"external-{index}", + definition_hash="definition", + version_hash=f"version-{index}", + schema_fingerprint="schema", + definition="SELECT 'o''rders 🧾' AS order_note", + metadata_json='{"note":"orders 🧾"}', + ts=test_case.observed_at + timedelta(microseconds=index), + ) + for index in range(104) + ) + for batch in (fingerprints, fingerprints[:1]): + write_fingerprints( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + fingerprints=batch, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + render_create_table_sql=adapter.render_create_fingerprint_table_sql, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, + ) + adapter.close(connection) + for journal_mode in test_case.journal_modes: + with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: + assert predecessor.execute("PRAGMA journal_mode=DELETE").fetchone() == (journal_mode,) + pages: list[dict[str, Any]] = [page()] + while pages[-1]["next_after"] is not None: + assert len(pages[-1]["next_after"]) == 13 + pages.append(page(pages[-1]["next_after"])) + assert len(pages) < 4 + assert tuple(len(item["records"]) for item in pages) == ( + 100, + 5 + len(test_case.expected_legacy_runs), + ) + imported: list[dict[str, Any]] = [] + for item in pages: + imported.extend(item["records"]) + assert {row["run_id"] for row in imported} == set(test_case.expected_legacy_runs) | { + f"external-{index}" for index in range(104) + } + assert len({row["event_id"] for row in imported} - {None}) == 104 + assert sum(row["event_id"] is None for row in imported) == len(test_case.expected_legacy_runs) + assert {row["target_database"] for row in imported} == {None, ""} + first: list[dict[str, Any]] = list(filter(lambda row: row["run_id"] == "external-0", imported)) + assert len(first) == 2 and first[0] == first[1] + assert first[0]["definition"] == "SELECT 'o''rders 🧾' AS order_note" + assert first[0]["metadata_json"] == '{"note":"orders 🧾"}' + assert ( + first[0]["event_id"] + == hashlib.sha256( + b'["sqlbuild.fingerprint.event.v1","model","Orders",null,"main","orders","external-0","version-0"]' + ).hexdigest() + ) + assert datetime.fromisoformat(first[0]["ts"]) == datetime(2026, 2, 1, tzinfo=UTC) + observed_since: str = "2026-02-01T01:00:00.000050+01:00" + recent: list[dict[str, Any]] = [page(observed_since=observed_since)] + while recent[-1]["next_after"] is not None: + recent.append(page(recent[-1]["next_after"], observed_since)) + assert len(recent) < 4 + recent_records: list[dict[str, Any]] = [] + for item in recent: + recent_records.extend(item["records"]) + assert len(recent_records) == 54 + assert {row["run_id"] for row in recent_records} == { + f"external-{index}" for index in range(50, 104) + } + empty_since: str = "2026-02-01T00:00:00.000104Z" + empty: dict[str, Any] = page(observed_since=empty_since) + assert empty["records"] == [] and empty["next_after"] is not None + assert page(empty["next_after"], empty_since) == {"records": [], "next_after": None} + for journal_mode in test_case.journal_modes: + with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: + assert predecessor.execute("PRAGMA journal_mode").fetchone() == (journal_mode,) + assert not (project / ".sqlbuild/control.sqlite3").exists() + (project / "warehouse.db").rename(project / "retained-warehouse.db") + assert page() is None + assert not (project / "warehouse.db").exists() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py new file mode 100644 index 0000000000..12f864c08d --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py @@ -0,0 +1,336 @@ +"""Trusted owner requests cross the real HTTP channel to an isolated DuckDB connection check.""" + +import json +import os +import sqlite3 +import subprocess +import sys +from functools import partial +from pathlib import Path +from typing import Any +from urllib.error import HTTPError +from uuid import uuid4 + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + LocationWorkTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + location_process, + snapshot_api, + wait_for_generation, + wait_for_interpreter, + wait_for_location_health, + wait_for_location_work, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.person.helpers import ( + person_login, + person_read, + person_request, + provision_person, + verify_connection_test_transport, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + LocationWorkTestCase("healthy requested check", "warehouse.db", True), + LocationWorkTestCase("failed requested check", "missing/warehouse.db", False), + ], + ids=lambda case: case.description, +) +def test_given_a_published_location_when_an_owner_requests_work_then_results_return_outbound_and_stale_work_is_not_executed( + test_case: LocationWorkTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + blocked: Path = tmp_path / "block-debug" + child_pid: Path = tmp_path / "debug.pid" + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + f"\n[connections.local]\ndatabase = {json.dumps(test_case.database)}\n" + + '\n[targets."preview #1"]\nschema = "main"\n', + "adapters/slow.py": ( + "import os, sys, time\nfrom pathlib import Path\n" + f"if 'debug' in sys.argv and Path({str(blocked)!r}).exists():\n" + f" Path({str(child_pid)!r}).write_text(str(os.getpid()))\n" + f" while Path({str(blocked)!r}).exists():\n" + " time.sleep(0.02)\n" + ), + }, + ) + executable: str = str(Path(sys.executable).with_name("sqb")) + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + # The serving location must never open the owner's database. + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + owner_environment: dict[str, str] = os.environ | { + "SQLBUILD_CONTROL_DATABASE_URL": str(tmp_path / "control/.sqlbuild/control.sqlite3") + } + arguments: list[str] = [ + executable, + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "preview #1", + ] + request_id: str = str(uuid4()) + enqueue: list[str] = [ + executable, + "admin", + "check-location", + api.location_id, + "--target", + "preview #1", + "--request-id", + request_id, + "--reason", + "Check orders connectivity", + ] + log: Path = tmp_path / "location.log" + publications: Path = tmp_path / "publications.jsonl" + with location_process(arguments, log, publications) as process: + _ = wait_for_generation(api, "preview #1", 1, process, log) + _ = wait_for_location_health( + api, + "preview #1", + {"kind": "connection", "generation": 1, "healthy": test_case.expected_connection}, + process, + log, + ) + first: subprocess.CompletedProcess[str] = subprocess.run( + enqueue, env=owner_environment, capture_output=True, text=True, check=True + ) + assert json.loads(first.stdout) == {"request_id": request_id, "applied": True} + retry: subprocess.CompletedProcess[str] = subprocess.run( + enqueue, env=owner_environment, capture_output=True, text=True, check=True + ) + assert json.loads(retry.stdout) == {"request_id": request_id, "applied": False} + result: dict[str, Any] = wait_for_location_work(api, request_id, process, log) + assert result["generation"] == 1 + assert result["result_json"]["outcome"] == { + "kind": "connection", + "healthy": test_case.expected_connection, + } + read: subprocess.CompletedProcess[str] = subprocess.run( + [executable, "admin", "location-work", request_id], + env=owner_environment, + capture_output=True, + text=True, + check=True, + ) + assert json.loads(read.stdout) == result + process.terminate() + _ = process.wait(timeout=5) + stale_id: str = str(uuid4()) + enqueue[enqueue.index(request_id)] = stale_id + _ = subprocess.run( + enqueue, env=owner_environment, capture_output=True, text=True, check=True + ) + with location_process(arguments, log, publications) as process: + _ = wait_for_generation(api, "preview #1", 2, process, log) + result = wait_for_location_work(api, stale_id, process, log) + assert result["generation"] == 1 + assert result["result_json"]["outcome"] == {"kind": "superseded"} + changed_id: str = str(uuid4()) + enqueue[enqueue.index(stale_id)] = changed_id + blocked.touch() + _ = subprocess.run( + enqueue, env=owner_environment, capture_output=True, text=True, check=True + ) + _ = wait_for_interpreter(process, child_pid, log) + _ = (project / "models/stg_orders.sql").write_text( + TYPE_PROOF_FILES["models/stg_orders.sql"].replace( + "CAST(5 AS INTEGER)", "CAST(17 AS INTEGER)" + ) + ) + blocked.unlink() + result = wait_for_location_work(api, changed_id, process, log) + assert result["generation"] == 2 + assert result["result_json"]["outcome"] == {"kind": "superseded"} + _ = wait_for_generation(api, "preview #1", 3, process, log) + assert [ + json.loads(line)["generation"] for line in publications.read_text().splitlines() + ] == [1, 2, 3] + assert api.bearer not in log.read_text() + publications.read_text() + assert not (project / ".sqlbuild/control.sqlite3").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + LocationWorkTestCase("person healthy connection", "warehouse.db", True), + LocationWorkTestCase("person failed connection", "missing/warehouse.db", False), + ], + ids=lambda case: case.description, +) +def test_given_person_roles_when_requesting_connection_tests_then_live_scope_and_real_results_hold( + test_case: LocationWorkTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + f"\n[connections.local]\ndatabase = {json.dumps(test_case.database)}\n" + + '\n[targets.development]\nschema = "main"\n', + }, + ) + fixture: dict[str, Any] = json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/administrator_lifecycle.json" + ).read_text() + ) + with snapshot_api( + tmp_path / "control", + server_arguments=("--password-origin", "https://control.example.com"), + ) as api: + administrator: dict[str, Any] = provision_person( + tmp_path / "control", "create-user", "orders-admin" + ) + person: dict[str, Any] = provision_person( + tmp_path / "control", "create-principal", "orders-developer" + ) + admin: tuple[str, str] = person_login(api, administrator["name"]) + session: tuple[str, str] = person_login(api, person["name"]) + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + arguments: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ] + log: Path = tmp_path / "location.log" + path: str = f"/api/control/locations/{api.location_id}/targets/development/connection-tests" + with location_process(arguments, log, tmp_path / "publications.jsonl") as process: + _ = wait_for_generation(api, "development", 1, process, log) + request: dict[str, Any] = { + "request_id": str(uuid4()), + "reason": "Check orders connectivity", + } + for operation in [ + partial(person_request, api, path, body=request, session=session), + partial(person_read, api, path + "/" + request["request_id"], session=session), + ]: + with pytest.raises(HTTPError) as denied: + operation() + assert denied.value.code == 403 + for role in fixture["connection_test_roles"][1:3]: + assert role["allowed"] is False + assignment: dict[str, Any] = { + "principal_id": person["principal_id"], + "role": role["role"], + "location_id": api.location_id, + "target": "development", + } + assert person_request(api, "/api/admin/roles", body=assignment, session=admin) == { + "applied": True + } + for operation in [ + partial(person_request, api, path, body=request, session=session), + partial(person_read, api, path + "/" + request["request_id"], session=session), + ]: + with pytest.raises(HTTPError) as denied: + operation() + assert denied.value.code == 403 + assert person_request( + api, "/api/admin/roles", body=assignment | {"revoke": True}, session=admin + ) == {"applied": True} + scopes: dict[str, tuple[str | None, str | None]] = { + "admin": (None, None), + "developer": (api.location_id, "development"), + } + for role in fixture["connection_test_roles"][3:]: + assert role["allowed"] is True + assignment = { + "principal_id": person["principal_id"], + "role": role["role"], + "location_id": scopes[role["role"]][0], + "target": scopes[role["role"]][1], + } + assert person_request(api, "/api/admin/roles", body=assignment, session=admin) == { + "applied": True + } + request = {"request_id": str(uuid4()), "reason": "Check orders connectivity"} + assert person_request(api, path, body=request, session=session) == {"applied": True} + assert person_request(api, path, body=request, session=session) == { + "applied": False + } + result: dict[str, Any] = wait_for_location_work( + api, request["request_id"], process, log + ) + assert ( + person_read(api, path + "/" + request["request_id"], session=session) == result + ) + assert result["result_json"]["outcome"] == { + "kind": "connection", + "healthy": test_case.expected_connection, + } + assert result["requested_by"] == "principal:" + person["principal_id"] + assert result["generation"] == 1 + assert person_request( + api, "/api/admin/roles", body=assignment | {"revoke": True}, session=admin + ) == {"applied": True} + assert person_request(api, "/api/admin/roles", body=assignment, session=admin) == { + "applied": True + } + for other in [ + path.replace("development", "preview"), + path.replace(api.location_id, str(uuid4())), + ]: + with pytest.raises(HTTPError) as denied: + person_request(api, other, body=request, session=session) + assert denied.value.code == 403 + with pytest.raises(HTTPError) as denied: + person_read(api, other + "/" + request["request_id"], session=session) + assert denied.value.code == 403 + assert person_request( + api, "/api/admin/roles", body=assignment | {"revoke": True}, session=admin + ) == {"applied": True} + for operation in [ + partial(person_request, api, path, body=request, session=session), + partial(person_read, api, path + "/" + request["request_id"], session=session), + ]: + with pytest.raises(HTTPError) as denied: + operation() + assert denied.value.code == 403 + verify_connection_test_transport(api, path, request, admin) + audit: list[dict[str, Any]] = person_read( + api, "/api/control/audit?limit=100", session=admin + ) + accepted: list[dict[str, Any]] = list( + filter(lambda event: event["action"] == "location.work_requested", audit) + ) + assert {event["granting_role"] for event in accepted} == {"admin", "developer"} + assert all(event["actor"] == "principal:" + person["principal_id"] for event in accepted) + assert all( + event["scope_json"] == {"location_id": api.location_id, "target": "development"} + for event in accepted + ) + with sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3") as store: + assert store.execute("SELECT count(*) FROM sqb_location_work").fetchone() == (2,) + assert store.execute("SELECT count(*) FROM sqb_runs").fetchone() == (0,) + assert store.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert api.bearer not in log.read_text() + assert not (project / ".sqlbuild/control.sqlite3").exists() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py new file mode 100644 index 0000000000..08a0766e5c --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -0,0 +1,1056 @@ +"""Retained compiler baselines cross the native worker boundary before lifecycle writes.""" + +import json +import os +import signal +import sqlite3 +import time +from dataclasses import replace +from pathlib import Path +from typing import Any +from urllib.request import Request, urlopen +from uuid import uuid4 + +import duckdb +import pytest + +from sqlbuild._native import ( + NativeOrchestrationSnapshot, + build_compiled_project, + build_orchestration_snapshot, + discover_project_inputs, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + METADATA, + TYPE_PROOF_FILES, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + CheckExecutionTestCase, + ClaimedWorker, + GroupExecutionTestCase, + SnapshotWorkerTestCase, + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + cancel_running_statement, + claimed_worker, + scheduled_worker, + snapshot_api, + wait_for_log, + wait_for_run, + worker_result, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + GroupExecutionTestCase( + "group execution preserves independent downstream work", + "WITH source(value) AS (VALUES ('invalid')) SELECT CAST(value AS INTEGER) AS amount FROM source", + "SELECT amount FROM orders", + "partial", + [(11,)], + expected_child_statuses={"staging": ("partial",), "marts": ("partial",)}, + ), + GroupExecutionTestCase( + "successful jobs honor compiled boundaries", + "SELECT 17 AS amount", + "SELECT amount FROM totals", + "success", + [(17,)], + expected_child_statuses={"staging": ("success",), "marts": ("success",)}, + expected_totals=("success", None), + ), + GroupExecutionTestCase( + "a platform sensor starts group parents and children", + "SELECT 17 AS amount", + "SELECT amount FROM totals", + "success", + [(17,)], + expected_origin="sensor", + expected_child_statuses={"staging": ("success",), "marts": ("success",)}, + expected_totals=("success", None), + files={ + "orchestration/orders.py": ( + "from sqlbuild.orchestration import PlatformEventKind, job, job_group, platform_event, sensor\n" + "staging = job('staging', description='Stage orders.', select='stg_orders broken_orders')\n" + "marts = job('marts', description='Build independent marts.', select='orders totals')\n" + "annual_orders = job_group('annual_orders', description='Stage then build marts.', jobs=[marts, staging])\n" + "health_orders = sensor('health_orders', description='Refresh on platform heartbeat.', " + "when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), run=annual_orders, " + "minimum_interval_seconds=60)\n" + ) + }, + ), + GroupExecutionTestCase( + "critical node stops the group under the default policy", + "WITH source(value) AS (VALUES ('invalid')) SELECT CAST(value AS INTEGER) AS amount FROM source", + "SELECT count(*) FROM information_schema.tables WHERE table_name IN ('orders','totals')", + "failed", + [(0,)], + expected_child_statuses={"staging": ("partial", "cancelled"), "marts": ("cancelled",)}, + expected_totals=("cancelled", None), + expected_marts_started=False, + files={ + "models/broken_orders.sql": ( + "MODEL (description 'Critical orders.', materialized table, critical true);\n" + "WITH source(value) AS (VALUES ('invalid')) SELECT CAST(value AS INTEGER) AS amount FROM source\n" + ), + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_failure_policy_hold( + test_case: GroupExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project_dir: Path = tmp_path / "project" + project_dir.mkdir() + with duckdb.connect(str(project_dir / "warehouse.duckdb")) as warehouse: + warehouse.execute("CREATE TABLE broken_orders AS SELECT 99 AS amount") + test_case = replace( + test_case, + files={ + "models/orders.sql": ( + "MODEL (description 'Independent orders.', materialized table);\n" + 'SELECT amount FROM __ref("stg_orders")\n' + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', materialized table);\nSELECT 11 AS amount\n" + ), + "models/broken_orders.sql": ( + "MODEL (description 'Source values.', materialized table);\n" + test_case.model_sql + ), + "models/totals.sql": ( + "MODEL (description 'Dependent totals.', materialized table);\n" + 'SELECT amount FROM __ref("broken_orders")\n' + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, job_group, schedule\n" + 'staging = job("staging", description="Stage orders.", ' + 'select="stg_orders broken_orders")\n' + 'marts = job("marts", description="Build independent marts.", select="orders totals")\n' + 'annual_orders = job_group("annual_orders", description="Stage then build marts.", ' + "jobs=[marts, staging], " + 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ), + } + | test_case.files, + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + parent: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert parent["status"] == test_case.expected_status, worker.log.read_text() + request: Request = Request( + f"{worker.api.url}/api/locations/{worker.api.location_id}/targets/development/runs", + headers={"Authorization": f"Bearer {worker.api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + runs: dict[str, Any] = {run["job_key"]: run for run in json.load(response)} + assert set(runs) == {"annual_orders", *test_case.expected_child_statuses} + assert all( + runs[name]["status"] in statuses + for name, statuses in test_case.expected_child_statuses.items() + ) + assert (runs["marts"]["started_at"] is not None) == test_case.expected_marts_started + assert runs["staging"]["ended_at"] <= ( + runs["marts"]["started_at"] or runs["staging"]["ended_at"] + ) + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT origin FROM sqb_runs WHERE run_id=?", (parent["run_id"],) + ).fetchone() == (test_case.expected_origin,) + assert ( + control.execute( + "SELECT parent_run_id, root_run_id FROM sqb_runs WHERE kind='job' ORDER BY job_key" + ).fetchall() + == [(runs["annual_orders"]["run_id"],) * 2] * 2 + ) + assert ( + control.execute( + "SELECT status, skipped_because_of FROM sqb_run_nodes WHERE run_id=? AND node_key='model:totals'", + (runs["marts"]["run_id"],), + ).fetchone() + == test_case.expected_totals + ) + assert control.execute( + "SELECT status FROM sqb_run_nodes WHERE run_id=? AND node_key='model:broken_orders'", + (runs["staging"]["run_id"],), + ).fetchone() == ({"success": "success"}.get(test_case.expected_status, "failed"),) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute( + "SELECT count(*) FROM sqb_runs WHERE worker_instance IS NOT NULL" + ).fetchone() == (0,) + with duckdb.connect(str(project_dir / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + description="opposing parent and independent child limits", + model_sql="SELECT 11 AS amount", + warehouse_query="SELECT amount FROM orders UNION ALL SELECT amount FROM inventory ORDER BY amount", + expected_status="success", + expected_rows=[(11,), (23,)], + concurrency_limits={"orders_limit": 1, "inventory_limit": 1}, + files={ + "models/inventory.sql": ( + "MODEL (description 'Inventory.', materialized table);\nSELECT 23 AS amount\n" + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, job_group, schedule\n" + "orders = job('orders_job', description='Build orders.', select='orders', " + "concurrency_groups=('inventory_limit',))\n" + "inventory = job('inventory_job', description='Build inventory.', select='inventory', " + "concurrency_groups=('orders_limit',))\n" + "annual_orders = job_group('annual_orders', description='Orders group.', jobs=[orders], " + "concurrency_groups=('orders_limit',), " + "schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "annual_inventory = job_group('annual_inventory', description='Inventory group.', " + "jobs=[inventory], concurrency_groups=('inventory_limit',), " + "schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + ), + }, + store_setup=( + "CREATE TABLE test_slot_claims (run_id TEXT, group_key TEXT, occupied INTEGER);\n" + "CREATE TRIGGER test_observe_slots AFTER INSERT ON sqb_concurrency_slots BEGIN " + "INSERT INTO test_slot_claims SELECT NEW.run_id, NEW.group_key, count(*) " + "FROM sqb_concurrency_slots WHERE group_key=NEW.group_key; END;\n" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_opposing_group_limits_when_scheduled_then_each_group_holds_every_limit_and_both_finish( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + orders: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + inventory: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_inventory" + ) + assert orders["status"] == inventory["status"] == "success", worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT r.job_key, c.group_key, max(c.occupied) FROM test_slot_claims c " + "JOIN sqb_runs r ON r.run_id=c.run_id GROUP BY r.job_key, c.group_key ORDER BY r.job_key" + ).fetchall() == [ + ("annual_inventory", "inventory_limit", 1), + ("annual_inventory", "orders_limit", 1), + ("annual_orders", "inventory_limit", 1), + ("annual_orders", "orders_limit", 1), + ] + assert control.execute( + "SELECT count(*) FROM sqb_runs WHERE status='success'" + ).fetchone() == (4,) + assert control.execute("SELECT count(*) FROM sqb_concurrency_slots").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotWorkerTestCase( + description="unselected dependency drift", + model_file="models/stg_orders.sql", + original_sql="CAST(5 AS INTEGER)", + replacement_sql="CAST(17 AS INTEGER)", + expected_error="node content differs", + ), + SnapshotWorkerTestCase( + description="selected model drift", + model_file="models/orders.sql", + original_sql="SELECT order_id, amount", + replacement_sql="SELECT order_id, amount * CAST(2 AS BIGINT) AS amount", + expected_error="node content differs", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_claim_lifetime_are_native( + test_case: SnapshotWorkerTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = tmp_path / "project" + write_files(project_dir=project_dir, files=TYPE_PROOF_FILES) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + with snapshot_api(tmp_path / "control") as api: + metadata: dict[str, Any] = METADATA | {"location_id": api.location_id} + baseline: NativeOrchestrationSnapshot = build_orchestration_snapshot( + project, json.dumps(metadata), project_dir + ) + publication: dict[str, Any] = json.loads(baseline.publish(api.url, api.bearer)) + claim: ClaimedWorker = claimed_worker(tmp_path / "control", api, publication) + identity: tuple[str, str] = (claim.run_id, claim.claim_id) + changed: dict[str, str] = TYPE_PROOF_FILES | { + test_case.model_file: TYPE_PROOF_FILES[test_case.model_file].replace( + test_case.original_sql, test_case.replacement_sql + ) + } + write_files(project_dir=project_dir, files=changed) + drifted: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + wrong: NativeOrchestrationSnapshot = build_orchestration_snapshot( + drifted, json.dumps(metadata), project_dir + ) + with pytest.raises(RuntimeError, match=test_case.expected_error): + _ = wrong.start_worker(api.url, claim.bearer, identity) + runs_url: str = f"{api.url}/api/locations/{api.location_id}/targets/development/runs" + location_headers: dict[str, str] = {"Authorization": f"Bearer {api.bearer}"} + with urlopen(Request(runs_url, headers=location_headers), timeout=5) as response: + assert json.load(response)[0]["status"] == "queued" + with pytest.raises(RuntimeError, match="does not match"): + _ = baseline.start_worker(api.url, claim.bearer, (claim.run_id, str(uuid4()))) + with baseline.start_worker(api.url, claim.bearer, identity) as worker: + execution: dict[str, Any] = json.loads(worker.execution_json()) + assert execution["claim"]["request"]["selection"] == ["model:orders"] + assert execution["claim"]["request"]["settings"] == {"full_refresh": True} + assert frozenset(execution["node_hashes"]) == { + "model:orders", + "model:stg_orders", + } + assert not worker.cancel_requested() + event_id: str = str(uuid4()) + change: str = json.dumps({"kind": "node_started", "node_key": "model:orders"}) + assert worker.report(event_id, change) + assert not worker.report(event_id, change) + assert claim.bearer not in repr(worker) + worker.close() + assert worker.cancel_requested() + with pytest.raises(RuntimeError, match="closed"): + _ = worker.execution_json() + with pytest.raises(RuntimeError, match="closed"): + _ = worker.report(str(uuid4()), change) + with pytest.raises(RuntimeError, match="closed"), worker: + pytest.fail("closed worker cannot be entered") + # Closing an interpreter-side owner is not proof of physical warehouse termination. + with urlopen(Request(runs_url, headers=location_headers), timeout=5) as response: + assert json.load(response)[0]["status"] == "running" + with baseline.start_worker(api.url, claim.bearer, identity) as resumed: + assert resumed.report( + str(uuid4()), + json.dumps( + { + "kind": "node_finished", + "node_key": "model:orders", + "outcome": "success", + "rows_affected": 7, + "error_code": None, + "skipped_because_of": None, + } + ), + ) + assert resumed.report( + str(uuid4()), + json.dumps( + { + "kind": "finished", + "outcome": "success", + "reason": "worker execution completed", + "exit_code": 0, + } + ), + ) + with urlopen(Request(runs_url, headers=location_headers), timeout=5) as response: + result: dict[str, Any] = json.load(response)[0] + assert result["status"] == "success" + assert result["node_counts_json"] == {"success": 1} + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "successful model writes actual rows", + "WITH source(value) AS (VALUES ('23')) SELECT CAST(value AS BIGINT) AS amount FROM source", + "SELECT amount FROM orders", + "success", + [(23,)], + ), + WorkerExecutionTestCase( + "a platform sensor catches up retained events through a real worker", + "SELECT 47 AS amount", + "SELECT amount FROM orders", + "success", + [(47,)], + expected_origin="sensor", + files={ + "orchestration/orders.py": ( + "from sqlbuild.orchestration import PlatformEventKind, job, platform_event, sensor\n" + "annual_orders = job('annual_orders', select='orders', description='Build orders.')\n" + "health_orders = sensor('health_orders', description='Refresh on platform heartbeat.', " + "when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), run=annual_orders, " + "minimum_interval_seconds=60)\n" + ) + }, + ), + WorkerExecutionTestCase( + "authored variables and reload use ordinary build argument conversion", + "SELECT @@batch AS amount", + "SELECT amount FROM orders", + "success", + [(23,)], + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n[vars]\nbatch = "3"\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n' + '[targets.development]\nschema = "main"\n' + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, schedule\n" + "annual_orders = job('annual_orders', select='orders', description='Build orders.', " + "settings={'vars': {'batch': 23}, 'reload': True, 'run_tests': False}, " + "schedule=schedule(cron='0 0 * * *', timezone='UTC', " + "starts_at='2000-01-01T00:00:00Z'))\n" + ), + }, + ), + WorkerExecutionTestCase( + "warehouse error retains failed node evidence", + "WITH source(value) AS (VALUES ('invalid')) SELECT CAST(value AS BIGINT) AS amount FROM source", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "failed", + [(0,)], + ), + WorkerExecutionTestCase( + "scalar function dependency is claimed and executed", + 'SELECT __udf("order_total")(23) AS amount', + "SELECT amount FROM orders", + "success", + [(46,)], + expected_node_count=2, + files={ + "functions/sql/order_total.sql": ( + "FUNCTION (description 'Order total.', " + "arguments (amount BIGINT), returns BIGINT);\nSELECT amount * 2\n" + ) + }, + ), + WorkerExecutionTestCase( + "table function dependency is claimed and executed", + 'SELECT amount FROM __table_fn("order_amounts")()', + "SELECT amount FROM orders", + "success", + [(31,)], + expected_node_count=2, + files={ + "functions/sql/order_amounts.sql": ( + "FUNCTION (description 'Order amounts.', returns table (amount BIGINT));\n" + "SELECT CAST(31 AS BIGINT) AS amount\n" + ) + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_real_warehouse_results( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "unchanged attached Python check", + "SELECT 17 AS amount", + "SELECT amount FROM orders", + "success", + [(17,)], + files={"python/checks/orders.py": "initial"}, + ), + WorkerExecutionTestCase( + "attached Python check changed after publication", + "SELECT 17 AS amount", + "SELECT amount FROM orders", + "failed", + [], + files={"python/checks/orders.py": "initial"}, + files_after_publication={"python/checks/orders.py": "changed"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_attached_python_code_when_a_worker_starts_then_publication_pins_it_before_execution( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + fixture: dict[str, str] = json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/python_execution_inputs.json" + ).read_text() + ) + test_case = replace( + test_case, + files={path: fixture[key] for path, key in test_case.files.items()}, + files_after_publication={ + path: fixture[key] for path, key in test_case.files_after_publication.items() + }, + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = wait_for_run(worker.api, worker.process, worker.log) + assert run["status"] == test_case.expected_status, worker.log.read_text() + runs_url: str = ( + f"{worker.api.url}/api/locations/{worker.api.location_id}/targets/development/runs" + ) + with urlopen( + Request( + f"{runs_url}/{run['run_id']}/nodes", + headers={"Authorization": f"Bearer {worker.api.bearer}"}, + ), + timeout=5, + ) as response: + nodes: list[dict[str, Any]] = json.load(response) + assert [(node["node_key"], node["started_at"] is not None) for node in nodes] == [ + ("model:orders", test_case.expected_status == "success") + ] + assert (worker.project / "warehouse.duckdb").exists() == ( + test_case.expected_status == "success" + ) + + +@pytest.mark.parametrize( + "test_case", + [ + CheckExecutionTestCase( + "passed", "SELECT 17 AS amount", "SELECT amount FROM orders", "success", [(17,)] + ), + CheckExecutionTestCase( + "warned", + "SELECT 17 AS amount", + "SELECT amount FROM orders", + "success", + [(17,)], + expected_check_outcome="warn", + ), + CheckExecutionTestCase( + "failed", + "SELECT 17 AS amount", + "SELECT amount FROM orders", + "failed", + [(17,)], + expected_check_outcome="failed", + ), + ], + ids=lambda case: case.description, +) +def test_given_an_attached_check_when_a_worker_executes_then_its_own_outcome_is_durable( + test_case: CheckExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + fixture: dict[str, str] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/attached_checks.json").read_text() + ) + test_case = replace( + test_case, files={"python/checks/orders.py": fixture[test_case.description]} + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = wait_for_run(worker.api, worker.process, worker.log) + assert run["status"] == test_case.expected_status, worker.log.read_text() + assert run["selection_json"] == ["model:orders"] + assert run["node_counts_json"] == {"success": 1} + with sqlite3.connect(worker.control_store) as control: + events: list[tuple[str, str]] = control.execute( + "SELECT event_type,payload FROM recall_events WHERE run_id=? " + "AND event_type IN ('run_check_started','run_check_finished') " + "ORDER BY storage_id", + (run["run_id"],), + ).fetchall() + assert [(kind, json.loads(payload)["change"]["change"]) for kind, payload in events] == [ + ("run_check_started", {"kind": "check_started", "node_key": "check:orders_valid"}), + ( + "run_check_finished", + { + "kind": "check_finished", + "node_key": "check:orders_valid", + "outcome": test_case.expected_check_outcome, + }, + ), + ] + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "actual attached check failure starts a sensor destination", + "SELECT 17 AS amount", + "SELECT amount FROM inventory", + "success", + [(59,)], + files={ + "python/checks/orders.py": "failed", + "orchestration/orders.py": "on_failure", + "models/inventory.sql": "inventory", + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_attached_check_failure_when_a_sensor_observes_it_then_its_destination_executes( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + fixture: dict[str, str] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/attached_checks.json").read_text() + ) + test_case = replace( + test_case, files={path: fixture[key] for path, key in test_case.files.items()} + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + source: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert source["status"] == "failed", worker.log.read_text() + destination: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="refresh_inventory" + ) + assert destination["status"] == test_case.expected_status, worker.log.read_text() + assert destination["selection_json"] == ["model:inventory"] + assert destination["snapshot_id"] == source["snapshot_id"] + assert source["selection_json"] == ["model:orders"] + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "cancelling active attached-check SQL cannot set failure health", + "SELECT 17 AS amount", + "SELECT amount FROM orders", + "cancelled", + [(17,)], + files={ + "python/checks/orders.py": "cancelled", + "orchestration/orders.py": "on_failure", + "models/inventory.sql": "inventory", + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_attached_check_query_when_cancelled_then_it_cannot_trigger_failure_health( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + fixture: dict[str, str] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/attached_checks.json").read_text() + ) + test_case = replace( + test_case, files={path: fixture[key] for path, key in test_case.files.items()} + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + cancel_running_statement(worker, job_key="annual_orders") + run: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert run["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + outcomes: list[tuple[str]] = control.execute( + "SELECT payload -> 'change' -> 'change' ->> 'outcome' " + "FROM recall_events WHERE run_id=? AND event_type='run_check_finished'", + (run["run_id"],), + ).fetchall() + assert outcomes == [("cancelled",)] + completed_query: str = ( + "SELECT count(*) FROM sqb_sensor_evaluation_events WHERE phase='completed'" + ) + # Two more completions mean at least one evaluation started after the outcome. + target: int = control.execute(completed_query).fetchone()[0] + 2 + deadline: float = time.monotonic() + 15 + evaluated: int = 0 + while evaluated < target: + assert worker.process.poll() is None, worker.log.read_text() + assert time.monotonic() < deadline, worker.log.read_text() + evaluated = control.execute(completed_query).fetchone()[0] + time.sleep(0.02) + assert control.execute("SELECT count(*) FROM sqb_run_requests").fetchone() == (0,) + assert control.execute( + "SELECT count(*) FROM sqb_runs WHERE job_key='refresh_inventory'" + ).fetchone() == (0,) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "group completion starts a real sensor worker", + "SELECT 59 AS amount", + "SELECT amount FROM orders", + "success", + [(59,)], + expected_origin="sensor", + files={ + "orchestration/orders.py": ( + "from sqlbuild.orchestration import RunEventKind, job, job_group, run_event, schedule, sensor\n" + "source = job('source_orders', select='orders', description='Source orders.')\n" + "source_group = job_group('source_group', description='Source group.', jobs=[source], " + "schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "annual_orders = job('annual_orders', select='orders', description='Refresh orders.', settings={'reload': True})\n" + "on_group = sensor('on_group', description='Refresh after group completion.', " + "when=run_event(job=source_group, kinds=[RunEventKind.COMPLETED, RunEventKind.SUCCESS], executed_successfully=('orders',)), " + "run=annual_orders, minimum_interval_seconds=1)\n" + ) + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_real_group_completion_when_a_run_sensor_matches_then_one_destination_worker_executes( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + destination: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert destination["status"] == test_case.expected_status, worker.log.read_text() + result: dict[str, Any] = worker_result(worker, test_case) + assert result["run_id"] == destination["run_id"] + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT run_key,run_id FROM sqb_run_requests").fetchall() == [ + ( + control.execute( + "SELECT run_id FROM sqb_runs WHERE job_key='source_group'" + ).fetchone()[0], + destination["run_id"], + ) + ] + assert control.execute( + "SELECT job_key,status FROM sqb_runs ORDER BY job_key" + ).fetchall() == [ + ("annual_orders", "success"), + ("source_group", "success"), + ("source_orders", "success"), + ] + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "group child node sensor", + "SELECT 67 AS amount", + "SELECT amount FROM orders", + "success", + [(67,)], + expected_origin="sensor", + files={ + "models/stg_orders.sql": "MODEL (description 'Staging orders.');\nSELECT 71 AS amount\n", + "orchestration/orders.py": ( + "from sqlbuild.orchestration import NodeEventKind, job, job_group, node_event, schedule, sensor\n" + "source = job('source_orders', select='orders', description='Source orders.')\n" + "source_group = job_group('source_group', description='Source group.', jobs=[source], " + "schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "annual_orders = job('annual_orders', select='stg_orders', description='Refresh staging.', settings={'reload': True})\n" + "on_node = sensor('on_node', description='React to actual orders rebuilds.', " + "when=node_event(select='orders', kinds=[NodeEventKind.SUCCESS]), " + "run=annual_orders, minimum_interval_seconds=1)\n" + ), + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_real_group_child_rebuild_when_a_node_sensor_matches_then_only_the_child_finish_queues_a_worker( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + destination: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert destination["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT job_key,status FROM sqb_runs ORDER BY job_key" + ).fetchall() == [ + ("annual_orders", "success"), + ("source_group", "success"), + ("source_orders", "success"), + ] + assert control.execute( + "SELECT q.run_key,q.run_id FROM sqb_run_requests q" + ).fetchall() == [ + ( + control.execute( + "SELECT run_id FROM sqb_runs WHERE job_key='source_orders'" + ).fetchone()[0] + + "/model:orders", + destination["run_id"], + ) + ] + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "cancellation stops an active warehouse statement", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "cancelled", + [(0,)], + ), + ], + ids=lambda case: case.description, +) +def test_given_an_executing_scheduled_worker_when_cancelled_then_the_statement_stops_before_claim_release( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + cancel_running_statement(worker) + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "parent cancellation interrupts its active child and cancels its waiting child", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name IN ('orders','totals')", + "cancelled", + [(0,)], + files={ + "models/totals.sql": ( + "MODEL (description 'Downstream totals.', materialized table);\n" + 'SELECT amount FROM __ref("orders")\n' + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, job_group, schedule\n" + "orders = job('orders', description='Long query.', select='orders')\n" + "totals = job('totals', description='Waiting query.', select='totals')\n" + "annual_orders = job_group('annual_orders', description='Cancellable group.', " + "jobs=[totals, orders], schedule=schedule(cron='0 0 * * *', timezone='UTC', " + "starts_at='2000-01-01T00:00:00Z'))\n" + ), + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_executing_group_when_parent_cancelled_then_all_children_stop_before_parent_completion( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + parent: dict[str, Any] = wait_for_run( + worker.api, + worker.process, + worker.log, + pending=frozenset({"queued"}), + job_key="annual_orders", + ) + assert parent["status"] == "running", worker.log.read_text() + cancel_running_statement(worker, job_key="orders", run_id=parent["run_id"]) + completed: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert completed["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT job_key, status FROM sqb_runs ORDER BY job_key" + ).fetchall() == [ + ("annual_orders", "cancelled"), + ("orders", "cancelled"), + ("totals", "cancelled"), + ] + assert control.execute( + "SELECT count(*) FROM sqb_runs WHERE parent_run_id=? AND ended_at>?", + (parent["run_id"], completed["ended_at"]), + ).fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute( + "SELECT count(*) FROM sqb_runs WHERE worker_instance IS NOT NULL" + ).fetchone() == (0,) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "credential failure after a committed claim", + "SELECT 23 AS amount", + "", + "failed", + [], + store_setup=( + "CREATE TRIGGER reject_run_token BEFORE INSERT ON sqb_access_tokens " + "WHEN NEW.run_id IS NOT NULL " + "BEGIN SELECT RAISE(ABORT, 'synthetic run credential failure'); END;" + ), + ), + WorkerExecutionTestCase( + "launch publication failure after a committed claim", + "SELECT 23 AS amount", + "", + "failed", + [], + store_setup=( + "CREATE TRIGGER reject_launch BEFORE UPDATE OF status ON sqb_runs " + "WHEN NEW.status = 'launched' " + "BEGIN SELECT RAISE(ABORT, 'synthetic launch publication failure'); END;" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_committed_claim_when_launch_preparation_fails_then_unstarted_work_is_terminal_and_unreserved( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + assert worker.process.wait(timeout=15) != 0, worker.log.read_text() + request: Request = Request( + f"{worker.api.url}/api/locations/{worker.api.location_id}/targets/development/runs", + headers={"Authorization": f"Bearer {worker.api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + runs: list[dict[str, Any]] = json.load(response) + assert len(runs) == 1 + assert runs[0]["status"] == test_case.expected_status, worker.log.read_text() + assert runs[0]["node_counts_json"] == {"skipped": 1} + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_concurrency_slots").fetchone() == (0,) + assert control.execute( + "SELECT count(*) FROM sqb_runs WHERE worker_instance IS NOT NULL" + ).fetchone() == (0,) + assert not (worker.project / "warehouse.duckdb").exists() + assert "Worker launched for run" not in worker.log.read_text() + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "failed terminal publication keeps the owner alive until recovery", + "SELECT 23 AS amount", + "", + "failed", + [], + store_setup=( + "CREATE TRIGGER reject_run_token BEFORE INSERT ON sqb_access_tokens " + "WHEN NEW.run_id IS NOT NULL " + "BEGIN SELECT RAISE(ABORT, 'synthetic run credential failure'); END;" + "CREATE TRIGGER reject_terminal BEFORE UPDATE OF status ON sqb_runs " + "WHEN NEW.status = 'failed' " + "BEGIN SELECT RAISE(ABORT, 'synthetic terminal publication failure'); END;" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_a_stopped_worker_when_terminal_publication_fails_then_shutdown_retains_and_retries_its_claim( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + wait_for_log(worker.process, worker.log, "Retrying worker shutdown publication") + assert worker.process.poll() is None, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (1,) + assert control.execute( + "SELECT count(*) FROM sqb_runs WHERE worker_instance IS NOT NULL" + ).fetchone() == (1,) + _ = control.execute("DROP TRIGGER reject_terminal") + assert worker.process.wait(timeout=10) != 0, worker.log.read_text() + request: Request = Request( + f"{worker.api.url}/api/locations/{worker.api.location_id}/targets/development/runs", + headers={"Authorization": f"Bearer {worker.api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + runs: list[dict[str, Any]] = json.load(response) + assert len(runs) == 1 + assert runs[0]["status"] == test_case.expected_status + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute( + "SELECT count(*) FROM sqb_runs WHERE worker_instance IS NOT NULL" + ).fetchone() == (0,) + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_terminated'" + ).fetchone() == (1,) + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX TERM/KILL escalation") +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "uncooperative child is reaped after a lease-preserving hard kill", + "SELECT 23 AS amount", + "", + "failed", + [], + launcher="uncooperative", + ) + ], + ids=lambda case: case.description, +) +def test_given_an_uncooperative_worker_when_scheduler_stops_then_the_fence_and_claims_survive_until_reaping( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + pidfile: Path = worker.project / "worker.pid" + deadline: float = time.monotonic() + 15 + while not pidfile.exists(): + assert worker.process.poll() is None, worker.log.read_text() + assert time.monotonic() < deadline, worker.log.read_text() + time.sleep(0.02) + pid: int = int(pidfile.read_text()) + worker.process.send_signal(signal.SIGTERM) + time.sleep(32) + assert worker.process.poll() is None, worker.log.read_text() + os.kill(pid, 0) + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (1,) + assert control.execute( + "SELECT julianday(expires_at) > julianday('now') FROM sqb_scheduler_lease" + ).fetchone() == (1,) + assert worker.process.wait(timeout=40) == 0, worker.log.read_text() + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + request: Request = Request( + f"{worker.api.url}/api/locations/{worker.api.location_id}/targets/development/runs", + headers={"Authorization": f"Bearer {worker.api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + runs: list[dict[str, Any]] = json.load(response) + assert len(runs) == 1 + assert runs[0]["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute( + "SELECT count(*) FROM sqb_runs WHERE worker_instance IS NOT NULL" + ).fetchone() == (0,) + assert control.execute( + "SELECT json_extract(payload, '$.change.change.exit_code') FROM recall_events " + "WHERE event_type='run_terminated'" + ).fetchone() == (137,) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/api/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/api/helpers.py new file mode 100644 index 0000000000..020788e0e5 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/api/helpers.py @@ -0,0 +1,44 @@ +"""Exercise installed-server browser sessions over a disposable local HTTP transport.""" + +from __future__ import annotations + +import json +from email.message import Message +from http.client import HTTPResponse +from http.cookies import SimpleCookie +from typing import Any +from urllib.error import HTTPError +from urllib.request import Request, urlopen + + +def request( + endpoint: str, body: bytes | None = None, session: tuple[str, str] = ("", "") +) -> tuple[int | None, Message[str, str], dict[str, Any]]: + """Return denials as ordinary results; never print credentials or response payloads.""" + headers: dict[str, str] = { + "Host": "control.example.com", + "Origin": "https://control.example.com", + "Content-Type": "application/json", + "Cookie": session[0], + "x-sqb-csrf": session[1], + } + call: Request = Request(endpoint, data=body, headers=headers) + response: HTTPResponse | HTTPError + try: + response = urlopen(call, timeout=5) + except HTTPError as error: + response = error + with response: + return response.status, response.headers, json.load(response) + + +def login(url: str, username: str, password: str) -> tuple[str, str]: + """Authenticate with the real Argon2/password route and retain its host-only cookie.""" + status, headers, body = request( + f"{url}/api/auth/password/login", + json.dumps({"username": username, "password": password}).encode("utf-8"), + ) + assert status == 200, f"Password sign-in failed with HTTP {status}" + cookies: SimpleCookie[str] = SimpleCookie("\n".join(headers.get_all("Set-Cookie", []))) + cookie: str = "__Host-sqb-session=" + cookies["__Host-sqb-session"].value + return cookie, body["csrf_token"] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/cli/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/cli/helpers.py new file mode 100644 index 0000000000..45c0911a60 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/cli/helpers.py @@ -0,0 +1,88 @@ +"""Start and reap an installed native executable in a disposable installation.""" + +import json +import os +import subprocess +import time +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Any + + +def environment(extra: dict[str, str] | None = None) -> dict[str, str]: + """Never inherit a real control store into a disposable acceptance run.""" + values: dict[str, str] = os.environ.copy() + values.pop("SQLBUILD_CONTROL_DATABASE_URL", None) + return values | (extra or {}) + + +def administer( + installation: tuple[Path, Path], + arguments: list[str], + password: str | None = None, + *, + extra_environment: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + """Password input remains on stdin and is never included in process arguments or logs.""" + binary, root = installation + return subprocess.run( + [str(binary), "admin", *arguments], + cwd=root, + env=environment(extra_environment), + input=password, + text=True, + encoding="utf-8", + capture_output=True, + timeout=20, + check=False, + ) + + +def successful(output: subprocess.CompletedProcess[str]) -> dict[str, Any]: + """Parse machine stdout only after the native command completed successfully.""" + assert output.returncode == 0, f"Native administration failed: {output.stderr}" + return json.loads(output.stdout) + + +@contextmanager +def serve( + installation: tuple[Path, Path], + *, + extra_environment: dict[str, str] | None = None, + authentication_arguments: tuple[str, ...] = ( + "--password-origin", + "https://control.example.com", + ), +) -> Iterator[str]: + """Kill and reap the server before returning, on Windows as well as Unix.""" + binary, root = installation + log: Path = root / "server.log" + with log.open("w", encoding="utf-8") as stderr: + process: subprocess.Popen[bytes] = subprocess.Popen( + [ + str(binary), + "server", + "--port", + "0", + "--no-scheduler", + *authentication_arguments, + ], + cwd=root, + env=environment(extra_environment), + stdout=subprocess.DEVNULL, + stderr=stderr, + ) + try: + deadline: float = time.monotonic() + 10 + while "Control API ready: " not in log.read_text(encoding="utf-8"): + assert process.poll() is None, log.read_text(encoding="utf-8") + assert time.monotonic() < deadline, log.read_text(encoding="utf-8") + time.sleep(0.02) + address: str = ( + log.read_text(encoding="utf-8").split("Control API ready: ", 1)[1].splitlines()[0] + ) + yield f"http://{address}" + finally: + process.kill() + process.wait(timeout=10) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/helpers.py new file mode 100644 index 0000000000..fa3dda2513 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/wheel/helpers.py @@ -0,0 +1,114 @@ +"""Bootstrap, authenticate, lose authority, restore it with grant-admin and authenticate again.""" + +import hashlib +import json +import sqlite3 +import subprocess +import sys +from contextlib import closing +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.api.helpers import ( + login, + request, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.cli.helpers import ( + administer, + serve, + successful, +) + + +def logical_table_state(connection: Any, tables: list[str]) -> dict[str, str]: + """Hash complete logical rows and multiplicity without exposing stored credentials.""" + state: dict[str, str] = {} + for table in tables: + rows: list[str] = [ + json.dumps(row, sort_keys=True, default=str) + for row in connection.execute(f'SELECT * FROM "{table}"').fetchall() + ] + state[table] = hashlib.sha256(json.dumps(sorted(rows)).encode()).hexdigest() + connection.commit() + return state + + +def postgres_recovery_state(connection: Any) -> dict[str, str]: + """Compare every retained security row without logging credential hashes.""" + tables: tuple[str, ...] = ( + "sqb_principals", + "sqb_role_assignments", + "sqb_password_credentials", + "sqb_sessions", + "sqb_security_audit", + "sqb_installation", + ) + return { + table: hashlib.sha256( + json.dumps( + connection.execute( + f"SELECT to_jsonb(state) FROM {table} AS state ORDER BY to_jsonb(state)::text" + ).fetchall(), + sort_keys=True, + ).encode() + ).hexdigest() + for table in tables + } + + +def verify_recovery(binary: Path) -> str: + """Exercise the wheel's native executable using only the Python standard library.""" + binary = binary.resolve(strict=True) + print("Starting installed-artifact administrator recovery acceptance...", flush=True) + with TemporaryDirectory(prefix="sqlbuild-wheel-recovery-") as temporary: + root: Path = Path(temporary) + installation: tuple[Path, Path] = binary, root + person: dict[str, Any] = successful( + administer(installation, ["create-user", "orders-admin"]) + ) + principal: str = person["principal_id"] + password = "orders-recovery-password-2026" + successful(administer(installation, ["set-password", principal], password)) + database: Path = root / ".sqlbuild/control.sqlite3" + with serve(installation) as url: + previous: tuple[str, str] = login(url, person["name"], password) + with closing(sqlite3.connect(database)) as control: + control.execute("UPDATE sqb_principals SET disabled_at='2026-01-01T00:00:00Z'") + control.commit() + rejected: subprocess.CompletedProcess[str] = administer( + installation, ["create-user", "replacement-admin"] + ) + assert rejected.returncode != 0 and not rejected.stdout + assert "bootstrap is closed" in rejected.stderr + result: dict[str, Any] = successful( + administer(installation, ["grant-admin", principal]) + ) + assert result == {"principal_id": principal, "role": "admin"} + with closing(sqlite3.connect(database)) as control: + assert control.execute( + "SELECT COUNT(*) FROM sqb_security_audit WHERE action='admin.granted'" + ).fetchone() == (1,) + assert control.execute( + "SELECT COUNT(*) FROM sqb_sessions WHERE revoked_at IS NULL" + ).fetchone() == (0,) + print( + "grant-admin committed one audit while serving; bootstrap stayed closed; " + "sessions revoked.", + flush=True, + ) + with serve(installation) as url: + assert request(f"{url}/api/auth/session", session=previous)[0] == 401 + session: tuple[str, str] = login(url, person["name"], password) + assert request(f"{url}/api/auth/session", session=session)[0] == 200 + status, _, body = request(f"{url}/api/admin/people/{principal}", b'"revoke"', session) + assert status == 403 + assert "cannot remove or disable the final installation administrator" in body["error"] + rejected = administer(installation, ["create-user", "replacement-admin"]) + assert rejected.returncode != 0 and "bootstrap is closed" in rejected.stderr + print("Installed-artifact administrator recovery acceptance passed.", flush=True) + return body["error"] + + +if __name__ == "__main__": + verify_recovery(Path(sys.argv[1])) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py new file mode 100644 index 0000000000..7437f8ca6a --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -0,0 +1,1247 @@ +"""Real compilation produces immutable, relocatable control-plane catalog data.""" + +from __future__ import annotations + +import json +from dataclasses import replace +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild._native import ( + NativeOrchestrationSnapshot, + PythonNodeDiscoveryError, + build_compiled_project, + build_orchestration_snapshot, + discover_project_inputs, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.orchestration import ( + RunEventKind, + compile_snapshot, + job, + job_group, + run_event, + schedule, + sensor, +) +from sqlbuild.orchestration_declarations.models.authoring import Job +from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + SnapshotCatalogTestCase, + SnapshotDeclarationRejectionTestCase, + SnapshotDiscoveryRejectionTestCase, + SnapshotDiscoveryTestCase, + SnapshotEnvironmentTestCase, + SnapshotIdentityTestCase, + SnapshotRejectionTestCase, + SnapshotScheduleTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + FUNCTION_PROJECT_FILES, + METADATA, + PARITY_FILES, + TYPE_PROOF_FILES, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotCatalogTestCase( + "model and seed catalog", + TYPE_PROOF_FILES + | { + "seeds/payment_methods.csv": PARITY_FILES["seeds/payment_methods.csv"], + "seeds/payment_methods.yml": PARITY_FILES["seeds/payment_methods.yml"], + "models/payments.sql": ( + "MODEL (description 'Payment methods.');\n" + 'SELECT method FROM __seed("payment_methods")\n' + ), + }, + frozenset({"model", "seed"}), + ), + SnapshotCatalogTestCase( + "attached audits", + TYPE_PROOF_FILES + | { + "models/orders.sql": ( + "MODEL (description 'Orders.', columns (order_id (audits [not_null])));\n" + 'SELECT order_id, amount FROM __ref("stg_orders")\n' + ), + }, + frozenset({"model", "audit"}), + ), + SnapshotCatalogTestCase( + "SQL and Python functions", + { + path: value + for path, value in FUNCTION_PROJECT_FILES.items() + if path != "enums/status.sql" + } + | {"functions/sql/_enums/status.sql": FUNCTION_PROJECT_FILES["enums/status.sql"]}, + frozenset({"model", "udf", "table_fn"}), + ), + SnapshotCatalogTestCase( + "Python hooks", + TYPE_PROOF_FILES + | { + "hooks/python/notifications.py": ( + "from sqlbuild.hooks import hook\n\n" + "@hook\ndef notify(ctx, channel):\n" + ' """Notify support."""\n return None\n' + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', pre_hooks [python('notify', channel: 'support')]);\n" + 'SELECT order_id, amount FROM __ref("stg_orders")\n' + ), + }, + frozenset({"model", "hook"}), + ), + SnapshotCatalogTestCase( + "DuckShop Python execution inputs", + TYPE_PROOF_FILES + | { + f"python/{kind}s/orders.py": json.loads( + Path( + "benchmarks/duckshop/edge_cases/orchestration/python_execution_inputs.json" + ).read_text() + )[source] + for kind, source in (("task", "task"), ("asset", "asset"), ("check", "initial")) + }, + frozenset({"model", "task", "asset", "check"}), + ), + SnapshotCatalogTestCase( + "SQL hooks", + TYPE_PROOF_FILES + | { + "hooks/sql/record_load.sql": "HOOK (description 'Record load.');\nSELECT 3 AS load_count\n", + "models/orders.sql": ( + "MODEL (description 'Orders.', pre_hooks [sql('record_load')]);\n" + 'SELECT order_id FROM __ref("stg_orders")\n' + ), + }, + frozenset({"model", "sql_hook"}), + ), + SnapshotCatalogTestCase( + "source catalog", + TYPE_PROOF_FILES + | { + "sources/orders.yml": ( + "sources:\n - name: raw_orders\n description: Raw orders.\n" + " schema: raw\n table: orders\n" + " columns:\n - name: order_id\n type: INTEGER\n" + ), + }, + frozenset({"model", "source"}), + ), + SnapshotCatalogTestCase( + "DuckShop polling callback with nested code is relocatable", + TYPE_PROOF_FILES + | { + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/custom_orders.py" + ).read_text(), + }, + frozenset({"model"}), + ), + SnapshotCatalogTestCase( + "DuckShop inherited seed and function namespaces", + TYPE_PROOF_FILES + | { + path: (Path("benchmarks/duckshop/project") / path).read_text() + for path in ( + "seeds/edge_cases/edge_quantity_steps.csv", + "seeds/edge_cases/edge_quantity_steps.yml", + "functions/sql/edge_cases/edge_next_quantity.sql", + ) + }, + frozenset({"model", "seed", "udf"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_real_projects_when_building_snapshots_then_catalogs_are_stable_and_relocatable( + test_case: SnapshotCatalogTestCase, + tmp_path: Path, +) -> None: + publications: list[dict[str, Any]] = [] + metadata: str = json.dumps(METADATA) + for directory in (tmp_path / "original", tmp_path / "relocated"): + write_files(project_dir=directory, files=test_case.files) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=directory), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + assert not any(item.severity == "error" for item in project.diagnostics) + snapshot: NativeOrchestrationSnapshot = build_orchestration_snapshot( + project, metadata, directory + ) + publication: dict[str, Any] = json.loads(snapshot.to_json()) + publications.append(publication) + assert publication["sqb_version"] != "not-the-installed-version" + assert publication["environment_hash"] != "test-environment" + assert test_case.expected_kinds <= {node["kind"] for node in publication["nodes"].values()} + assert str(directory) not in snapshot.to_json() + for model in project.models: + node: dict[str, Any] = publication["nodes"][f"model:{model.name}"] + assert node["compiled_sql"] == model.query_sql + assert ( + node["definition"]["destination"]["qualified_name"] + == model.destination.qualified_name + ) + assert set(node["definition"]["deps"]) == { + f"{dependency.resource_type}:{dependency.name}" for dependency in model.deps + } + for function in project.functions: + assert ( + publication["nodes"][f"{function.key.resource_type}:{function.name}"]["definition"][ + "destination" + ]["qualified_name"] + == function.destination.qualified_name + ) + for seed in project.seeds: + assert ( + publication["nodes"][f"seed:{seed.name}"]["definition"]["destination"][ + "qualified_name" + ] + == seed.destination.qualified_name + ) + assert publications[0] == publications[1] + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotIdentityTestCase(**case) + for case in json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/semantic_identity.json").read_text() + )["cases"] + ], + ids=lambda case: case.description, +) +def test_given_semantic_edits_when_publishing_then_local_identities_keep_planner_boundaries( + test_case: SnapshotIdentityTestCase, tmp_path: Path +) -> None: + fixture: dict[str, Any] = json.loads( + Path("benchmarks/duckshop/edge_cases/orchestration/semantic_identity.json").read_text() + ) + files: dict[str, str] = ( + TYPE_PROOF_FILES + | fixture["files"] + | { + path: (Path("benchmarks/duckshop/project") / path).read_text() + for path in ( + "seeds/edge_cases/edge_quantity_steps.csv", + "seeds/edge_cases/edge_quantity_steps.yml", + "functions/sql/edge_cases/edge_next_quantity.sql", + ) + } + ) + original: str = files[test_case.path] + assert test_case.before in original + changed: dict[str, str] = files | { + test_case.path: original.replace(test_case.before, test_case.after) + } + identities: list[dict[str, str]] = [] + for label, inputs in (("original", files), ("changed", changed)): + directory: Path = tmp_path / label + write_files(project_dir=directory, files=inputs) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=directory), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + assert not any(item.severity == "error" for item in project.diagnostics) + snapshot: dict[str, Any] = json.loads( + build_orchestration_snapshot(project, json.dumps(METADATA), directory).to_json() + ) + hashes: dict[str, str] = {} + for key in ( + "model:orders", + "model:stg_orders", + "udf:edge_next_quantity", + "seed:edge_quantity_steps", + ): + identity: dict[str, str] = snapshot["nodes"][key]["definition"]["planner_identity"] + assert set(identity) == {"local_hash"} + assert len(identity["local_hash"]) == 64 + hashes[key] = identity["local_hash"] + identities.append(hashes) + assert {key: identities[0][key] != identities[1][key] for key in identities[0]} == { + key: key in test_case.expected_changed for key in identities[0] + } + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotRejectionTestCase( + description="a valid compile cannot publish caller-forged nodes", + files=TYPE_PROOF_FILES, + nodes={ + "model:forged": { + "kind": "model", + "definition": {}, + "raw_sql": None, + "compiled_sql": "select 2", + "python_source": None, + } + }, + expected_compiler_errors=False, + expected_message="nodes must come from the compiled project", + ), + SnapshotRejectionTestCase( + description="compiler errors cannot replace the last good publication", + files=TYPE_PROOF_FILES | {"models/orders.sql": "MODEL ();\nSELECT 1 AS order_id\n"}, + nodes={}, + expected_compiler_errors=True, + expected_message="cannot publish a project with compiler errors", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_compilation_or_forged_nodes_when_publishing_then_no_snapshot_is_emitted( + test_case: SnapshotRejectionTestCase, + tmp_path: Path, +) -> None: + write_files(project_dir=tmp_path, files=test_case.files) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + assert ( + any(item.severity == "error" for item in project.diagnostics) + == test_case.expected_compiler_errors + ) + with pytest.raises(ValueError, match=test_case.expected_message): + build_orchestration_snapshot( + project, + json.dumps(METADATA | {"nodes": test_case.nodes}), + tmp_path, + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotEnvironmentTestCase( + description="installed dependencies and two lock revisions identify different environments", + first_lockfile="version = 1\n# dependency revision one\n", + second_lockfile="version = 1\n# dependency revision two\n", + expected_environment_count=3, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_dependency_lock_when_it_changes_then_environment_identity_changes_not_node_content( + test_case: SnapshotEnvironmentTestCase, + tmp_path: Path, +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + installed: dict[str, Any] = json.loads( + build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + ) + (tmp_path / "uv.lock").write_text(test_case.first_lockfile) + locked: dict[str, Any] = json.loads( + build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + ) + (tmp_path / "uv.lock").write_text(test_case.second_lockfile) + changed: dict[str, Any] = json.loads( + build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + ) + assert ( + len( + {installed["environment_hash"], locked["environment_hash"], changed["environment_hash"]} + ) + == test_case.expected_environment_count + ) + assert installed["nodes"] == locked["nodes"] == changed["nodes"] + assert changed == json.loads( + build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotScheduleTestCase( + description="daylight saving zone", + schedule=schedule(cron="15 2 * * *", timezone="Europe/London"), + expected_selection=("model:orders",), + ), + SnapshotScheduleTestCase( + description="fixed offset zone", + schedule=schedule(cron="15 2 * * *", timezone="UTC"), + expected_selection=("model:orders",), + ), + SnapshotScheduleTestCase( + description="paused with a zoned start", + schedule=schedule( + cron="15 2 * * *", + timezone="Europe/London", + starts_at="2026-07-03T01:17:23.456789+05:30", + default_status="paused", + ), + expected_selection=("model:orders",), + expected_controls={ + "starts_at": "2026-07-02T19:47:23.456789Z", + "default_status": "paused", + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_selection_and_cron( + tmp_path: Path, test_case: SnapshotScheduleTestCase +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + nightly: Job = job( + "nightly_orders", + description="Refresh order reporting.", + select="+orders", + exclude="stg_orders", + settings={"run_tests": False, "vars": {"batch": 3}, "warehouse": "reporting"}, + concurrency=4, + concurrency_groups=("order_reporting",), + schedule=test_case.schedule, + targets=("development",), + ) + excluded: Job = replace(nightly, name="other_target", targets=("production",)) + first: dict[str, Any] = json.loads( + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=[nightly, excluded], + ).to_json() + ) + assert first["documents"]["jobs"] == { + "nightly_orders": { + "description": "Refresh order reporting.", + "selection": list(test_case.expected_selection), + "settings": { + "concurrency": 4, + "run_tests": False, + "vars": {"batch": 3}, + "warehouse": "reporting", + }, + "concurrency_groups": ["order_reporting"], + } + } + trigger: dict[str, Any] = first["documents"]["triggers"]["nightly_orders"] + assert trigger["job_key"] == "nightly_orders" + assert trigger["schedule"] == { + "kind": "cron", + "expression": "15 2 * * *", + "timezone": test_case.schedule.timezone, + } + assert {key: trigger[key] for key in test_case.expected_controls} == test_case.expected_controls + edited: Job = replace(nightly, schedule=schedule(cron="30 3 * * *", timezone="UTC")) + second: dict[str, Any] = json.loads( + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=[edited], + ).to_json() + ) + assert second["documents"]["triggers"].keys() == first["documents"]["triggers"].keys() + assert second["documents"]["triggers"]["nightly_orders"]["schedule"]["expression"] == ( + "30 3 * * *" + ) + assert second["nodes"] == first["nodes"] + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotDeclarationRejectionTestCase( + description="blank description", + job_description=" ", + expected_message="needs a description", + ), + SnapshotDeclarationRejectionTestCase( + description="blank selection", select="", expected_message="needs a selection" + ), + SnapshotDeclarationRejectionTestCase( + description="missing model", select="missing", expected_message="missing" + ), + SnapshotDeclarationRejectionTestCase( + description="invalid cron", cron="not a cron", expected_message="cron" + ), + SnapshotDeclarationRejectionTestCase( + description="invalid timezone", timezone="Unknown/Zone", expected_message="time zone" + ), + SnapshotDeclarationRejectionTestCase( + description="start needs an explicit offset", + schedule=schedule(cron="0 2 * * *", timezone="UTC", starts_at="2026-07-03T01:17:23"), + expected_message="starts_at", + ), + SnapshotDeclarationRejectionTestCase( + description="duplicate names", copies=2, expected_message="duplicate job orders" + ), + SnapshotDeclarationRejectionTestCase( + description="unknown setting", + settings={"concurreny": 4}, + expected_message="unknown build setting concurreny", + ), + SnapshotDeclarationRejectionTestCase( + description="selection cannot escape its pinned run", + settings={"select": "customers"}, + expected_message="owned by the run request", + ), + SnapshotDeclarationRejectionTestCase( + description="zero concurrency", + concurrency=0, + expected_message="must be positive", + ), + SnapshotDeclarationRejectionTestCase( + description="boolean is not concurrency", + concurrency=True, + expected_message="concurrency must be an integer", + ), + SnapshotDeclarationRejectionTestCase( + description="integer is not a boolean flag", + settings={"full_refresh": 1}, + expected_message="must be boolean", + ), + SnapshotDeclarationRejectionTestCase( + description="integer strings are not typed settings", + settings={"concurrency": "3"}, + expected_message="must be an integer", + ), + SnapshotDeclarationRejectionTestCase( + description="variables need an object", + settings={"vars": ["orders"]}, + expected_message="must be a JSON object", + ), + SnapshotDeclarationRejectionTestCase( + description="mutually exclusive load settings", + settings={"load_sources": False, "reload": True}, + expected_message="mutually exclusive", + ), + SnapshotDeclarationRejectionTestCase( + description="conflicting cursor families", + settings={"start_cursor": "2", "end_cursor_int": "4"}, + expected_message="cannot be combined", + ), + SnapshotDeclarationRejectionTestCase( + description="duplicate concurrency declarations", + concurrency=3, + settings={"concurrency": 4}, + expected_message="declared twice", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_declarations_when_compiling_then_native_validation_rejects_them( + tmp_path: Path, test_case: SnapshotDeclarationRejectionTestCase +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + declarations: list[Job] = [ + job( + "orders", + description=test_case.job_description, + select=test_case.select, + settings=test_case.settings, + concurrency=test_case.concurrency, + schedule=test_case.schedule + or schedule(cron=test_case.cron, timezone=test_case.timezone), + ) + ] * test_case.copies + with pytest.raises(ValueError, match=test_case.expected_message): + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=declarations, + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotDiscoveryTestCase( + description="relative imports aliases and factory containers", + files={ + "orchestration/shared.py": ( + "from sqlbuild.orchestration import job\n" + "staging = job('staging', description='Stage orders.', select='stg_orders')\n" + ), + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job\n" + "from .shared import staging\n" + "alias = staging\n" + "def factory():\n" + " return [job('daily', description='Daily orders.', select='orders')]\n" + "jobs = [factory(), staging]\n" + "jobs.append(jobs)\n" + ), + }, + expected_jobs=("daily", "staging"), + ), + SnapshotDiscoveryTestCase( + description="group-only references retain lineage and explicit waits", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job, job_group\n" + "def factory():\n" + " staging = job('staging', description='Stage orders.', select='stg_orders')\n" + " marts = job('marts', description='Build orders.', select='orders')\n" + " inventory = job('inventory', description='Repeat staging.', select='stg_orders')\n" + " return job_group('daily', description='Ordered jobs.', " + "jobs=[marts, inventory, staging], after={inventory: [staging]})\n" + "daily = factory()\n" + ), + }, + expected_jobs=("inventory", "marts", "staging"), + expected_groups={ + "daily": { + "inventory": ["staging"], + "marts": ["inventory", "staging"], + "staging": [], + } + }, + ), + SnapshotDiscoveryTestCase( + description="explicit empty declarations do not execute project files", + files={"orchestration/jobs.py": "raise RuntimeError('must not import')\n"}, + expected_jobs=(), + declarations=(), + ), + SnapshotDiscoveryTestCase( + description="DuckShop edge-case declaration containers", + files={ + "orchestration/edge_cases/definitions.py": Path( + "benchmarks/duckshop/project/orchestration/edge_cases/definitions.py" + ).read_text(), + "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( + "MODEL (", "MODEL (tags ['edge_case'],", 1 + ), + }, + expected_jobs=( + "edge_snapshot_inventory", + "edge_snapshot_orders", + "edge_snapshot_products", + ), + expected_groups={ + "edge_snapshot_group": {"edge_snapshot_orders": []}, + "edge_inventory_group": {"edge_snapshot_inventory": []}, + "edge_products_group": {"edge_snapshot_products": []}, + }, + expected_sensors={ + "edge_platform_orders": { + "description": "Catch up retained platform observations with a stopped group sensor.", + "job_key": "edge_snapshot_group", + "when": { + "kind": "platform_event", + "kinds": ["tick_overdue", "location_broken", "launch_failed"], + }, + "minimum_interval_seconds": 7, + "default_status": "paused", + }, + "edge_job_outcomes": { + "description": "Keep warning and partial outcomes distinct from success.", + "job_key": "edge_snapshot_inventory", + "when": { + "kind": "run_event", + "job_key": "edge_snapshot_orders", + "kinds": ["warn", "partial"], + }, + "minimum_interval_seconds": 30, + "default_status": "paused", + }, + "edge_group_outcomes": { + "description": "Observe group completion including cancellation and presumed failure.", + "job_key": "edge_snapshot_inventory", + "when": { + "kind": "run_event", + "job_key": "edge_snapshot_group", + "kinds": ["completed"], + "executed_successfully": ["model:orders"], + }, + "minimum_interval_seconds": 30, + "default_status": "paused", + }, + }, + expected_schedules=("edge_snapshot_orders",), + ), + SnapshotDiscoveryTestCase( + description="sensor-only roots collect source groups and destination jobs natively", + files={}, + declarations=( + sensor( + "on_orders", + description="Build inventory after group completion.", + when=run_event( + job=job_group( + "source_group", + description="Source orders.", + jobs=[job("source_orders", description="Orders.", select="orders")], + ), + kinds=[RunEventKind.COMPLETED], + executed_successfully=("orders", "orders*"), + ), + run=job("inventory", description="Inventory.", select="stg_orders"), + ), + ), + expected_jobs=("inventory", "source_orders"), + expected_groups={"source_group": {"source_orders": []}}, + expected_sensors={ + "on_orders": { + "description": "Build inventory after group completion.", + "job_key": "inventory", + "when": { + "kind": "run_event", + "job_key": "source_group", + "kinds": ["completed"], + "executed_successfully": ["model:orders"], + }, + "minimum_interval_seconds": 30, + "default_status": "running", + } + }, + ), + SnapshotDiscoveryTestCase( + description="any-run node selectors freeze canonical keys without a source job reference", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" + ).read_text() + }, + expected_jobs=("inventory", "source_orders"), + expected_sensors={ + "on_orders": { + "description": "Build inventory for every genuine orders rebuild.", + "job_key": "inventory", + "when": { + "kind": "node_event", + "selection": ["model:orders"], + "kinds": ["success"], + }, + "minimum_interval_seconds": 30, + "default_status": "running", + } + }, + ), + SnapshotDiscoveryTestCase( + description="failure and recovery selectors preserve their individual transition kinds", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" + ) + .read_text() + .replace("NodeEventKind.SUCCESS", "NodeEventKind.FAILED, NodeEventKind.RECOVERED") + }, + expected_jobs=("inventory", "source_orders"), + expected_sensors={ + "on_orders": { + "description": "Build inventory for every genuine orders rebuild.", + "job_key": "inventory", + "when": { + "kind": "node_event", + "selection": ["model:orders"], + "kinds": ["failed", "recovered"], + }, + "minimum_interval_seconds": 30, + "default_status": "running", + } + }, + ), + SnapshotDiscoveryTestCase( + description="source_new_data selectors freeze only source nodes without event kinds", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.py" + ).read_text(), + "sources/orders.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" + ).read_text(), + }, + expected_jobs=("orders_refresh",), + expected_sensors={ + "on_source_orders": { + "description": "Refresh orders for the first source version and each changed version.", + "job_key": "orders_refresh", + "when": {"kind": "source_new_data", "selection": ["source:raw_orders"]}, + "minimum_interval_seconds": 1, + "default_status": "running", + } + }, + ), + SnapshotDiscoveryTestCase( + description="source_stale freezes source nodes and a positive timestamp-age threshold", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" + ).read_text(), + "sources/orders.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" + ).read_text(), + }, + expected_jobs=("orders_refresh",), + expected_sensors={ + "on_stale_orders": { + "description": "Refresh orders once for each timestamp version older than one minute.", + "job_key": "orders_refresh", + "when": { + "kind": "source_stale", + "selection": ["source:raw_orders"], + "after_seconds": 60, + }, + "minimum_interval_seconds": 1, + "default_status": "running", + } + }, + ), + SnapshotDiscoveryTestCase( + description="source-job execution selectors compile to frozen canonical keys", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ).read_text() + }, + expected_jobs=("inventory", "source_orders"), + expected_sensors={ + "on_orders": { + "description": "Build inventory only after orders actually succeeds.", + "job_key": "inventory", + "when": { + "kind": "run_event", + "job_key": "source_orders", + "kinds": ["completed"], + "executed_successfully": ["model:orders"], + }, + "minimum_interval_seconds": 30, + "default_status": "running", + } + }, + ), + SnapshotDiscoveryTestCase( + description="sensor-only roots preserve referenced jobs and filter targets", + files={ + "orchestration/health.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ).read_text() + }, + expected_jobs=("health_orders",), + expected_sensors={ + "overdue_orders": { + "description": "Catch up overdue order reporting.", + "job_key": "health_orders", + "when": {"kind": "platform_event", "kinds": ["tick_overdue"]}, + "minimum_interval_seconds": 7, + "default_status": "paused", + } + }, + ), + SnapshotDiscoveryTestCase( + description="a project without declarations has no job document", + files={}, + expected_jobs=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_declaration_modules_when_compiling_then_native_discovery_collects_jobs( + tmp_path: Path, test_case: SnapshotDiscoveryTestCase +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES | test_case.files) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + publication: dict[str, Any] = json.loads( + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=test_case.declarations, + ).to_json() + ) + assert tuple(publication["documents"].get("jobs", {})) == test_case.expected_jobs + assert { + name: definition["jobs"] + for name, definition in publication["documents"].get("groups", {}).items() + } == test_case.expected_groups + assert { + name: publication["documents"]["triggers"][name] for name in test_case.expected_sensors + } == test_case.expected_sensors + assert tuple(publication["documents"].get("triggers", {})) == tuple( + sorted((*test_case.expected_schedules, *test_case.expected_sensors)) + ) + for definition in publication["documents"].get("jobs", {}).values(): + assert set(definition["selection"]) <= publication["nodes"].keys() + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotDiscoveryRejectionTestCase( + description="group members are jobs, not nested groups", + files={ + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/nested_groups.py" + ).read_text(), + }, + expected_error=TypeError, + expected_message="group members must be Job objects", + ), + SnapshotDiscoveryRejectionTestCase( + description="a polling sensor needs an actual callback", + files={ + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, sensor\n" + "orders = job('orders', description='Build orders.', select='orders')\n" + "poll = sensor('poll', description='Poll orders.', run=orders)\n" + ), + }, + expected_error=ValueError, + expected_message="either a built-in condition or a decorated polling callback", + ), + SnapshotDiscoveryRejectionTestCase( + description="callbacks with event conditions remain explicitly unsupported", + files={ + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, sensor, platform_event, PlatformEventKind\n" + "orders = job('orders', description='Build orders.', select='orders')\n" + "@sensor('poll', description='Poll orders.', run=orders, " + "when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]))\n" + "def poll(ctx):\n return []\n" + ), + }, + expected_error=ValueError, + expected_message="callbacks with when are not supported yet", + ), + SnapshotDiscoveryRejectionTestCase( + description="explicit group cycles cannot publish", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job, job_group\n" + "staging = job('staging', description='Stage orders.', select='stg_orders')\n" + "marts = job('marts', description='Build orders.', select='orders')\n" + "daily = job_group('daily', description='Cyclic jobs.', " + "jobs=[marts, staging], after={staging: [marts]})\n" + ), + }, + expected_error=ValueError, + expected_message="group dependencies contain a cycle", + ), + SnapshotDiscoveryRejectionTestCase( + description="group waits cannot refer to nonmember jobs", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job, job_group\n" + "staging = job('staging', description='Stage orders.', select='stg_orders')\n" + "marts = job('marts', description='Build orders.', select='orders')\n" + "daily = job_group('daily', description='Unknown upstream.', " + "jobs=[marts], after={marts: [staging]})\n" + ), + }, + expected_error=ValueError, + expected_message="group needs known jobs and dependencies within its membership", + ), + SnapshotDiscoveryRejectionTestCase( + description="distinct objects cannot share a job name", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job\n" + "jobs = [job('daily', description='Orders.', select='orders') for _ in range(2)]\n" + ), + }, + expected_error=ValueError, + expected_message="duplicate job daily", + ), + SnapshotDiscoveryRejectionTestCase( + description="broken imports fail on every retry", + files={ + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/failed_import.py" + ).read_text() + }, + expected_error=PythonNodeDiscoveryError, + expected_message="Failed to import orchestration file orchestration/jobs.py", + ), + SnapshotDiscoveryRejectionTestCase( + description="DuckShop settings cannot replace the selection", + files={ + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py" + ).read_text() + }, + expected_error=ValueError, + expected_message="owned by the run request", + ), + SnapshotDiscoveryRejectionTestCase( + description="DuckShop mutually exclusive settings fail on every retry", + files={ + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py" + ).read_text() + }, + expected_error=ValueError, + expected_message="mutually exclusive", + ), + SnapshotDiscoveryRejectionTestCase( + description="a zero sensor interval cannot publish", + files={ + "orchestration/health.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ) + .read_text() + .replace("minimum_interval_seconds=7", "minimum_interval_seconds=0") + }, + expected_error=ValueError, + expected_message="nonzero u32", + ), + SnapshotDiscoveryRejectionTestCase( + description="empty platform conditions cannot publish", + files={ + "orchestration/health.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ) + .read_text() + .replace("kinds=[PlatformEventKind.TICK_OVERDUE]", "kinds=[]") + }, + expected_error=ValueError, + expected_message="nonempty event kinds", + ), + SnapshotDiscoveryRejectionTestCase( + description="platform kinds are a closed vocabulary", + files={ + "orchestration/health.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ) + .read_text() + .replace("kinds=[PlatformEventKind.TICK_OVERDUE]", "kinds=['unknown_event']") + }, + expected_error=ValueError, + expected_message="unknown variant", + ), + SnapshotDiscoveryRejectionTestCase( + description="empty node selectors cannot broaden matching to all nodes", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" + ) + .read_text() + .replace('select="orders orders*"', 'select=" "') + }, + expected_error=ValueError, + expected_message="node_event needs nonempty selectors", + ), + SnapshotDiscoveryRejectionTestCase( + description="unknown node event kinds cannot publish", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" + ) + .read_text() + .replace("kinds=[NodeEventKind.SUCCESS]", "kinds=['unknown_event']") + }, + expected_error=ValueError, + expected_message="unknown variant", + ), + SnapshotDiscoveryRejectionTestCase( + description="source_new_data rejects model selections", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.py" + ) + .read_text() + .replace('select="source:raw_orders"', 'select="orders"'), + }, + expected_error=ValueError, + expected_message="source_new_data must select only source nodes", + ), + SnapshotDiscoveryRejectionTestCase( + description="source_new_data rejects blank selections", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.py" + ) + .read_text() + .replace('select="source:raw_orders"', 'select=" "'), + }, + expected_error=ValueError, + expected_message="source_new_data needs nonempty selectors", + ), + SnapshotDiscoveryRejectionTestCase( + description="source_stale rejects a zero threshold", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" + ) + .read_text() + .replace("after_seconds=60", "after_seconds=0"), + "sources/orders.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" + ).read_text(), + }, + expected_error=ValueError, + expected_message="sensor on_stale_orders", + ), + SnapshotDiscoveryRejectionTestCase( + description="source_stale rejects a boolean threshold", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" + ) + .read_text() + .replace("after_seconds=60", "after_seconds=True"), + "sources/orders.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" + ).read_text(), + }, + expected_error=ValueError, + expected_message="sensor on_stale_orders", + ), + SnapshotDiscoveryRejectionTestCase( + description="source_stale rejects an overflowing threshold", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" + ) + .read_text() + .replace("after_seconds=60", "after_seconds=2**32"), + "sources/orders.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" + ).read_text(), + }, + expected_error=ValueError, + expected_message="sensor on_stale_orders", + ), + SnapshotDiscoveryRejectionTestCase( + description="source_stale rejects model selections", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" + ) + .read_text() + .replace('select="source:raw_orders"', 'select="orders"'), + }, + expected_error=ValueError, + expected_message="source_stale must select only source nodes", + ), + SnapshotDiscoveryRejectionTestCase( + description="explicit empty execution selectors cannot broaden matching", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', "()") + }, + expected_error=ValueError, + expected_message="executed_successfully needs nonempty selectors", + ), + SnapshotDiscoveryRejectionTestCase( + description="blank execution selectors cannot broaden matching", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', '(" ",)') + }, + expected_error=ValueError, + expected_message="executed_successfully needs nonempty selectors", + ), + SnapshotDiscoveryRejectionTestCase( + description="unresolved execution selectors fail closed", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', '("unknown_orders",)') + }, + expected_error=ValueError, + expected_message="unknown selector name 'unknown_orders'", + ), + SnapshotDiscoveryRejectionTestCase( + description="unmatched execution patterns cannot become an omitted filter", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', '("unknown_*",)') + }, + expected_error=ValueError, + expected_message="unknown selector pattern", + ), + SnapshotDiscoveryRejectionTestCase( + description="invalid execution selector syntax fails closed", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', '("tag:",)') + }, + expected_error=ValueError, + expected_message="has empty value after ':'", + ), + SnapshotDiscoveryRejectionTestCase( + description="execution selectors outside the source job fail closed", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', '("stg_orders",)') + }, + expected_error=ValueError, + expected_message="executed_successfully must select nonempty source job/group nodes", + ), + SnapshotDiscoveryRejectionTestCase( + description="sensor code defaults use running or paused", + files={ + "orchestration/health.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ) + .read_text() + .replace('default_status="paused"', 'default_status="stopped"') + }, + expected_error=ValueError, + expected_message="unknown variant", + ), + ], + ids=lambda case: case.description, +) +def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapshot_is_returned( + tmp_path: Path, test_case: SnapshotDiscoveryRejectionTestCase +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES | test_case.files) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + for _ in range(2): + with pytest.raises(test_case.expected_error, match=test_case.expected_message): + compile_snapshot( + project=project, metadata_json=json.dumps(METADATA), project_dir=tmp_path + ) diff --git a/tests/unit/scripts/delta_compile_series/main/_test_types.py b/tests/unit/scripts/delta_compile_series/main/_test_types.py new file mode 100644 index 0000000000..30c764273b --- /dev/null +++ b/tests/unit/scripts/delta_compile_series/main/_test_types.py @@ -0,0 +1,15 @@ +"""Comparison evidence controls.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ComparisonEvidenceTestCase: + description: str + stderr: str = "SQL test planning START\npassword=synthetic-secret\n" + stdout: str = '{"models": ["orders"]}\n' + steps: int = 1 + expected_mismatches: int = 1 + cache_relative_path: str = "orders/check.sql" diff --git a/tests/unit/scripts/delta_compile_series/main/helpers.py b/tests/unit/scripts/delta_compile_series/main/helpers.py new file mode 100644 index 0000000000..22d1a9df03 --- /dev/null +++ b/tests/unit/scripts/delta_compile_series/main/helpers.py @@ -0,0 +1,122 @@ +"""Synthetic compile callbacks exercising the comparison harness's public interface.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +from pathlib import Path + +from scripts.delta_compile_series.main.run_delta_compile_series import run_delta_compile_series +from scripts.delta_compile_series.main.write_delta_compile_project import ( + write_delta_compile_project, +) +from scripts.delta_compile_series.models import DeltaCompileOutcome, DeltaCompileSeries +from tests.unit.scripts.delta_compile_series.main._test_types import ComparisonEvidenceTestCase + + +def run_comparison( + *, + root: Path, + destination: Path | None, + test_case: ComparisonEvidenceTestCase, +) -> DeltaCompileSeries: + """Compare real copied cache/artifacts, with controlled output and cache changes.""" + + project: Path = root / "orders" + write_delta_compile_project(project_dir=project, model_count=8) + artifact: Path = project / "target/compiled/tests/orders/check.sql" + artifact.parent.mkdir(parents=True) + artifact.write_text("SELECT 1 AS order_id\n") + cache: Path = project / "target/cache/compiler-native-v1/sql-test-artifacts.json" + cache.parent.mkdir(parents=True) + cache.write_text( + json.dumps( + { + "version": 5, + "records": { + "orders": { + "relative_path": test_case.cache_relative_path, + "identity": "original-identity", + "size": artifact.stat().st_size, + "mtime_ns": artifact.stat().st_mtime_ns, + "content_sha256": hashlib.sha256(artifact.read_bytes()).hexdigest(), + } + }, + } + ) + ) + + def compile_project() -> DeltaCompileOutcome: + return DeltaCompileOutcome(0, test_case.stdout, "", ("sqb", "compile", "--json")) + + def compile_full(copy: Path) -> DeltaCompileOutcome: + _ = (copy / cache.relative_to(project)).write_text("{") + return DeltaCompileOutcome( + 0, + test_case.stdout, + test_case.stderr, + ("sqb", "compile", "--json"), + (("SQLBUILD_DELTA_COMPILE_DISABLE", "1"), ("PASSWORD", "synthetic-environment-secret")), + ) + + return run_delta_compile_series( + project_dir=project, + seed=16, + steps=test_case.steps, + compile_project=compile_project, + compile_in_full=compile_full, + diagnostics_dir=destination, + case="orders comparison", + ) + + +def diagnostic_recipient(root: Path) -> tuple[Path, Path]: + """Generate disposable test keys without printing or persisting them outside tmp_path.""" + + root.mkdir(mode=0o700) + key: Path = root / "key.pem" + certificate: Path = root / "recipient.pem" + _ = subprocess.run( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-noenc", + "-keyout", + str(key), + "-out", + str(certificate), + "-days", + "1", + "-subj", + "/CN=orders-diagnostics-test", + ], + check=True, + capture_output=True, + ) + key.chmod(0o600) + return certificate, key + + +def decrypt_evidence(path: Path, key: Path) -> subprocess.CompletedProcess[bytes]: + """Keep all unauthenticated output in memory; callers must check success before use.""" + + return subprocess.run( + [ + "openssl", + "cms", + "-decrypt", + "-binary", + "-inform", + "DER", + "-in", + str(path), + "-inkey", + str(key), + ], + check=False, + capture_output=True, + ) diff --git a/tests/unit/scripts/delta_compile_series/main/test_comparison_evidence.py b/tests/unit/scripts/delta_compile_series/main/test_comparison_evidence.py new file mode 100644 index 0000000000..98c2548a32 --- /dev/null +++ b/tests/unit/scripts/delta_compile_series/main/test_comparison_evidence.py @@ -0,0 +1,180 @@ +"""Private failure evidence must not change comparison acceptance or successful runs.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +import pytest + +from scripts.delta_compile_series.models import DeltaCompileSeries +from tests.unit.scripts.delta_compile_series.main._test_types import ComparisonEvidenceTestCase +from tests.unit.scripts.delta_compile_series.main.helpers import run_comparison + + +@pytest.mark.parametrize( + "test_case", + [ComparisonEvidenceTestCase(description="private paired evidence")], + ids=lambda case: case.description, +) +def test_given_mismatch_when_capturing_then_keeps_private_paired_evidence( + tmp_path: Path, test_case: ComparisonEvidenceTestCase +) -> None: + destination: Path = tmp_path / "private" + result: DeltaCompileSeries = run_comparison( + root=tmp_path / "run", destination=destination, test_case=test_case + ) + + assert len(result.unexpected) == test_case.expected_mismatches + assert ( + "stderr differs from a full compile: '' vs 'SQL test planning START'" + in result.unexpected[0] + ) + directory: Path = destination / "failure-00" + manifest: dict[str, Any] = json.loads((directory / "manifest.json").read_text()) + before: dict[str, Any] = json.loads((directory / "cache-before.json").read_text()) + after: dict[str, Any] = json.loads((directory / "cache-after.json").read_text()) + key = "target/cache/compiler-native-v1/sql-test-artifacts.json" + assert manifest["seed"] == 16 + assert manifest["case"] == "orders comparison" + assert manifest["step"].startswith("step 0: column_description") + assert manifest["runs"]["full"]["command"] == ["sqb", "compile", "--json"] + assert manifest["runs"]["full"]["environment"] == [ + ["SQLBUILD_DELTA_COMPILE_DISABLE", "1"], + ["PASSWORD", "[REDACTED]"], + ] + assert "synthetic-environment-secret" not in (directory / "manifest.json").read_text() + assert manifest["runs"]["delta"]["returncode"] == manifest["runs"]["full"]["returncode"] == 0 + assert ( + manifest["runs"]["delta"]["semantic_json_sha256"] + == manifest["runs"]["full"]["semantic_json_sha256"] + ) + assert before["full"]["caches"][key]["parseable"] is True + assert after["full"]["caches"][key]["parseable"] is False + physical: dict[str, Any] = before["delta"]["caches"][key]["physical_artifacts"]["orders"] + assert physical["recorded_digest_matches"] is True + assert physical["recorded_mtime_matches"] is True + assert physical["recorded_identity"] == "original-identity" + assert (directory / "delta.stdout").read_text() == '{"models": ["orders"]}\n' + assert "synthetic-secret" not in (directory / "full.stderr").read_text() + assert manifest["files"]["full.stderr"]["redacted"] is True + artifacts: dict[str, Any] = json.loads((directory / "delta.artifacts.json").read_text()) + assert ( + artifacts["files"]["target/compiled/tests/orders/check.sql"]["sha256"] + == hashlib.sha256(b"SELECT 1 AS order_id\n").hexdigest() + ) + assert destination.stat().st_mode & 0o777 == 0o700 + assert directory.stat().st_mode & 0o777 == 0o700 + assert all(path.stat().st_mode & 0o777 == 0o600 for path in directory.iterdir()) + + +@pytest.mark.parametrize( + "test_case", + [ + ComparisonEvidenceTestCase( + description="no capture on success", stderr="", expected_mismatches=0 + ) + ], + ids=lambda case: case.description, +) +def test_given_success_when_capturing_then_changes_neither_result_nor_files( + tmp_path: Path, test_case: ComparisonEvidenceTestCase +) -> None: + destination: Path = tmp_path / "private" + captured: DeltaCompileSeries = run_comparison( + root=tmp_path / "captured", destination=destination, test_case=test_case + ) + normal: DeltaCompileSeries = run_comparison( + root=tmp_path / "normal", destination=None, test_case=test_case + ) + + assert captured == normal + assert len(captured.unexpected) == test_case.expected_mismatches + assert captured.unexpected == () + assert not destination.exists() + + +@pytest.mark.parametrize( + "test_case", + [ + ComparisonEvidenceTestCase( + description="bounded failures", + stdout="x" * (512 * 1024 + 19), + steps=6, + expected_mismatches=6, + ) + ], + ids=lambda case: case.description, +) +def test_given_large_repeated_mismatches_when_capturing_then_bounds_size( + tmp_path: Path, test_case: ComparisonEvidenceTestCase +) -> None: + destination: Path = tmp_path / "private" + result: DeltaCompileSeries = run_comparison( + root=tmp_path / "run", destination=destination, test_case=test_case + ) + + assert len(result.unexpected) == test_case.expected_mismatches + assert len(list(destination.iterdir())) == 4 + assert ( + sum(path.stat().st_size for path in filter(Path.is_file, destination.rglob("*"))) + <= 16 * 1024 * 1024 + ) + manifest: dict[str, Any] = json.loads((destination / "failure-00/manifest.json").read_text()) + assert manifest["files"]["delta.stdout"]["truncated"] is True + assert ( + manifest["files"]["delta.stdout"]["original_sha256"] + == hashlib.sha256(test_case.stdout.encode()).hexdigest() + ) + assert (destination / "failure-00/delta.stdout").stat().st_size == 512 * 1024 + + +@pytest.mark.parametrize( + "test_case", + [ComparisonEvidenceTestCase(description="capture cannot mask mismatch")], + ids=lambda case: case.description, +) +def test_given_capture_io_failure_when_comparing_then_preserves_original_failure( + tmp_path: Path, + test_case: ComparisonEvidenceTestCase, +) -> None: + destination: Path = tmp_path / "not-a-directory" + destination.write_text("occupied") + result: DeltaCompileSeries = run_comparison( + root=tmp_path / "run", destination=destination, test_case=test_case + ) + normal: DeltaCompileSeries = run_comparison( + root=tmp_path / "normal", destination=None, test_case=test_case + ) + + assert result == normal + assert len(result.unexpected) == test_case.expected_mismatches + assert destination.read_text() == "occupied" + + +@pytest.mark.parametrize( + "test_case", + [ + ComparisonEvidenceTestCase( + description="malformed cache path", cache_relative_path="bad\x00.sql" + ) + ], + ids=lambda case: case.description, +) +def test_given_invalid_cache_path_when_capturing_then_preserves_comparison( + tmp_path: Path, test_case: ComparisonEvidenceTestCase +) -> None: + destination: Path = tmp_path / "private" + result: DeltaCompileSeries = run_comparison( + root=tmp_path / "captured", destination=destination, test_case=test_case + ) + normal: DeltaCompileSeries = run_comparison( + root=tmp_path / "normal", destination=None, test_case=test_case + ) + + assert result == normal + assert len(result.unexpected) == test_case.expected_mismatches + before: dict[str, Any] = json.loads((destination / "failure-00/cache-before.json").read_text()) + assert "capture_error" in before["delta"] diff --git a/tests/unit/scripts/delta_compile_series/main/test_encrypted_evidence.py b/tests/unit/scripts/delta_compile_series/main/test_encrypted_evidence.py new file mode 100644 index 0000000000..9df10e2312 --- /dev/null +++ b/tests/unit/scripts/delta_compile_series/main/test_encrypted_evidence.py @@ -0,0 +1,144 @@ +"""Real OpenSSL retention through the unchanged comparison interface.""" + +from __future__ import annotations + +import io +import subprocess +import zipfile +from concurrent.futures import Future, ThreadPoolExecutor +from pathlib import Path + +import pytest + +from scripts.delta_compile_series.models import DeltaCompileSeries +from tests.unit.scripts.delta_compile_series.main._test_types import ComparisonEvidenceTestCase +from tests.unit.scripts.delta_compile_series.main.helpers import ( + decrypt_evidence, + diagnostic_recipient, + run_comparison, +) + + +@pytest.mark.parametrize( + "test_case", + [ComparisonEvidenceTestCase(description="authenticated failure retention")], + ids=lambda case: case.description, +) +def test_given_mismatch_when_encrypting_then_decrypts_exact_evidence_and_rejects_damage( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: ComparisonEvidenceTestCase +) -> None: + certificate, key = diagnostic_recipient(tmp_path / "recipient") + _, wrong_key = diagnostic_recipient(tmp_path / "other-recipient") + encrypted: Path = tmp_path / "encrypted" + private: Path = tmp_path / "private" + monkeypatch.setenv("SQLBUILD_COMPARISON_ENCRYPTED_DIR", str(encrypted)) + monkeypatch.setenv("SQLBUILD_COMPARISON_RECIPIENT", str(certificate)) + result: DeltaCompileSeries = run_comparison( + root=tmp_path / "run", destination=private, test_case=test_case + ) + assert len(result.unexpected) == test_case.expected_mismatches + ciphertext: Path = encrypted / "failure-00/failure.cms" + decrypted: subprocess.CompletedProcess[bytes] = decrypt_evidence(ciphertext, key) + assert decrypted.returncode == 0 + with zipfile.ZipFile(io.BytesIO(decrypted.stdout)) as archive: + assert set(archive.namelist()) == {path.name for path in (private / "failure-00").iterdir()} + assert all( + archive.read(name) == (private / "failure-00" / name).read_bytes() + for name in archive.namelist() + ) + assert decrypt_evidence(ciphertext, wrong_key).returncode != 0 + original: bytes = ciphertext.read_bytes() + damaged: Path = tmp_path / "damaged.cms" + damaged.write_bytes(original[:-1] + bytes([original[-1] ^ 1])) + assert decrypt_evidence(damaged, key).returncode != 0 + damaged.write_bytes(original[: len(original) // 2]) + assert decrypt_evidence(damaged, key).returncode != 0 + assert b"SQL test planning" not in original + assert ciphertext.stat().st_mode & 0o777 == 0o600 + assert not list(encrypted.rglob("*.partial")) + + +@pytest.mark.parametrize( + "test_case", + [ + ComparisonEvidenceTestCase( + description="successful comparison", stderr="", expected_mismatches=0 + ) + ], + ids=lambda case: case.description, +) +def test_given_success_when_retention_enabled_then_creates_no_private_or_encrypted_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: ComparisonEvidenceTestCase +) -> None: + encrypted: Path = tmp_path / "encrypted" + private: Path = tmp_path / "private" + monkeypatch.setenv("SQLBUILD_COMPARISON_ENCRYPTED_DIR", str(encrypted)) + result: DeltaCompileSeries = run_comparison( + root=tmp_path / "run", destination=private, test_case=test_case + ) + assert result.unexpected == () + assert len(result.unexpected) == test_case.expected_mismatches + assert not encrypted.exists() + assert not private.exists() + + +@pytest.mark.parametrize( + "test_case", + [ComparisonEvidenceTestCase(description="failed encryption")], + ids=lambda case: case.description, +) +def test_given_invalid_recipient_when_retaining_then_preserves_failure_without_uploadable_output( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: ComparisonEvidenceTestCase +) -> None: + encrypted: Path = tmp_path / "encrypted" + monkeypatch.setenv("SQLBUILD_COMPARISON_ENCRYPTED_DIR", str(encrypted)) + monkeypatch.setenv("SQLBUILD_COMPARISON_RECIPIENT", str(tmp_path / "missing.pem")) + result: DeltaCompileSeries = run_comparison( + root=tmp_path / "captured", destination=tmp_path / "private", test_case=test_case + ) + normal: DeltaCompileSeries = run_comparison( + root=tmp_path / "normal", destination=None, test_case=test_case + ) + assert result == normal + assert len(result.unexpected) == test_case.expected_mismatches + assert not list(encrypted.rglob("*.cms")) + assert not list(encrypted.rglob("*.partial")) + + +@pytest.mark.parametrize( + "test_case", + [ + ComparisonEvidenceTestCase( + description="bounded encrypted retention", + steps=4, + expected_mismatches=4, + stdout="x" * (512 * 1024 + 19), + ) + ], + ids=lambda case: case.description, +) +def test_given_repeated_failures_when_retaining_then_bounds_ciphertext_count_and_bytes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: ComparisonEvidenceTestCase +) -> None: + certificate, _ = diagnostic_recipient(tmp_path / "recipient") + encrypted: Path = tmp_path / "encrypted" + monkeypatch.setenv("SQLBUILD_COMPARISON_ENCRYPTED_DIR", str(encrypted)) + monkeypatch.setenv("SQLBUILD_COMPARISON_RECIPIENT", str(certificate)) + with ThreadPoolExecutor(max_workers=4) as executor: + futures: list[Future[DeltaCompileSeries]] = [ + executor.submit( + run_comparison, + root=tmp_path / f"run-{index}", + destination=tmp_path / f"private-{index}", + test_case=test_case, + ) + for index in range(4) + ] + assert all( + len(future.result().unexpected) == test_case.expected_mismatches for future in futures + ) + files: list[Path] = list(encrypted.rglob("*.cms")) + assert len(files) == 12 + assert all(0 < path.stat().st_size <= 4 * 1024 * 1024 + 64 * 1024 for path in files) + assert sum(path.stat().st_size for path in files) <= 12 * (4 * 1024 * 1024 + 64 * 1024) + assert not list(encrypted.rglob("*.partial")) diff --git a/tests/unit/scripts/fensu_policy/rules/test_native_boundary.py b/tests/unit/scripts/fensu_policy/rules/test_native_boundary.py index dd3e4ba20f..9422bcd69a 100644 --- a/tests/unit/scripts/fensu_policy/rules/test_native_boundary.py +++ b/tests/unit/scripts/fensu_policy/rules/test_native_boundary.py @@ -135,6 +135,37 @@ def test_given_test_module_when_checking_native_patches_then_matches_contract( ), expected_lines=(3, 4, 5, 5, 6, 6), ), + NativeLinesTestCase( + description="pyclass module metadata is not a Python crossing", + source='#[pyclass(frozen, module = "sqlbuild._native")]\nstruct Snapshot;\n', + expected_lines=(), + ), + NativeLinesTestCase( + description="multiline namespaced pyclass metadata preserves callback line numbers", + source=dedent( + """ + #[pyo3::pyclass( + frozen, + module = "sqlbuild._native", + )] + struct Snapshot; + fn read(py: Python<'_>) { + py.import("sqlbuild.compiler"); + } + """ + ), + expected_lines=(8, 8), + ), + NativeLinesTestCase( + description="raw pyclass module metadata does not hide an adjacent real crossing", + source='#[pyclass(module = r#"sqlbuild._native"#)] struct Snapshot; fn read() { imported(py, "sqlbuild.compiler", "compile"); }\n', + expected_lines=(1, 1), + ), + NativeLinesTestCase( + description="module assignments outside pyclass remain crossings", + source='fn read() { let module = "sqlbuild.compiler"; load(module); }\n', + expected_lines=(1,), + ), NativeLinesTestCase( description="comments, imports and test modules are skipped", source=dedent( diff --git a/tests/unit/scripts/release_gate/_helpers/_test_types.py b/tests/unit/scripts/release_gate/_helpers/_test_types.py index 0792f1eb63..a1e9dc6c9e 100644 --- a/tests/unit/scripts/release_gate/_helpers/_test_types.py +++ b/tests/unit/scripts/release_gate/_helpers/_test_types.py @@ -48,6 +48,15 @@ class RequiredCheckTestCase: expected_passes: bool +@dataclass(frozen=True) +class VerificationSuitesTestCase: + description: str + selected: str + checks: str + native_checks: str + expected_exit_code: int + + @dataclass(frozen=True) class ExpressionTestCase: description: str diff --git a/tests/unit/scripts/release_gate/_helpers/test_gate.py b/tests/unit/scripts/release_gate/_helpers/test_gate.py index d43aa66213..1a291b3b67 100644 --- a/tests/unit/scripts/release_gate/_helpers/test_gate.py +++ b/tests/unit/scripts/release_gate/_helpers/test_gate.py @@ -1,4 +1,7 @@ +import os +import subprocess from pathlib import Path +from typing import Any import pytest @@ -7,6 +10,7 @@ from tests.unit.scripts.release_gate._helpers._test_types import ( GateViolationTestCase, RepositoryGateTestCase, + VerificationSuitesTestCase, VerifyCheckNameTestCase, ) from tests.unit.scripts.release_gate._helpers.helpers import ( @@ -164,5 +168,50 @@ def test_given_workflows_when_checking_release_gate_then_reports_violations( assert get_release_gate_errors(tmp_path) == test_case.expected_errors +@pytest.mark.parametrize( + "test_case", + [ + VerificationSuitesTestCase("both selected suites pass", "true", "success", "success", 0), + VerificationSuitesTestCase( + "native failure cannot be masked by static success", "true", "success", "failure", 1 + ), + VerificationSuitesTestCase( + "selected native skip fails closed", "true", "success", "skipped", 1 + ), + VerificationSuitesTestCase( + "native cancellation fails closed", "true", "success", "cancelled", 1 + ), + VerificationSuitesTestCase( + "static failure cannot be masked by native success", "true", "failure", "success", 1 + ), + VerificationSuitesTestCase( + "docs-only skips terminate successfully", "false", "skipped", "skipped", 0 + ), + ], + ids=lambda case: case.description, +) +def test_given_selected_suites_when_aggregating_then_failures_and_skips_resolve( + test_case: VerificationSuitesTestCase, +) -> None: + job: dict[str, Any] = read_verify_job() + assert "native_checks" in job["needs"] + step: dict[str, Any] = job["steps"][0] + environment: dict[str, str] = {name: "success" for name in step["env"]} + environment.update( + CHECKS_SELECTED=test_case.selected, + CHECKS_RESULT=test_case.checks, + NATIVE_CHECKS_RESULT=test_case.native_checks, + ) + result: subprocess.CompletedProcess[str] = subprocess.run( + ["bash", "-e", "-c", step["run"]], + env=os.environ | environment, + capture_output=True, + text=True, + check=False, + timeout=5, + ) + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/_test_types.py b/tests/unit/src/sqlbuild/adapters/_test_types.py index b54f5f8247..d86173bd94 100644 --- a/tests/unit/src/sqlbuild/adapters/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/_test_types.py @@ -145,6 +145,9 @@ class RelationRowCountEstimateTestCase: expected_sql_fragment: str database: str | None = "analytics" schema: str | None = "dev" + metadata_only: bool = False + expected_byte_count: int | None = None + expected_byte_count_basis: str | None = None @dataclass(frozen=True) @@ -154,6 +157,9 @@ class BigQueryRowCountEstimateTestCase: num_rows: int | None expected_row_count: int | None expected_table_id: str = "proj.dev.orders" + metadata_only: bool = False + num_bytes: int | None = None + expected_byte_count: int | None = None @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/adapters/helpers.py b/tests/unit/src/sqlbuild/adapters/helpers.py index e5d33e4c46..bc838d5b69 100644 --- a/tests/unit/src/sqlbuild/adapters/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/helpers.py @@ -118,6 +118,7 @@ class FakeBigQueryTable: table_type: str num_rows: int | None + num_bytes: int | None = None class FakeBigQueryClient: diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_state_table_ddl.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_state_table_ddl.py index 00496491bd..2991ef7135 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_state_table_ddl.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_state_table_ddl.py @@ -121,7 +121,7 @@ def test_given_state_table_when_rendering_on_snowflake_then_permanent_with_90_da "target_schema VARCHAR, target_name VARCHAR, run_id VARCHAR NOT NULL, " "definition_hash VARCHAR NOT NULL, version_hash VARCHAR NOT NULL, " "schema_fingerprint VARCHAR NOT NULL, definition_b64 VARCHAR NOT NULL, " - "metadata_json_b64 VARCHAR NOT NULL, ts TIMESTAMP NOT NULL" + "metadata_json_b64 VARCHAR NOT NULL, ts TIMESTAMP NOT NULL, event_id VARCHAR" ") DATA_RETENTION_TIME_IN_DAYS = 90" ), ) diff --git a/tests/unit/src/sqlbuild/adapters/test_relation_row_count_estimate.py b/tests/unit/src/sqlbuild/adapters/test_relation_row_count_estimate.py index a286283dff..dd28c03fbe 100644 --- a/tests/unit/src/sqlbuild/adapters/test_relation_row_count_estimate.py +++ b/tests/unit/src/sqlbuild/adapters/test_relation_row_count_estimate.py @@ -9,6 +9,7 @@ from sqlbuild.adapter.contract.models import RelationRowCountEstimate from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter @@ -159,6 +160,80 @@ expected_detail="relation not found", expected_sql_fragment="schemas.name = N'dev'", ), + RelationRowCountEstimateTestCase( + description="postgres bytes remain available without analyzed rows", + adapter=PostgresAdapter(), + metadata_rows=(("r", -1.0, 8192),), + expected_row_count=None, + expected_detail="table statistics not collected; run ANALYZE", + expected_sql_fragment="pg_relation_size", + metadata_only=True, + expected_byte_count=8192, + expected_byte_count_basis="pg_relation_size_main_fork", + ), + RelationRowCountEstimateTestCase( + description="postgres empty storage is a known zero", + adapter=PostgresAdapter(), + metadata_rows=(("r", -1.0, 0),), + expected_row_count=0, + expected_detail=None, + expected_sql_fragment="pg_relation_size", + metadata_only=True, + expected_byte_count=0, + expected_byte_count_basis="pg_relation_size_main_fork", + ), + RelationRowCountEstimateTestCase( + description="snowflake bytes and rows are distinct", + adapter=SnowflakeAdapter(), + metadata_rows=((17, 4096),), + expected_row_count=17, + expected_detail=None, + expected_sql_fragment="SELECT row_count,bytes FROM", + metadata_only=True, + expected_byte_count=4096, + expected_byte_count_basis="information_schema_bytes", + ), + RelationRowCountEstimateTestCase( + description="snowflake missing view statistics remain unavailable", + adapter=SnowflakeAdapter(), + metadata_rows=((None, None),), + expected_row_count=None, + expected_detail="no row count (view)", + expected_sql_fragment="SELECT row_count,bytes FROM", + metadata_only=True, + ), + RelationRowCountEstimateTestCase( + description="databricks bytes do not imply an analyzed row count", + adapter=DatabricksAdapter(), + metadata_rows=_DATABRICKS_DETAIL_HEADER + + (("Type", "MANAGED", ""), ("Statistics", "2048 bytes", "")), + expected_row_count=None, + expected_detail="table statistics not collected; run ANALYZE TABLE ... COMPUTE STATISTICS", + expected_sql_fragment="DESCRIBE TABLE EXTENDED", + metadata_only=True, + expected_byte_count=2048, + expected_byte_count_basis="describe_table_statistics_bytes", + ), + RelationRowCountEstimateTestCase( + description="clickhouse bytes do not imply a known row count", + adapter=ClickHouseAdapter(), + metadata_rows=(("MergeTree", None, 16384),), + expected_row_count=None, + expected_detail="engine MergeTree does not report row counts", + expected_sql_fragment="SELECT engine, total_rows, total_bytes FROM system.tables", + metadata_only=True, + expected_byte_count=16384, + expected_byte_count_basis="system_tables_total_bytes", + ), + RelationRowCountEstimateTestCase( + description="clickhouse missing bytes do not imply empty storage", + adapter=ClickHouseAdapter(), + metadata_rows=(("Memory", 3, None),), + expected_row_count=3, + expected_detail=None, + expected_sql_fragment="SELECT engine, total_rows, total_bytes FROM system.tables", + metadata_only=True, + ), ], ids=lambda case: case.description, ) @@ -174,10 +249,13 @@ def test_given_catalog_metadata_when_estimating_row_count_then_reads_only_metada database=test_case.database, schema=test_case.schema, name="orders", + metadata_only=test_case.metadata_only, ) assert estimate.row_count == test_case.expected_row_count assert estimate.detail == test_case.expected_detail + assert estimate.byte_count == test_case.expected_byte_count + assert estimate.byte_count_basis == test_case.expected_byte_count_basis assert len(connection.executed) == 1 assert test_case.expected_sql_fragment in connection.executed[0] @@ -203,6 +281,23 @@ def test_given_catalog_metadata_when_estimating_row_count_then_reads_only_metada num_rows=0, expected_row_count=None, ), + BigQueryRowCountEstimateTestCase( + description="native bytes remain available without row statistics", + table_type="TABLE", + num_rows=None, + num_bytes=32768, + metadata_only=True, + expected_row_count=None, + expected_byte_count=32768, + ), + BigQueryRowCountEstimateTestCase( + description="external bytes are not native-table storage", + table_type="EXTERNAL", + num_rows=12, + num_bytes=65536, + metadata_only=True, + expected_row_count=None, + ), ], ids=lambda case: case.description, ) @@ -210,15 +305,24 @@ def test_given_bigquery_table_metadata_when_estimating_row_count_then_uses_table test_case: BigQueryRowCountEstimateTestCase, ) -> None: client: FakeBigQueryClient = FakeBigQueryClient( - FakeBigQueryTable(table_type=test_case.table_type, num_rows=test_case.num_rows) + FakeBigQueryTable( + table_type=test_case.table_type, + num_rows=test_case.num_rows, + num_bytes=test_case.num_bytes, + ) ) connection: Any = cast(Any, FakeBigQueryConnection(client=client)) estimate: RelationRowCountEstimate = BigQueryAdapter().estimate_relation_row_count( - connection=connection, database="proj", schema="dev", name="orders" + connection=connection, + database="proj", + schema="dev", + name="orders", + metadata_only=test_case.metadata_only, ) assert estimate.row_count == test_case.expected_row_count + assert estimate.byte_count == test_case.expected_byte_count assert client.requested == [test_case.expected_table_id] diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py index e9aac9965c..d31b9ea6c7 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py @@ -16,6 +16,7 @@ SourceFreshnessRecord, build_source_freshness_create_table_sql, ) +from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cost.models import CostResourceContext @@ -53,6 +54,14 @@ def render_qualified_name( def render_framework_type(self, framework_type: FrameworkType) -> str: return framework_type.value + def get_columns(self, **_scope: object) -> tuple[ColumnInfo, ...]: + return (ColumnInfo(name="event_id", type="VARCHAR"),) + + def render_add_columns( + self, *, destination: str, columns: tuple[ColumnInfo, ...] + ) -> tuple[str, ...]: + return DuckDbAdapter().render_add_columns(destination=destination, columns=columns) + def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: return build_source_freshness_create_table_sql( database=database, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py index fd1642cba4..f8964da4f8 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py @@ -11,6 +11,7 @@ ProjectConfig, ProjectConfigError, TargetConfig, + build_effective_connection_config, resolve_connection_config, resolve_project_connection_config, resolve_target_connection_config, @@ -174,6 +175,61 @@ def test_given_named_connection_layers_when_resolving_then_uses_locked_precedenc assert connection == test_case.expected_connection +@pytest.mark.parametrize( + "test_case", + [ + NamedConnectionBehaviorTestCase( + description="catalog credentials never inherit writer fields", + expected_connection={ + "account": "catalog-account", + "user": "catalog-reader", + "warehouse": "catalog-warehouse", + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_catalog_connection_when_resolving_then_only_its_own_named_layers_and_secrets_apply( + test_case: NamedConnectionBehaviorTestCase, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("SQLBUILD_MISSING_WRITER_PASSWORD", raising=False) + monkeypatch.setenv("SQLBUILD_CATALOG_USER", "catalog-reader") + discovered: DiscoveredProjectInputs = DiscoveredProjectInputs( + project_config=ProjectConfig( + name="orders", + adapter="snowflake", + default_target="development", + connection={"password": "${ENV:SQLBUILD_MISSING_WRITER_PASSWORD}"}, + connections={ + "writer": {"account": "writer-account", "private_key_file": "writer.pem"}, + "catalog": {"account": "catalog-account", "user": "${ENV:SQLBUILD_CATALOG_USER}"}, + }, + targets={ + "development": TargetConfig( + connection_name="writer", + catalog_connection="catalog", + connection={"role": "writer-role"}, + ) + }, + ), + local_config=LocalConfig( + connection={"token": "writer-token"}, + connections={"catalog": {"warehouse": "catalog-warehouse"}}, + targets={"development": LocalTargetConfig(connection={"user": "writer-user"})}, + ), + ) + assert ( + build_effective_connection_config(discovered_inputs=discovered, catalog=True) + == test_case.expected_connection + ) + with pytest.raises(CompileInputError, match="SQLBUILD_MISSING_WRITER_PASSWORD"): + build_effective_connection_config(discovered_inputs=discovered) + discovered.local_config.targets["development"] = LocalTargetConfig(catalog_connection="missing") + with pytest.raises(CompileInputError, match="catalog_connection: Unknown connection 'missing'"): + build_effective_connection_config(discovered_inputs=discovered, catalog=True) + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py index 3d5ba43db7..1c3e3d91f9 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py @@ -26,6 +26,13 @@ class CliCaptureOutcomeTestCase: expected_error_type: type[BaseException] | None = None +@dataclass(frozen=True) +class InterruptedCommandTestCase: + description: str + argv: list[str] + expected_error_type: type[BaseException] + + @dataclass(frozen=True) class ProjectCreationRoutingTestCase: description: str diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py index 47022aea48..630be312c8 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py @@ -3,6 +3,8 @@ from __future__ import annotations import importlib.util +import signal +import weakref from collections.abc import Callable from importlib.machinery import ModuleSpec from pathlib import Path @@ -11,6 +13,7 @@ from sqlbuild._native import ( CliEntrypointHandlers, + CompileCommandRequest, SpecConfigError, loader_destination_parts, ) @@ -45,6 +48,20 @@ def noop_handler(*_a: Any, **_k: Any) -> int: return 0 +def compile_with_callback_interrupt(_request: CompileCommandRequest) -> int: + """Deliver one real SIGINT from a callback where Python cannot propagate its exception.""" + + def value() -> None: + return None + + reference: weakref.ReferenceType[Callable[[], None]] = weakref.ref( + value, lambda _: signal.raise_signal(signal.SIGINT) + ) + del value + assert reference() is None + return 0 + + def extract_json_output_fields(args: tuple[object, ...]) -> tuple[bool, Path | None]: """Return (json_output, json_output_path) from a typed request or positional args.""" diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 52f80dc426..299b15387b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -1,9 +1,11 @@ from __future__ import annotations import json +import signal import sys from collections.abc import Callable from pathlib import Path +from typing import Any from unittest.mock import patch import pytest @@ -43,6 +45,7 @@ from tests.unit.src.sqlbuild.cli.commands.main.entry._test_types import ( AuditAggregateRenderingTestCase, CompileJsonErrorRenderingTestCase, + InterruptedCommandTestCase, MainErrorRenderingTestCase, MainTestCase, SkillFreshnessNoticeTestCase, @@ -52,6 +55,7 @@ UnprintableError, build_handlers, build_json_recording_handler, + compile_with_callback_interrupt, invalid_loader_destination_error, ) from tests.unit.src.sqlbuild.presentation.helpers import use_color_terminal @@ -2522,6 +2526,72 @@ def run_compile(_request: CompileCommandRequest) -> int: assert current_execution_identity() is None +@pytest.mark.parametrize( + "test_case", + ( + InterruptedCommandTestCase( + description="default interrupt survives an unraisable weakref callback", + argv=["compile"], + expected_error_type=KeyboardInterrupt, + ), + ), + ids=lambda case: case.description, +) +def test_given_sigint_in_weakref_callback_when_cli_runs_then_default_interrupt_is_retained( + test_case: InterruptedCommandTestCase, +) -> None: + unraisable: list[type[BaseException]] = [] + previous: Any = signal.signal(signal.SIGINT, signal.default_int_handler) + try: + with ( + patch("sys.unraisablehook", side_effect=lambda args: unraisable.append(args.exc_type)), + pytest.raises(test_case.expected_error_type), + ): + _main_with_dependencies( + argv=test_case.argv, + handlers=build_handlers(run_compile=compile_with_callback_interrupt), + ) + assert signal.getsignal(signal.SIGINT) is signal.default_int_handler + assert unraisable == [test_case.expected_error_type] + assert _main_with_dependencies(argv=test_case.argv, handlers=build_handlers()) == 0 + finally: + signal.signal(signal.SIGINT, previous) + + +@pytest.mark.parametrize( + "test_case", + ( + MainTestCase( + description="embedding interrupt handler remains authoritative", + argv=["compile"], + expected_exit_code=0, + ), + ), + ids=lambda case: case.description, +) +def test_given_embedding_sigint_handler_when_cli_runs_then_handler_remains_authoritative( + test_case: MainTestCase, +) -> None: + received: list[int] = [] + + def custom_handler(number: int, _frame: object) -> None: + received.append(number) + + previous: Any = signal.signal(signal.SIGINT, custom_handler) + try: + assert ( + _main_with_dependencies( + argv=test_case.argv, + handlers=build_handlers(run_compile=compile_with_callback_interrupt), + ) + == test_case.expected_exit_code + ) + assert signal.getsignal(signal.SIGINT) is custom_handler + assert received == [signal.SIGINT] + finally: + signal.signal(signal.SIGINT, previous) + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py index bd12b224da..118a40d383 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py @@ -20,9 +20,10 @@ SeedPlanEntry, ) from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -class CloneFingerprintAdapter: +class CloneFingerprintAdapter(DuckDbAdapter): adapter_name: str = "test" def execute(self, connection: object, sql: str) -> object: @@ -59,8 +60,8 @@ def render_read_latest_fingerprints_sql(self, **kwargs: object) -> str: del kwargs return "select * from fingerprints" - def render_framework_type(self, value: object) -> str: - return str(value) + def render_framework_type(self, type_name: object) -> str: + return str(type_name) def render_create_fingerprint_table_sql(self, **kwargs: object) -> str: del kwargs diff --git a/tests/unit/src/sqlbuild/sql_values/test_state_table_sql.py b/tests/unit/src/sqlbuild/sql_values/test_state_table_sql.py index 74e61e3e14..487e929215 100644 --- a/tests/unit/src/sqlbuild/sql_values/test_state_table_sql.py +++ b/tests/unit/src/sqlbuild/sql_values/test_state_table_sql.py @@ -27,9 +27,10 @@ expected_sql=( "INSERT INTO analytics._sqlbuild_source_freshness (source_name, " "target_database, target_schema, target_name, run_id, strategy, value_kind, " - "data_version, data_version_hash, observed_at) VALUES ('raw.o''rders', NULL, " + "data_version, data_version_hash, observed_at, event_id) VALUES ('raw.o''rders', NULL, " "'raw', 'orders', 'run-1', 'adapter', 'timestamp', " - "'2026-01-01T01:00:00+01:00', 'hash', '2026-01-01T00:00:00+00:00')" + "'2026-01-01T01:00:00+01:00', 'hash', '2026-01-01T00:00:00+00:00', " + "'a7a96057679375d1ba89ff2842c691f288d410dc1f502726feeb9cc0675bfd26')" ), ) ], diff --git a/uv.lock b/uv.lock index eda041b554..95b2e27b10 100644 --- a/uv.lock +++ b/uv.lock @@ -1343,31 +1343,31 @@ wheels = [ [[package]] name = "fensu" -version = "0.28.0" +version = "0.28.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fensu-cli" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/69/a132ffc407be97a0fdcfa052cb8b49408091db748e6afe0f7a6c77e6fa7d/fensu-0.28.0.tar.gz", hash = "sha256:11bcea3207f7fd92babc200f40d074e3da789ee11e7fc7024575a415de7c1176", size = 538990, upload-time = "2026-10-27T00:39:16.19Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8f/3d/d5b4fb0ad09426dae015af2817ba8416d2ea974a0c4a6ebea63112f90b1b/fensu-0.28.3.tar.gz", hash = "sha256:a7fc87cb60eaef964b4e980d19bd6de9c95cecc75c019d8b0058e02b6751f4b5", size = 539031, upload-time = "2026-11-01T09:41:23.715Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/59/e9/7434bd0565d54ff58796105728b3ccc8b838c7200997184978e6ebfdec4d/fensu-0.28.0-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:29df31aff900d20b0c845940af0cd91eead55e158a2511e682e3e887a22dada8", size = 4154207, upload-time = "2026-10-27T00:38:52.311Z" }, - { url = "https://files.pythonhosted.org/packages/7f/d8/e0875b269515d9a75643d5488bdc6f70a4b6c346995f12ce07304a3610f7/fensu-0.28.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:33a6d8b7d2f734c48f48087462c0e5a1ff94535c0e27e6a4820bc10303ff3653", size = 4004243, upload-time = "2026-10-27T00:38:54.994Z" }, - { url = "https://files.pythonhosted.org/packages/e8/05/b87c0ca5cf3f724517d6958f1ebd68948f9399306bd6d6814783635bad9a/fensu-0.28.0-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de07b3fee5d908d2a766ca136753c8040d6fe25a1346d9b3b5a98483fc23d874", size = 4230192, upload-time = "2026-10-27T00:38:57.321Z" }, - { url = "https://files.pythonhosted.org/packages/cb/d2/fdfc7fd322c3497c53c22754a33433e33678a6de84832fd41add09481a21/fensu-0.28.0-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f44293a2da6bf4558d00eb4f859b28f3fb69d396dad1c25ac466a92a7d1dd734", size = 4307267, upload-time = "2026-10-27T00:38:59.679Z" }, - { url = "https://files.pythonhosted.org/packages/56/c0/2ab2c2c1af4ddd1b45c750679bd9f0b777d7eca3c0e00a233c991aaa13e7/fensu-0.28.0-cp312-abi3-win_amd64.whl", hash = "sha256:ffa88f1d259a1830d513ae930ff856d5b5e4941ba433889dea04d3db5753f281", size = 3977828, upload-time = "2026-10-27T00:39:01.761Z" }, + { url = "https://files.pythonhosted.org/packages/2f/a7/9493fbcf3158ae009651d19c7289855e8ef571dfaf5c7c77154a8a6a4a21/fensu-0.28.3-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e3e3896f173d2bb676fe04defdd4ebdf4db38aee3ee6e57d7a5a297df32d7c5c", size = 4154274, upload-time = "2026-11-01T09:41:03.614Z" }, + { url = "https://files.pythonhosted.org/packages/a2/71/20fdb7147e268c5dbd9eae0fc9dd99641e416f9e75e8fd067ac089f6db78/fensu-0.28.3-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:d74db7466b2ddd533413d2daf4c704aa4acbcae61e2a0edba2564a4910666707", size = 4004309, upload-time = "2026-11-01T09:41:05.719Z" }, + { url = "https://files.pythonhosted.org/packages/d0/4c/40b01d8e46ff3464eb503529e00ad23eddd2ffbd605282618672d0aff45f/fensu-0.28.3-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:da5843aa40c7d6398a95b340b3c60d4360b5feddac34960663ee84aec09df8ef", size = 4230261, upload-time = "2026-11-01T09:41:07.481Z" }, + { url = "https://files.pythonhosted.org/packages/e7/7a/243cccd83531c3c86c660b993e77a7014c65d1d3f20c07e45b067dba81cf/fensu-0.28.3-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:52e1623e614f101c5ada2eda59548ff441b7efe4228813e69bc0f0ef9b3fbca6", size = 4307334, upload-time = "2026-11-01T09:41:09.938Z" }, + { url = "https://files.pythonhosted.org/packages/88/a3/38dbb50d5079a1a5fcaf62bcdd582c12a53609c575a747cb2461e83ddfcf/fensu-0.28.3-cp312-abi3-win_amd64.whl", hash = "sha256:a39c407725ea4d92123414bdbe9f80d5db0b7a412af49082a87378203c9ecb04", size = 3977904, upload-time = "2026-11-01T09:41:11.672Z" }, ] [[package]] name = "fensu-cli" -version = "0.28.0" +version = "0.28.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/79/d9/f347f514fb224f6a1f4ca6e616ac49b573f9dceba4ff8d9efcd334f48bea/fensu_cli-0.28.0.tar.gz", hash = "sha256:1feb379ed92e193cbd4d28ab3d7729798b88120aaee2f59fd7f341519b6ee945", size = 765638, upload-time = "2026-10-27T00:39:17.752Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/17/3d0953c33387caa0a6becc9bb02210d4a020eaaf73ef07bdc0c8f8952562/fensu_cli-0.28.3.tar.gz", hash = "sha256:3ba38bedc88a9e6db8563f638aa6ba8649a3e5750de176e22a7b35561d14bd9e", size = 766571, upload-time = "2026-11-01T09:41:25.271Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/b3/57f1de89e1cb7534b0ef6415822c8982a247b9d2efe8d2fddd85e35389c8/fensu_cli-0.28.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:13ecc7a224a4b32eef19456f21659e682e636ef53f305919aa57cae27c8e3fec", size = 7978247, upload-time = "2026-10-27T00:39:04.644Z" }, - { url = "https://files.pythonhosted.org/packages/ff/ff/2e887d727f82c61ea7cb6773429b61d4711b93f7f94814eb9a7e891384dc/fensu_cli-0.28.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:149e5e22c8895c0a3ed3956b30e693e78df7dfb67ce8e0c5682b84c77e811e7e", size = 7490546, upload-time = "2026-10-27T00:39:07.384Z" }, - { url = "https://files.pythonhosted.org/packages/76/4e/daa4c2d70e5b0398382f84c846aacc4a937c4d00b3938d00a67351b0f212/fensu_cli-0.28.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:99932d17956bb10da1ac677867567b504a961d0565fcda730c1e682434244113", size = 7570220, upload-time = "2026-10-27T00:39:09.852Z" }, - { url = "https://files.pythonhosted.org/packages/f4/f9/c9e1095c507da3a53b3e7bf3f7c95f75824429ea45a72cb2c7af67a605f7/fensu_cli-0.28.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee51c5d6d0d5fa0b36c0f70d3d6fde429793b643f0075cdfdf3fc8b8d73e3ccb", size = 8086364, upload-time = "2026-10-27T00:39:12.052Z" }, - { url = "https://files.pythonhosted.org/packages/9c/89/ece32e452d2b4ade243dd66a2612619be7400f6acf59f854ce674bb78e30/fensu_cli-0.28.0-py3-none-win_amd64.whl", hash = "sha256:acca6ac39cbe0614d3a8e4ce7c2c12cc5fd3c800fc4c66b5484c54dc19ef4a66", size = 7700477, upload-time = "2026-10-27T00:39:14.15Z" }, + { url = "https://files.pythonhosted.org/packages/ff/bb/09753cc0584c21af1d76eca7c9a6ea5e2d876d71c079c5505dda93742f3d/fensu_cli-0.28.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5179b032074f4ff5097950c0c0b063fd872cffcd23a4bf8a89c71ab0d3d8074b", size = 7963143, upload-time = "2026-11-01T09:41:13.908Z" }, + { url = "https://files.pythonhosted.org/packages/47/59/663fb61d56172bfb053375e8623a23db389b746052fb00dd087cf6c1a76b/fensu_cli-0.28.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:41077b9ec6e2eb46405af809d1010b26519c8928d4264137ca4c6d47ce81d2e1", size = 7479791, upload-time = "2026-11-01T09:41:16.253Z" }, + { url = "https://files.pythonhosted.org/packages/f7/7c/1a03bd25bc2a231ca03478ba3dfd8f21c909eae564c3df751a8982222beb/fensu_cli-0.28.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ac929d598046846c18eae62c8b75a9a49e9b6bbcadb46bd413a5da96ef6453e2", size = 7854377, upload-time = "2026-11-01T09:41:18.109Z" }, + { url = "https://files.pythonhosted.org/packages/17/5e/8395e4bd4e9e4d282a0949e42e721101127fb30f45cb71d6e5af05329051/fensu_cli-0.28.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9ce453889008381184a416acaf274ffa4a46c7cb96ad4397de264816bb10626a", size = 8316670, upload-time = "2026-11-01T09:41:20.045Z" }, + { url = "https://files.pythonhosted.org/packages/89/40/6e3831cd39b16f163916b36c03010754c1cfb58583fd737517bd544751d5/fensu_cli-0.28.3-py3-none-win_amd64.whl", hash = "sha256:66ec08187a6dcc43392244ba4b7349ee29e8da63fbe7110114468f07a000d577", size = 7710605, upload-time = "2026-11-01T09:41:22.073Z" }, ] [[package]] @@ -4346,7 +4346,7 @@ dev = [ { name = "dbt-snowflake", specifier = "==1.12.0b1" }, { name = "dbt-sqlserver", specifier = ">=1.9.0,<1.11" }, { name = "dlt", extras = ["duckdb", "filesystem", "sql-database"], specifier = ">=1.28.0,<2" }, - { name = "fensu", specifier = ">=0.28.0" }, + { name = "fensu", specifier = ">=0.28.3" }, { name = "ingestr", specifier = ">=1.1.73" }, { name = "jsonschema", specifier = ">=4.23.0" }, { name = "maturin", specifier = ">=1.9,<2" }, diff --git a/website/src/content/docs/docs/cli/login.mdx b/website/src/content/docs/docs/cli/login.mdx new file mode 100644 index 0000000000..f00d73ceb3 --- /dev/null +++ b/website/src/content/docs/docs/cli/login.mdx @@ -0,0 +1,64 @@ +--- +title: "sqb login and logout" +description: "Authorize a bounded CLI session through your server's browser sign-in." +sidebar: + label: "login / logout" +--- + +Sign in to a SQLBuild server using its configured password, SSO or organization authentication: + +```bash +sqb login --server https://control.example.com +``` + +Only authorize a request you started yourself. Compare the browser's verification code with +your terminal, sign in, and choose **Authorize CLI**. Return to the terminal for the completion +message. Choose **Cancel** to refuse the request. + +The browser confirmation expires after five minutes and can be exchanged once. The CLI receives +its own bounded person session, not your browser cookie or a personal API token. Your existing +permissions still apply; signing in does not create a person, assign a role or grant access. + +## Use a browser on another device + +```bash +sqb login --server https://control.example.com --no-browser +``` + +Open the printed URL in your browser and compare its verification code with the terminal. +Keep the terminal running until it confirms completion. If the request expires or is cancelled, +start `sqb login` again. + +## Check your saved session + +```bash +sqb login status --server https://control.example.com +``` + +This command verifies the saved session with the live server and prints only your person and +session identity as JSON. Disabled people, revoked sessions and password changes are enforced +by the server. If the session is unavailable, sign in again. + +## Sign out + +```bash +sqb logout --server https://control.example.com +``` + +Logout revokes the current CLI session and removes its local credential. It does not sign your +browser out. If the server cannot be reached, local cleanup still runs and the command reports +that remote revocation could not be confirmed. + +Credentials use your OS credential store. On headless Linux without a desktop session, an +unavailable credential service permits a private fallback under +`~/.config/sqlbuild/credentials`, restricted to your user. A locked or ambiguous OS credential +store is an error: unlock it and retry. Credentials are never printed or written into a project. + +| Option | Purpose | +| --- | --- | +| `--server HTTPS_ORIGIN` | Required server origin, without a path, query or embedded credentials | +| `--ca-cert PEM` | Trust an additional CA certificate for this server; certificate and hostname verification remain enabled | +| `--no-browser` | Print the confirmation URL without opening a browser; applies to sign-in | + +Lifecycle messages go to stderr. Successful login and logout leave stdout empty; status writes +identity JSON to stdout. A failure exits nonzero and never prints a credential. diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx new file mode 100644 index 0000000000..3f74aecbb0 --- /dev/null +++ b/website/src/content/docs/docs/cli/server.mdx @@ -0,0 +1,86 @@ +--- +title: "sqb server" +description: "Run the control service and scheduler." +sidebar: + label: "server" +--- + +Run the control API and scheduler without loading project Python: + +```bash +sqb server +``` + +Use `sqb server --help` for service, authentication and launcher options. + +## Set retention + +Set how much history the server keeps: + +```bash +sqb server --statistics-retention-days 30 --audit-retention-days 730 --snapshot-retention-count 20 +``` + +| Data | Option | Default | Range | +| --- | --- | --- | --- | +| Hourly warehouse statistics | `--statistics-retention-days` | `90` | `1`–`65535` | +| Security audit records | `--audit-retention-days` | `400` | `1`–`65535` | +| Published snapshots per location and target | `--snapshot-retention-count` | `10` | `1`–`65535` | +| Skipped and covered schedule ticks | None | 30 days | Fixed | + +Statistics samples older than the horizon disappear from statistics reads at once and are then +deleted in the background. + +A security audit record older than the horizon is deleted only when it is no longer needed: + +- The webhook in `SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL`, if set, has received it. +- No retained run's retry depends on it. A manual job or rerun request keeps its record while + the run exists. +- It is not the newest record, so audit page cursors never see a reused position. + +Records kept past the horizon for these reasons are reported on stderr as +`Security audit retention: retained past horizon: N` whenever the count changes. A growing count +usually means the audit webhook is failing. + +The server keeps the newest published snapshots for each location and target. It deletes an older +snapshot only when no retained run, job, trigger, trigger change or location request uses it, and +deletes its node definitions and SQL text when no other snapshot shares them. Repeating or +interrupting this cleanup is safe. + +The scheduler deletes covered ticks older than 30 days. It keeps each trigger's latest tick and +every tick that started a run. A new schedule, a resumed trigger or a cursor reset catches up only +on the latest missed time, so deleted ticks never come back. + +An invalid value stops the server before it opens the control database. Servers that share one +control database delete at the shortest horizon among them and count only their own audit +webhook, so give them the same values and webhook. + +## Inspect published SQL + +Sign in with [`sqb login`](/docs/cli/login/), then inspect one exact node in the current +successful publication: + +```bash +sqb control sql model:orders \ + --server https://control.example.com \ + --location-id "$LOCATION_ID" \ + --target development +``` + +The command uses your saved person session and prints JSON. It does not load a local project +or open a warehouse connection. Use `--ca-cert PATH` for an additional trusted PEM certificate. +The node key is exact: `model:orders+` does not expand a selector. Progress and failures go to +stderr; a failed request prints no SQL to stdout. Permission denials include their decision ID. + +The equivalent person-authenticated API is a `GET` request to +`/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/sql`. + +Developer or Admin access must cover the requested location and target. Viewer, Operator and +the configurable default Viewer role do not grant SQL inspection. The response contains the +snapshot ID, exact node key, raw SQL and compiled SQL. Missing SQL fields are `null`; an absent +current publication or node returns JSON `null` after authorization. + +SQL inspection does not execute SQL or return Python source, execution artifacts, logs or sample +data. Every successful inspection records an audit entry without copying SQL into it. If the +audit cannot be saved, the request fails without returning SQL. Revocation takes effect on the +next request, including sessions created by [`sqb login`](/docs/cli/login/). diff --git a/website/src/sidebar.json b/website/src/sidebar.json index c8f1f25bdf..8e33f35aac 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -200,6 +200,8 @@ "docs/cli/init", "docs/cli/playground", "docs/cli/skills", + "docs/cli/server", + "docs/cli/login", "docs/cli/compile", "docs/cli/format", "docs/cli/contract",