diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index e14eaee..2e6b8d3 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -29,7 +29,19 @@ jobs: path: deployment_assets/bin/linux-x86_64/* build-macos-intel: - runs-on: macos-13 # Intel runner + # `macos-15-intel`, not `macos-13`. + # + # The macOS 13 image was retired in December 2025. Every tagged release + # since has hung here: the job sat queued for the full 24-hour limit + # waiting for a runner that no longer exists, while linux, macos-arm and + # windows finished in 8-15 minutes, and `create-release` was skipped + # because it `needs:` all four. That is why v0.4.0 through v0.6.0 produced + # no GitHub release and no binaries - the workflow had never once + # succeeded. + # + # `macos-15-intel` is GitHub's documented replacement for workflows that + # genuinely need x86_64. + runs-on: macos-15-intel steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable @@ -98,8 +110,13 @@ jobs: steps: - uses: actions/download-artifact@v4 - name: Create Release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: + body: | + See [CHANGELOG.md](https://github.com/AlexMercedCoder/Pangolin/blob/main/CHANGELOG.md) + for the full list of changes, and + [SECURITY.md](https://github.com/AlexMercedCoder/Pangolin/blob/main/SECURITY.md) + for the security advisory and upgrade steps. files: | linux-binaries/* macos-intel-binaries/* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c66eba9..063a4c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,14 @@ env: CARGO_TERM_COLOR: always # Keep the debug artefacts small; CI does not need line tables. CARGO_PROFILE_TEST_DEBUG: 0 - RUSTFLAGS: -D warnings + # No `RUSTFLAGS: -D warnings` here. It used to be, and it was a trap rather + # than a gate: the tree still carries a warning backlog, so every job that + # compiles Rust had to override it with `RUSTFLAGS: ""` - and the two that + # forgot (`guardrails` and `mongo-replica-set`) failed at *compile*, so the + # authz matrix, the parity suite and every MongoDB test they were added to + # run had never once executed. A guard that cannot run is worse than no + # guard, because it reads as protection. `clippy-ratchet` is the real gate on + # warnings; it counts them and fails if the count grows. defaults: run: @@ -55,8 +62,6 @@ jobs: # `clippy-ratchet` below — and it flips to blocking once the backlog is # cleared (Phase 2.2). - run: cargo clippy --workspace --all-targets - env: - RUSTFLAGS: "" clippy-ratchet: name: clippy warning count @@ -71,8 +76,6 @@ jobs: workspaces: pangolin - run: sudo apt-get update && sudo apt-get install -y pkg-config libssl-dev - name: Count warnings and compare against the budget - env: - RUSTFLAGS: "" run: | # `--message-format short` emits one `path:line:col: warning: ...` # line per finding, plus per-crate summary lines that start with @@ -125,14 +128,11 @@ jobs: done - name: cargo test --workspace - env: - RUSTFLAGS: "" run: cargo test --workspace --no-fail-fast - name: cargo test --workspace (against live databases) if: matrix.services env: - RUSTFLAGS: "" PANGOLIN_TEST_POSTGRES_URL: postgresql://testuser:testpass@localhost:5432/testdb PANGOLIN_TEST_MONGO_URL: mongodb://testuser:testpass@localhost:27017 # MongoDB parity is still incomplete; see the Known limitations section @@ -145,10 +145,86 @@ jobs: steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - - uses: rustsec/audit-check@v2.0.0 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: pangolin + - run: cargo install cargo-audit --locked + # Run from `pangolin/`, not through `rustsec/audit-check`. + # + # That action invokes `cargo audit --file pangolin/Cargo.lock` from the + # repository root, and `cargo audit` reads its config from + # `.cargo/audit.toml` relative to the *current directory* only - it does + # not walk up the tree. So the exceptions in `pangolin/.cargo/audit.toml` + # were silently not applied and the job failed on advisories that had + # already been reviewed and justified. + # + # Duplicating the config at the repository root would fix CI and leave a + # developer running `cargo audit` in `pangolin/` - where the workspace + # is - seeing a different answer from the one CI sees. This job now runs + # the identical command in the identical directory instead (the + # workflow's `defaults.run.working-directory` is `pangolin`). + - run: cargo audit + + # Every optional feature must compile. + # + # None of them did. `cargo build` and `cargo test` run with default features, + # and no job ever passed `--features`, so the entire cloud-credential surface + # - `aws-sts`, `azure-oauth`, `gcp-oauth` and the `cloud-credentials` bundle + # that unions them - had rotted into code that could not be built at all: + # parameters bound as `_name` and then referenced as `name` inside the `cfg` + # block, a missing `anyhow!` import, an `expiration()` that returns a + # DateTime being fed to an RFC3339 string parser. For a catalog whose job + # includes vending scoped cloud credentials, that is the feature set, and + # nothing was watching it. + features: + name: optional features build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + feature: + - aws-sts + - azure-oauth + - gcp-oauth + - cloud-credentials + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: pangolin + - run: sudo apt-get update && sudo apt-get install -y pkg-config libssl-dev + - run: cargo check -p pangolin_api --features ${{ matrix.feature }} --all-targets + - name: The store's own optional backends must build too + if: matrix.feature == 'cloud-credentials' + run: cargo check -p pangolin_store --features azure,gcp --all-targets + + # The declared MSRV must be true, not aspirational. + # + # `rust-version` is a promise to consumers, and every other job runs + # `stable`, so nothing checked it. It was 1.92 and is now 1.94 - raised + # deliberately to pick up the AWS SDK releases that carry fixed `aws-lc-sys` + # and `rustls-webpki`, which is what cleared the certificate-validation + # advisories. + msrv: + name: minimum supported rust version + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Read rust-version from the workspace manifest + id: msrv + run: | + v=$(grep -m1 '^rust-version' Cargo.toml | sed 's/.*"\(.*\)".*/\1/') + echo "version=$v" >> "$GITHUB_OUTPUT" + echo "declared MSRV: $v" + - uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ steps.msrv.outputs.version }} + - uses: Swatinem/rust-cache@v2 with: - token: ${{ secrets.GITHUB_TOKEN }} - working-directory: pangolin + workspaces: pangolin + - run: sudo apt-get update && sudo apt-get install -y pkg-config libssl-dev + - run: cargo check --workspace --all-targets helm: name: helm lint @@ -198,6 +274,44 @@ jobs: tags: pangolin-api:ci cache-from: type=gha cache-to: type=gha,mode=max + # The server must outlive its own shutdown grace period. + # + # `PANGOLIN_SHUTDOWN_GRACE_SECS` is meant to bound the *drain* - the + # window between SIGTERM and giving up on in-flight requests. It was + # briefly implemented as `timeout(grace, serve)`, which bounds the whole + # server instead: the process exited cleanly `grace` seconds after + # startup, with no signal, and every container would have crash-looped. + # Nothing caught it, because no test ran the binary for longer than the + # default 25s. This runs the real image with a short grace and asserts it + # is still serving well past it. + - name: The server must not exit when its grace period elapses + run: | + set -euo pipefail + docker run -d --name grace-probe -p 8181:8080 \ + -e PANGOLIN_JWT_SECRET=ci-only-secret-of-adequate-length-000000 \ + -e PANGOLIN_STORAGE_TYPE=memory \ + -e PANGOLIN_SHUTDOWN_GRACE_SECS=5 \ + pangolin-api:ci + for _ in $(seq 1 30); do + if curl -sf http://localhost:8181/health/ready >/dev/null 2>&1; then break; fi + sleep 1 + done + curl -sf http://localhost:8181/health/ready >/dev/null + echo "serving; now waiting 20s, four times the grace period" + sleep 20 + if ! docker ps --filter name=grace-probe --filter status=running -q | grep -q .; then + echo "::error::the server exited on its own $(echo 20)s after startup with a 5s grace" + docker logs grace-probe | tail -20 + exit 1 + fi + curl -sf http://localhost:8181/health/ready >/dev/null + echo "still serving after 20s" + # And it must still stop promptly when actually told to. + start=$(date +%s) + docker stop -t 30 grace-probe >/dev/null + echo "stopped in $(( $(date +%s) - start ))s" + docker rm grace-probe >/dev/null + - name: The container must not run as root run: | user=$(docker inspect --format '{{.Config.User}}' pangolin-api:ci) @@ -206,3 +320,413 @@ jobs: echo "::error::the image runs as root" exit 1 fi + + # Roadmap improvements #0 and #1: the two suites that make whole classes of + # finding *testable* rather than reviewable. + # + # * the permission matrix drives each sensitive route as Root / + # TenantAdmin / TenantUser / a foreign tenant's admin and asserts the + # expected 200 or 403. A handler that forgets `check_permission` is + # indistinguishable from one that calls it until something asserts the + # 403 - which is how the B0a-B0m cluster survived a security release. + # * the cross-backend parity suite runs the same assertions against all + # four stores, which is the only thing that catches one backend quietly + # disagreeing with the others. + # + # Called out as their own job so a failure here reads as "authorization or + # backend parity regressed", not as an anonymous test failure. + guardrails: + name: authz matrix + backend parity + runs-on: ubuntu-latest + services: + # The parity suite is only meaningful against real backends. Running it + # on memory and SQLite alone is what let four cross-backend defects reach + # a release. + postgres: + image: postgres:15-alpine + env: + POSTGRES_USER: testuser + POSTGRES_PASSWORD: testpass + POSTGRES_DB: testdb + ports: ["5432:5432"] + options: >- + --health-cmd "pg_isready -U testuser" + --health-interval 5s --health-timeout 5s --health-retries 10 + mongo: + image: mongo:7.0 + env: + MONGO_INITDB_ROOT_USERNAME: testuser + MONGO_INITDB_ROOT_PASSWORD: testpass + ports: ["27017:27017"] + options: >- + --health-cmd "mongosh --eval 'db.adminCommand(\"ping\")'" + --health-interval 5s --health-timeout 5s --health-retries 10 + # MinIO is not a service container - see the step that starts it below. + env: + S3_ENDPOINT: http://localhost:9000 + AWS_ENDPOINT_URL: http://localhost:9000 + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + AWS_REGION: us-east-1 + AWS_ALLOW_HTTP: "true" + # Without this the S3 client probes 169.254.169.254 for instance + # credentials and burns the retry budget before failing. + AWS_EC2_METADATA_DISABLED: "true" + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + + # The store compliance tests exercise file IO. Without an object store + # they fall through to the EC2 instance-metadata endpoint and spend ~11s + # timing out before failing, which reads as a code failure rather than a + # missing service. + # + # This is a step rather than a `services:` entry because `minio/minio` + # needs the `server /data` argument and service containers cannot + # override a command. The image that does not - `bitnami/minio`, whose + # MINIO_DEFAULT_BUCKETS created the buckets for free - was withdrawn from + # Docker Hub, so the service definition that used to be here no longer + # resolves. + - name: Object storage for the file-IO tests + run: | + set -euo pipefail + docker run -d --name minio -p 9000:9000 \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + minio/minio server /data + for _ in $(seq 1 30); do + if curl -sf http://localhost:9000/minio/health/live >/dev/null; then break; fi + sleep 2 + done + curl -sf http://localhost:9000/minio/health/live >/dev/null + # `warehouse` is the application's bucket; the other two are test + # fixtures. A missing one fails as `NoSuchBucket`, which the + # compliance test reports as "File IO should be supported". + docker run --rm --network host --entrypoint sh minio/mc -c ' + mc alias set m http://localhost:9000 minioadmin minioadmin + for b in warehouse bucket test-bucket; do mc mb --ignore-existing "m/$b"; done + ' + + - name: Permission matrix + run: cargo test -p pangolin_api --lib authz_matrix_tests -- --nocapture + - name: Cross-backend parity (all four backends) + env: + PANGOLIN_TEST_POSTGRES_URL: postgresql://testuser:testpass@localhost:5432/testdb + # A standalone mongod supports neither transactions nor retryable + # writes; the code degrades for the former and this disables the + # latter. A replica set would exercise both paths and is what + # production should use - see docs/operations/backend-parity.md. + PANGOLIN_TEST_MONGO_URL: mongodb://testuser:testpass@localhost:27017/?retryWrites=false + run: cargo test -p pangolin_store --test store_integration + + - name: The parity suite must not have skipped a backend + env: + PANGOLIN_TEST_POSTGRES_URL: postgresql://testuser:testpass@localhost:5432/testdb + PANGOLIN_TEST_MONGO_URL: mongodb://testuser:testpass@localhost:27017/?retryWrites=false + run: | + # The suite skips a backend with a printed note when its URL is + # unset, which passes. That is right for a developer laptop and wrong + # for CI: a silently skipped backend is how Postgres and MongoDB went + # untested through an entire security release. Assert all four ran. + out=$(cargo test -p pangolin_store --test store_integration 2>&1) + echo "$out" + for backend in memory sqlite postgres mongo; do + if ! grep -q "test test_${backend}_store_regression \.\.\. ok" <<<"$out"; then + echo "::error::the ${backend} backend did not run" + exit 1 + fi + done + if grep -qiE "^skipping:" <<<"$out"; then + echo "::error::a backend was skipped" + exit 1 + fi + echo "all four backends ran" + + # A standalone `mongod` supports neither transactions nor retryable writes, so + # the `guardrails` job above exercises only MongoDB's *degraded* paths. That + # is the topology in which `delete_catalog`'s fallback was found to be broken + # - and the transactional branch it falls back *from* has correspondingly + # never run. Testing one branch and not the other is the same gap one layer in. + # + # A single-node replica set is enough: a set of one supports transactions and + # retryable writes, which is the whole distinction. It is not a model of + # production redundancy and is not meant to be. + mongo-replica-set: + name: mongodb (replica set) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + + - name: Start a single-node replica set + run: | + # Not a `services:` container: the set has to be initiated after the + # process is listening, and a service container's healthcheck has no + # hook for that. + docker run -d --name mongo-rs -p 27018:27018 mongo:7.0 \ + mongod --replSet rs0 --bind_ip_all --port 27018 + + for i in $(seq 1 30); do + if docker exec mongo-rs mongosh --port 27018 --quiet --eval \ + "rs.initiate({_id:'rs0',members:[{_id:0,host:'localhost:27018'}]})" \ + >/dev/null 2>&1; then + break + fi + sleep 2 + done + + # Wait for a primary. Accepting connections is not the same as being + # able to serve a write, and a client that connects in between fails + # in ways that look like application bugs. + for i in $(seq 1 30); do + state=$(docker exec mongo-rs mongosh --port 27018 --quiet \ + --eval "try { rs.status().myState } catch (e) { 0 }" 2>/dev/null | tr -d '[:space:]') + if [ "$state" = "1" ]; then + echo "replica set has a primary" + exit 0 + fi + sleep 2 + done + echo "::error::the replica set never elected a primary" + docker logs mongo-rs | tail -50 + exit 1 + + - name: The set must actually support transactions + run: | + # Otherwise this job silently re-tests the standalone paths and the + # whole point of it is lost. + docker exec mongo-rs mongosh --port 27018 --quiet --eval ' + const s = db.getMongo().startSession(); + s.startTransaction(); + s.getDatabase("probe").c.insertOne({ok: 1}); + s.commitTransaction(); + print("transactions available"); + ' + + - name: MongoDB suites against a replica set + env: + PANGOLIN_TEST_MONGO_URL: mongodb://localhost:27018/?replicaSet=rs0&directConnection=true + PANGOLIN_TEST_MONGO_DB: pangolin_rs_test + run: | + cargo test -p pangolin_store --test store_integration test_mongo + cargo test -p pangolin_store --test mongo_comprehensive_tests + cargo test -p pangolin_store --test mongo_audit_tests + cargo test -p pangolin_store --test mongo_tests + cargo test -p pangolin_store --test mongo_uuid_round_trip_tests + + - name: The suites must not have skipped MongoDB + env: + PANGOLIN_TEST_MONGO_URL: mongodb://localhost:27018/?replicaSet=rs0&directConnection=true + PANGOLIN_TEST_MONGO_DB: pangolin_rs_test + run: | + out=$(cargo test -p pangolin_store --test store_integration test_mongo 2>&1) + echo "$out" + grep -q "test test_mongo_store_regression \.\.\. ok" <<<"$out" || { + echo "::error::the MongoDB backend did not run" + exit 1 + } + + # Deployment artefacts drift from the code that reads them, and nothing + # noticed. `PANGOLIN_STORE_TYPE` sat in both compose files for several + # releases while the server read `PANGOLIN_STORAGE_TYPE` (B9); the quick-start + # compose file could not start the API at all because it set no signing secret + # (B8); and the release compose file pinned an image four versions old and ran + # a verification script that did not exist (B10). All three are the same + # failure: config that is never validated. + config-drift: + name: config drift + runs-on: ubuntu-latest + defaults: + run: + working-directory: . + steps: + - uses: actions/checkout@v4 + + - name: Every compose file must be valid + env: + PANGOLIN_JWT_SECRET: ci-only-secret-of-adequate-length-0000000 + # docker-compose.release.yml takes RELEASE_* rather than PANGOLIN_* + # so that a developer's local .env cannot feed the release + # verification harness. They are required (`:?`), so `config` cannot + # render the file without them. + RELEASE_JWT_SECRET: ci-only-secret-of-adequate-length-0000000 + RELEASE_ROOT_PASSWORD: ci-only-root-password-not-a-placeholder + run: | + for f in docker-compose*.yml; do + echo "validating $f" + docker compose -f "$f" config > /dev/null + done + + - name: The quick start must fail loudly without a signing secret + run: | + # `docker compose config` must *fail* here: the `:?` form on + # PANGOLIN_JWT_SECRET is what turns a crash-looping container into an + # immediate, explicable error. + if docker compose -f docker-compose.yml config > /dev/null 2>&1; then + echo "::error::docker-compose.yml accepted an unset PANGOLIN_JWT_SECRET" + exit 1 + fi + echo "compose correctly refuses to start without a signing secret" + + - name: No compose file may set a variable the server does not read + run: | + # PANGOLIN_STORE_TYPE is the specific name that survived for releases; + # the general check below catches its successors. + # Comment lines are excluded: two compose files carry the offending + # name verbatim while explaining why it is wrong. The previous + # `grep -v 'not read'` was meant to do this and matched neither + # comment ("was renamed", "which nothing reads"), so this guard + # failed on its own documentation. + if grep -rn 'PANGOLIN_STORE_TYPE' docker-compose*.yml deployment_assets 2>/dev/null \ + | grep -vE ":[[:space:]]*#"; then + echo "::error::PANGOLIN_STORE_TYPE is not read by the server; use PANGOLIN_STORAGE_TYPE" + exit 1 + fi + # Comment lines are stripped first, for the same reason as above: + # several compose comments name a variable precisely to explain that + # it is *not* read, and the guard would flag its own documentation. + # + # Two names are legitimately not read by the server: + # PANGOLIN_VERSION - a compose-level image tag, not an app setting. + # PANGOLIN_API_URL - set on the `tests` service in + # docker-compose.release.yml and read by + # scripts/release_smoke_test.py, which is a Python script rather + # than the Rust server this guard greps. (It previously needed an + # exemption for a worse reason: nothing read it at all, because + # the script that service ran could not run in that container.) + exempt='PANGOLIN_VERSION|PANGOLIN_API_URL' + names=$(grep -rhE '^[^#]*PANGOLIN_[A-Z0-9_]+' docker-compose*.yml \ + | sed 's/#.*//' | grep -ohE 'PANGOLIN_[A-Z0-9_]+' | sort -u) + for name in $names; do + if echo "$name" | grep -qE "^($exempt)$"; then continue; fi + if ! grep -rq "$name" pangolin/pangolin_api/src pangolin/pangolin_store/src; then + echo "::error::$name appears in a compose file but no code reads it" + exit 1 + fi + done + echo "every PANGOLIN_* name in the compose files is read by the server" + + - name: The environment-variable reference must match the code + working-directory: pangolin + run: ./scripts/check_env_var_docs.sh + + - name: Every artifact must carry the same version + run: | + # Improvement #8. The "one version everywhere" property introduced in + # 0.6.0 had already drifted a day later in two places, because it was + # maintained by hand across five files. + version=$(grep -oE '^version = "[0-9]+\.[0-9]+\.[0-9]+[^"]*"' pangolin/Cargo.toml \ + | grep -oE '[0-9]+\.[0-9]+\.[0-9]+[^"]*') + echo "workspace version: $version" + ./scripts/bump_version.sh "$version" --check + + # Improvement #2: the SDK and UI had no CI at all - only Rust, Helm and + # Docker were covered. That is exactly where the 0.6.0 "no CI, silent rot" + # lesson was repeating: the SDK's test suite could not even *collect* + # (B41) and the UI's vitest config broke every suite that imported the API + # client (B46), and nothing noticed either. + sdk: + name: python sdk + runs-on: ubuntu-latest + defaults: + run: + working-directory: pypangolin + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install + run: | + python -m pip install --upgrade pip + pip install -e '.[iceberg]' + # Pinned deliberately: `pip install ruff` picked up 0.16, whose + # wider default rule set produced 316 findings in a package that had + # opted into ruff's defaults as they stood. Adopting the new rules is + # a separate, deliberate change - not something a runner should decide + # on the day it happens to run. + pip install pytest 'ruff==0.15.19' + - name: The package must import without the Iceberg extra + run: | + # B41 regression: `pip install pypangolin` then `import pypangolin` + # must not require the whole Iceberg stack. + pip uninstall -y pyiceberg + python -c "import pypangolin; print(pypangolin.__version__)" + - name: The version must match the package metadata + run: | + # B38: `__version__` was hardcoded to 0.1.0 against a 0.6.0 package. + python - <<'PY' + from importlib.metadata import version + import pypangolin + assert pypangolin.__version__ == version("pypangolin"), ( + f"__version__ {pypangolin.__version__} != metadata {version('pypangolin')}" + ) + print("version is single-sourced") + PY + - run: ruff check src + - run: pytest -q + + ui: + name: management ui + runs-on: ubuntu-latest + defaults: + run: + working-directory: pangolin_ui + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + - run: npm ci + - name: The UI must build + run: npm run build + + # This job built the UI and never ran its tests, so the suite drifted to + # 40 failures across 14 files without anything noticing - including tests + # that could not pass at all because the global setup mocked the very + # module they were testing. + - name: The UI test suite must pass + run: npm test + + # `svelte-check` reports a backlog inherited from before this job + # existed, so it is a ratchet rather than a gate - the same arrangement as + # `clippy-ratchet` above, and it flips to blocking once the backlog is + # cleared. + - name: Count type errors and compare against the budget + run: | + count=$(npm run check 2>&1 | grep -cE "^Error:" || true) + budget=$(cat svelte-check-budget.txt) + echo "svelte-check errors: $count (budget: $budget)" + if [ "$count" -gt "$budget" ]; then + echo "::error::svelte-check errors rose from $budget to $count." + exit 1 + fi + if [ "$count" -lt "$budget" ]; then + echo "::notice::svelte-check errors fell to $count; lower svelte-check-budget.txt." + fi + + - name: No raw /api/v1 fetches may reappear + run: | + # B32: a relative `fetch('/api/v1/...')` works under the dev proxy and + # 404s in production, while also skipping the tenant header. Every + # call must go through apiClient, which is not something a type check + # can express. + # Comment lines are excluded: several carry the offending pattern + # verbatim while explaining why it is wrong. + if grep -rnE "fetch\((['\`\"])/api/v1" src/ | grep -vE ":[[:space:]]*(//|\*|/\*)"; then + echo "::error::raw /api/v1 fetch found; route it through apiClient" + exit 1 + fi + echo "no raw /api/v1 fetches" + - name: Only PUBLIC_API_URL may name the API base URL + run: | + # B31: four different spellings coexisted and none agreed, so every + # deployed build called the visitor's own localhost. + if grep -rn "VITE_API_URL" src/ .env.example vitest.config.ts 2>/dev/null \ + | grep -vE ":[[:space:]]*(//|#|\*|/\*)"; then + echo "::error::VITE_API_URL is not read by SvelteKit; use PUBLIC_API_URL" + exit 1 + fi + echo "PUBLIC_API_URL is the only spelling" diff --git a/.gitignore b/.gitignore index fc9a6d0..0f4410d 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,14 @@ check_log.txt test_results.txt debug_output*.txt :memory: + +# Editor backup copies of source files (B45). +# +# `pangolin_store/src/memory.rs.bak` and `mongo.rs.bak` sat in the tree as ~4k +# lines of divergent, dead query copies - a grep trap that returned plausible +# but stale code for anyone searching the store layer. +*.bak +*.rs.bak + +# Logs live under logs/ directories that were previously tracked. +logs/ diff --git a/CHANGELOG.md b/CHANGELOG.md index fab1c57..897d10a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,400 @@ From 0.6.0 the server, both CLIs, the Python SDK, the UI and the Helm chart all carry the same version number. Before that they had drifted to five different values and there was no way to tell which combination had been tested together. +## [0.7.0] — 2026-08-10 + +Implements `roadmap_aug10.md`, the full-repo audit of 2026-08-10. **This is a +security release.** Five of the fixes below are exploitable by any +authenticated principal, including the lowest-privilege tenant user and any +service-user API key. If you run 0.6.0, upgrade and rotate tokens. + +### Security + +- **Any authenticated caller could mint a `Root` JWT for any tenant (B0a).** + `POST /api/v1/tokens` took no session at all and mapped a body-supplied + `roles: ["Root"]` straight into signed claims. Since `check_permission` + short-circuits for `Root`, this was a total privilege escalation reachable by + a tenant user. Minting is now restricted to `Root`, or to a `TenantAdmin` + within its own tenant and never above its own rank. +- **Any tenant member could vend read+write cloud credentials for the whole + warehouse (B0b).** The credential endpoint performed no authorization, never + looked the table up, and hardcoded `["read", "write"]` - so it issued + credentials for tables the caller had no rights to and that need not exist. + It now resolves the asset, requires `Read`, and adds `"write"` only when + `Write` is actually held. +- **Logout did not revoke anything (B0j).** Revocation is keyed by the token's + `jti`; the handler revoked `session.user_id`, which no token ever carries as + its `jti`. Logout returned 200 and the token kept working for its full + 24-hour lifetime. `UserSession` now carries the `jti`. +- **An expired service user could renew indefinitely (B0g).** The Iceberg OAuth + token endpoint checked `active` but not expiry, so an expired API key could + still exchange `client_credentials` for a fresh JWT - bypassing key expiry + entirely, and renewably. +- **`PANGOLIN_DEV_MODE` waived the `NO_AUTH` public-bind guard (B0h).** The two + flags are routinely set together in compose and dev setups, and together they + started a server on `0.0.0.0` that treated every anonymous request as + `TenantAdmin`. Dev mode now relaxes secret strength only, never exposure. +- **A tenant-wide grant applied across tenants (B0i).** `PermissionScope::Tenant` + matched without comparing the grant's tenant to the resource's. +- **OAuth linked accounts by unverified email (B0l).** Anyone who could set a + matching address on any configured provider - GitHub reports unverified ones - + logged in as that Pangolin user, including the seeded tenant admin. Identity + is now `(provider, subject)`; email linking needs a verified address and an + operator domain allowlist. +- **The OAuth login flow could not complete (B0k).** `POST /api/v1/oauth/exchange` + was not in the public-path allowlist, so the endpoint whose job is to issue + the first token demanded one. The browser landed with a `?code=` it could + never redeem. +- Missing authorization on `rename_table` (B0c), `update_namespace_properties` + (B0d), view create/read (B0e), `perform_maintenance` (B0f), `rebase_branch` + and `delete_business_metadata`. `perform_maintenance` additionally ran + destructive snapshot expiry against a hardcoded `"default"` catalog rather + than the one in the path. +- A caller-supplied `expires_in_hours` could panic token issuance and abort the + connection task (B0m); clamped, plus a `CatchPanicLayer`. +- A malformed or absent `jti` skipped the revocation check entirely, making such + tokens unrevocable for their lifetime (B0o). +- The admin CLI and Python SDK wrote their auth tokens world-readable, and + `generate-code` / `get-token` echoed live JWTs into copy-paste output + (B_cli7, B_sdk4). +- A live PyPI API token was removed from the repo-root `.env` (B44). **It was + present in plaintext and must be rotated.** + +### Fixed + +- **Storage backends disagreed with each other in ten ways (B1-B7, B17-B30).** + A cross-tenant audit read on Mongo, a revocation that was a silent no-op on + Mongo, a SQLite branch delete that orphaned its assets and referenced a + column that does not exist, a Postgres search that panicked on any hit, a + Mongo compare-and-swap that lost Iceberg snapshots, a memory index that + broke another tenant's lookups, and all three persistent backends silently + rewriting 15 of the 17 asset types to `IcebergTable`. Plus pagination that + could repeat or skip rows everywhere, and four different answers to the same + search. +- **Two defects the new parity suite found on its first run.** `SqliteStore` + had no inherent `revoke_token`/`is_token_revoked`, so the trait delegations + called themselves - revoking a token on SQLite recursed until the stack was + exhausted and *aborted the process*. And the SQLite `audit_logs` table still + declared its original column set while the code inserted the full entry, so + every audit write failed and the backend kept no audit trail at all. +- **Iceberg metadata was not spec-conformant (B11-B16o).** `default-spec-id` + was written under the wrong name, the required `last-partition-id` was + absent, schemas omitted `"type": "struct"`, and `metadata-log` was never + appended - so metadata Pangolin wrote could not be read as v2 metadata by an + external engine. On the commit path: nested namespaces registered under one + key and looked up under another (every commit to one 404'd), a client could + jump the sequence counter to `i64::MAX` and overflow the next commit, a + feature-branch commit moved `main`, `last-updated-ms` only advanced on + snapshots, `-1` resolved against the whole list rather than what the commit + added, `create_table` returned the table directory as `metadata-location` + and dropped every complex-typed column from the schema, and lost + compare-and-swaps orphaned metadata files. +- **`docker compose up` could not start the API (B8-B10).** No signing secret + was set and the server has refused to start without one since 0.6.0. Both + compose files also set a storage variable nothing reads, and the release + compose file pinned an image four versions old and ran a script that does + not exist. +- **The management UI was disconnected from the server (B31-B37).** Four + spellings of the API base URL coexisted and none agreed, so every deployed + build called the visitor's own localhost; ~13 raw `fetch('/api/v1/...')` + calls 404'd outside the dev proxy and skipped the tenant header; three + endpoints the UI called did not exist; nothing handled a 401, so an expired + token left a permanently broken session; tag-filtered search 400'd end to + end; there was no way to create a catalog from the catalogs page; and the + tenant switcher was dead code referencing an unimported store. +- **Both CLIs and the SDK called endpoints that do not exist (B_cli1-8, + B_sdk1-5).** Roughly 35 sites: wrong paths, wrong field names, wrong types, + commands that were `Ok(())` stubs reporting success. All of it survived + because every command swallowed its error and exited 0. + +### Fixed — found by running the suites against live databases + +The parity suite was written against memory and SQLite, the two backends CI +could run without a service container. Pointing it at a live PostgreSQL and +MongoDB for the first time failed on both. None were regressions; all had been +present for as long as the code had. + +- **PostgreSQL: asset search was broken outright.** No migration ever created + `business_metadata`, while `search_assets` joined it — so every search failed + with `relation "business_metadata" does not exist`, a hard SQL error rather + than an empty result. The three CRUD methods were unimplemented, so the + trait's "Operation not supported by this store" default answered them. Added + the migration and the implementation. +- **MongoDB: role assignments were unreadable.** `bson::to_document` writes a + `Uuid` as a string while the deserializer expects BSON Binary, so + `assign_role` wrote documents that `get_user_roles` could never match and + that could not be deserialized at all. Every role-derived permission silently + vanished: **a user holding an admin role was authorized as though they held + none.** The same asymmetry caused B1 and B2 in two other collections; one + helper now covers all of them. +- **MongoDB: `get_metadata_location` had no fallback** to the asset's own + `location`, unlike the other three backends. A table created with a location + but no explicit metadata-location property reported none, so its metadata + could not be loaded and its commits compared against a different value than + the read path returned. +- **MongoDB: the "no transaction support" fallback was unreachable.** + `start_transaction` is a local call in the Rust driver and cannot fail for + want of a replica set; the error arrives on the first operation *inside* the + transaction and was propagated rather than caught. `delete_catalog` failed + outright on any standalone `mongod` instead of degrading as its own comment + promised. +- **SQLite: the `audit_logs` fix did not reach existing databases.** The schema + file is written with `CREATE TABLE IF NOT EXISTS`, which does nothing when the + table already exists — so fresh installs got the corrected columns and every + upgraded database kept the broken ones, with a bumped version number now + claiming otherwise. Added a real v1→v2 migration, keyed off table + introspection rather than the recorded version, with the old table preserved + as `audit_logs_pre_v2`. + +### Fixed — the MongoDB UUID encoding audit + +The string/Binary asymmetry above had by then been fixed four times, in four +collections, each time as its own bug. Auditing every collection at once — with +a round-trip test per entity rather than per feature — found four more, and a +second encoding disagreement nobody had noticed. + +There are three ways this codebase converts a `Uuid` to BSON and they all differ: +`to_bson_uuid` gives Binary with the generic subtype, `doc! { "k": uuid }` gives +Binary with the *UUID* subtype, and `bson::to_document` gives a string. Reads +disagree too: a typed `Collection` demands binary, `bson::from_bson` demands a +string. A write and a read chosen independently agree only by luck, and when they +do not, nothing fails loudly — the filter just matches nothing. + +- **Every service-user method was a no-op.** `create_service_user` let Mongo + generate an `ObjectId` while the four by-id methods filtered on + `{"_id": }`, which matched nothing; the tenant listing and the + API-key lookup used snake_case field names for a kebab-case struct; and + `update_service_user_last_used` wrote to a field no reader looks at. The + consequence that matters: **API-key authentication could never resolve a + service user on MongoDB.** It fails closed, so this was an outage of + service-user auth rather than a bypass. Changing a role also wrote the Rust + variant name instead of its serde form, making the record unreadable + afterwards. +- **Business metadata could be written but never read.** Only `asset-id` was + rewritten as Binary; `id`, `created-by` and `updated-by` kept the string form, + so `get_business_metadata` failed on the first of them. Writing metadata made + an asset's metadata permanently unreadable. +- **Listing active tokens failed outright.** `store_token` writes timestamps as + BSON DateTime, which `bson::from_bson::>` rejects — chrono wants + an RFC3339 string. The `created_at` arm swallowed the same error and + substituted `now()`, so even without the hard failure every token would have + reported the listing time as its creation time. +- **A branch with a head commit could not be read.** `create_branch` writes the + head through `doc!` (Binary, UUID subtype) and the reader accepted only a + string. A freshly created branch has no head, so this only bit once a branch + had been committed to — which is why it survived every existing test. + +`from_bson_uuid` now accepts all three encodings, so records already written by +any of them still load, while writes go through `to_bson_uuid` alone. +`mongo_uuid_round_trip_tests.rs` covers all 21 collections and runs against both +MongoDB topologies in CI. + +### Fixed — test environment drift + +- **`docker-compose.db-test.yml` had no object store.** The store compliance + tests exercise file IO; with no S3 they fall through to the EC2 + instance-metadata endpoint, hang for eleven seconds and fail with a + credentials error that names nothing relevant. CI had MinIO and the documented + local workflow did not, so the two disagreed about what it takes to run the + suite. +- **The MinIO image CI pulled no longer exists.** `bitnami/minio:latest` was + withdrawn from Docker Hub and now fails with `manifest unknown`. Both CI and + the compose file use `minio/minio` with an explicit bucket-creation step — + `warehouse` for the application, `bucket` and `test-bucket` for the compliance + tests, whose absence surfaces as `NoSuchBucket`. + +### Fixed — the management UI + +The UI job in CI built the app and never ran its tests, so the suite had drifted +to **40 failures across 14 files**. Most could not have passed: the global test +setup replaces `$lib/api/catalogs`, `$lib/api/warehouses`, `$lib/stores/auth`, +`$lib/stores/tenant` and `$lib/stores/notifications` with stubs, so the unit +tests *for those modules* were asserting against the stub rather than the code. +Others were asserting behaviour the app no longer had. Running them turned up +real defects underneath: + +- **A root user could not create another root user.** The role option's value + was `Root`; the server's `UserRole` is kebab-case, so the request was + rejected. The same PascalCase leftovers meant a tenant admin was shown an Edit + control for root users (`row.role !== 'Root'` never matched), and the role + badge colours on the users page keyed off `Root`/`TenantAdmin`, which the API + never returns. +- **A warehouse created in the UI showed no bucket in the UI.** The list page + read `s3.bucket`/`azure.container`; the create form writes plain + `bucket`/`container`. Both conventions are now accepted on read, as the server + already does. The warehouse table also rendered its Type column twice, in + place of the Region column its own template already had a branch for. +- **`production` is not a branch type.** The API knows exactly `ingest` and + `experimental`, but the UI's types declared `'experimental' | 'production'` + and the green badge keyed off `production` - so every branch rendered as + though it were experimental, and `ingest`, the type that carries a distinct + permission, had no representation at all. +- **A branch with no recorded parent was displayed as branching from `main`**, + claiming a lineage the data does not contain. +- **A local catalog could be created with no storage location and no + warehouse**, leaving it with nowhere to write its tables. The server accepts + it (the field is optional there, for federated catalogs), so nothing rejected + it. +- **The role select on the user edit page had no accessible name** - the label + named nothing - and its fallback value, `TenantUser`, matched none of the + options, so a user record without a role showed an empty select. +- **`logout()` could throw**, skipping the caller's redirect and stranding the + user on a page they were no longer authenticated for, if the token-revocation + call misbehaved. Its `void`/`.catch` pair only covered a rejected promise. +- Every unparameterised list call left a bare `?` on the end of its URL. + +`DataTable` moved from `createEventDispatcher` to callback props, with its six +consumers. That is the Svelte 5 idiom, and it is what makes the component +testable at all: `component.$on(...)` was removed in Svelte 5, so the row-click +test had been left as a stub that asserted nothing. + +CI now runs `npm test`, and carries a `svelte-check` error budget on the same +ratchet as the clippy one. `svelte-check` errors fell from 166 to 150 - mostly by +giving `StorageConfig` the index signature the server's free-form +`HashMap` always implied. + +Note that the app runs in Svelte 5's **legacy mode**: none of its 90 components +use runes. That is supported and works; converting them is a separate piece of +work and has not been done here. + +### Fixed — the cloud-credential features had never compiled + +Found while bumping dependencies: `cargo check -p pangolin_api --features +cloud-credentials` fails, and had been failing at every version. So did each of +`aws-sts`, `azure-oauth` and `gcp-oauth` individually. Nothing built with +`--features`, so nothing noticed. + +For a catalog whose job includes vending scoped, time-limited cloud +credentials, that is the feature set. Every deployment using it was running +without it. + +The errors were the kind that only appear when a `cfg` block is never +type-checked: + +- parameters bound as `_duration`, `_resource_path`, `_permissions` to silence + unused warnings in the default build, then referenced as `duration`, + `resource_path`, `permissions` inside the feature block — three files, five + bindings; +- `anyhow!` used in `gcp_signer.rs` with only `anyhow::Result` imported; +- `creds.expiration()` fed to `chrono::DateTime::parse_from_rfc3339`, but it + returns an `aws_smithy_types::DateTime`, not text — so the STS credential + expiry was parsed from a value that was never a string. + +`aws-sdk-sts` was also pinned to an exact `=1.50.0` with no comment. It was +protecting nothing — the feature failed identically at that version — and it +blocked `aws-config` from reaching a release that drops the second, vulnerable +TLS stack. Relaxed to `1.109`. + +A `features` CI job now builds each optional feature, and `pangolin_store`'s +`azure` and `gcp` backends alongside them. + +### Changed — minimum supported Rust version is now 1.94 + +Raised from 1.92 to pick up the AWS SDK releases carrying fixed `aws-lc-sys` +and `rustls-webpki`. That is what cleared the certificate-validation +advisories — two of them high severity, on the path Pangolin uses to reach S3, +Azure Blob Storage and GCS. + +`rust-version` is a promise to consumers and nothing verified it: every job ran +`stable`. An `msrv` job now reads the declared version out of the workspace +manifest and builds with exactly that toolchain, so the floor is checked rather +than asserted. `Dockerfile`, `README.md`, `CONTRIBUTING.md` and the deployment +guide are all in step. + +### Security — dependency advisories + +`cargo audit` reported 26 vulnerabilities, the one job still red after the CI +repair. Now zero, by a combination of upgrades and eight deliberate, +individually justified exceptions in `.cargo/audit.toml`. + +Cleared by upgrading: the `aws-lc-sys` cluster (including two high-severity +certificate-validation bypasses and a PKCS7 signature-validation bypass), +`rustls-webpki` name-constraint and CRL-parsing defects, `quinn-proto`, +`hickory-proto`, `bytes`, `crossbeam-epoch`, `time`. + +Accepted, with the reason recorded against each ID: `rsa`'s Marvin attack +(never compiled — `sqlx-mysql` is an optional dependency this workspace does +not enable), `quick-xml` (held by `object_store` 0.11 and by `azure_core` 0.20, +which is behind an optional feature), the second `rustls-webpki` copy that +arrives via the AWS SDK's `rustls` 0.21, `http-types`, and `rand`'s +custom-logger unsoundness. The ignore list names specific advisory IDs, so a +new advisory — including a new one against these same crates — still fails CI. + +### Fixed — a regression caught by the release gate + +Found by running the new release smoke test against the built 0.7.0 image, and +fixed before 0.7.0 shipped. + +- **The server exited 25 seconds after startup, having received no signal.** + The B16n change meant to bound the shutdown *drain* was written as + `tokio::time::timeout(shutdown_grace, serve)`. `serve` is the whole server, + not the drain, so it bounded the lifetime of the process: + `PANGOLIN_SHUTDOWN_GRACE_SECS` became a countdown to a clean exit rather than + a limit on how long draining may take. Every container would have + crash-looped. The deadline is now armed inside `shutdown_signal`, only once a + signal has actually been seen. + + All 18 CI jobs passed with this present, as did the full workspace suite: + nothing ran the binary for longer than the 25-second default. The `docker` + job now starts the built image with a 5-second grace, waits 20 seconds, and + fails if it is no longer serving - then checks it still stops promptly when + told to. + +- **Release verification inherited the developer's `.env`.** Compose auto-loads + it, so a local `PANGOLIN_ROOT_USER` / `PANGOLIN_ROOT_PASSWORD` fed the + harness - and where that password is a placeholder, the server's config guard + refuses to start, so verification failed for reasons having nothing to do + with the artifact. The harness now takes `RELEASE_*` names that cannot + collide with a real deployment's. + +- **The release stack shared a Compose project with the development database + stack**, both deriving `pangolin` from the directory name, so `up` and + `down -v` in one stopped containers belonging to the other. It now declares + `name: pangolin-release`, and its MinIO no longer publishes host ports it + never used. + +### Added + +- **A permission matrix test (improvement #0).** Drives each sensitive route as + Root / tenant admin / ungranted tenant user / foreign tenant admin and + asserts the expected 200 or 403. Every bug in the authorization cluster was + invisible to CI precisely because nothing asserted this. +- **A cross-backend parity suite (improvement #1).** Runs the same assertions + against memory, SQLite, Postgres and Mongo. Nearly half the storage findings + were one backend diverging from the others, which no per-backend test can + see. +- `#[serde(deny_unknown_fields)]` on all 34 server request structs, so a client + sending the wrong field name gets a 422 naming it rather than a 200 for a + request the server silently emptied. It caught five such payloads in the + project's own tests immediately. +- CI jobs for the Python SDK, the UI, configuration drift, and the two + guardrail suites above (improvements #1-#3). Previously CI covered only Rust, + Helm and Docker - which is exactly where the SDK and UI rot happened. +- `loadNamespaceMetadata`, `namespaceExists`, `DELETE /api/v1/branches/{name}` + and `GET /api/v1/oauth/providers`; spec pagination (`pageToken`/`pageSize` + and `next-page-token`) and the spec error envelope across the Iceberg + handlers. +- `scripts/check_env_var_docs.sh`, which regenerates the environment-variable + reference check from `config.rs`. The old page documented three variables + that do not exist and omitted 34 that do (B43). +- `scripts/bump_version.sh`, which sets the version across all five artifacts + and their inter-crate requirements, with a `--check` mode wired into CI + (improvement #8). The "one version everywhere" property had already drifted + in two places a day after the release that introduced it. +- CI now runs the parity suite against live PostgreSQL and MongoDB service + containers, and **fails if any backend was skipped**. A skipped backend + passing silently is how two of them went untested through a security release. +- An upgrade guide at `docs/upgrading/0.6-to-0.7.md`, and a security advisory + in `SECURITY.md`. + +### Removed + +- 18 unused UI runtime dependencies, ~260 KB of tracked debug output, two + ~4k-line `.bak` store monoliths that any grep of the storage layer would hit, + 27 `console.log` calls (one logging a bearer-token prefix), and an + unauthenticated file-read route in the UI with no callers and a + blacklist-based traversal guard. + ## [0.6.0] — 2026-08-09 **This is a security release.** If you are running any earlier version, upgrade diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fc1b77a..15f1881 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ recently, nothing at all for contributors — this file closes that gap. ## From clone to green -You need Rust 1.92 or newer and Docker (only for the database-backed tests). +You need Rust 1.94 or newer and Docker (only for the database-backed tests). ```bash git clone https://github.com/AlexMercedCoder/pangolin diff --git a/README.md b/README.md index 00902bd..896a48a 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Finally, pangolins are rare and specialized. They exist for a specific purpose a ## 🚀 Quick Start ### Prerequisites -- Rust 1.92+ +- Rust 1.94+ - Docker (optional, for MinIO) ### Running Locally diff --git a/SECURITY.md b/SECURITY.md index 454d965..70c74d1 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -41,8 +41,130 @@ security fixes. | Version | Supported | |---|---| -| 0.6.x | Yes | -| 0.5.x and earlier | No — upgrade to 0.6.x | +| 0.7.x | Yes | +| 0.6.x | No — upgrade to 0.7.x, see the advisory below | +| 0.5.x and earlier | No — upgrade to 0.7.x | + +## Fixed in 0.7.0 + +**Every version before 0.7.0 is affected by a privilege-escalation +vulnerability and should not be run.** Upgrade to 0.7.0 and rotate every issued +token. The findings below come from a full-repo audit conducted the day after +the 0.6.0 release; the authorization cluster was outside that release's scope +and is new information, not a regression. + +Note which version you are actually running. The published container image +`alexmerced/pangolin-api` was last pushed at **0.5.1** — no 0.6.0 image was +ever released — so most deployments are on 0.5.1, which carries everything +described here *and* everything 0.6.0 fixed. The affected range is +`< 0.7.0`, not `0.6.0` alone. + +### Exploitable by any authenticated principal + +Every issue in this table needs nothing but a valid credential — including the +lowest-privilege `tenant-user` account, or any service-user API key. + +| ID | Issue | Impact | +|---|---|---| +| B0a | `POST /api/v1/tokens` took no session and mapped a body-supplied `roles: ["Root"]` straight into signed claims | **Full privilege escalation.** Any authenticated caller could mint a valid `Root` token for any tenant. `check_permission` short-circuits for `Root`, so the resulting token bypasses every subsequent authorization check in the system | +| B0b | The credential-vending endpoint performed no authorization, never resolved the table, and hardcoded `["read", "write"]` | **Cloud storage credential disclosure.** Any tenant member obtained read+write credentials for the entire warehouse, naming a table they had no rights to and which need not exist | +| B0j | Logout revoked `session.user_id`; revocation is keyed by the token's `jti`, which no token carries as its `user_id` | **Logout did nothing.** The token stayed valid for its full 24-hour lifetime. On a shared machine, "signing out" left a working credential behind | +| B0g | The Iceberg OAuth token endpoint checked `active` but not expiry, unlike the API-key path | **Expired credentials were renewable indefinitely.** An expired service user could exchange `client_credentials` for a fresh JWT, repeatedly | +| B0h | The `NO_AUTH` public-bind guard read `no_auth && !dev_mode && !is_loopback(..)` | **Unauthenticated tenant-admin access.** `PANGOLIN_NO_AUTH=true` with `PANGOLIN_DEV_MODE=true` — routinely set together in compose and development setups — started happily on `0.0.0.0` and treated every anonymous request as `TenantAdmin` | +| B0l | OAuth matched existing users on email with no `email_verified` check and no provider binding | **Account takeover.** Anyone able to set a matching address on any configured provider — GitHub permits unverified addresses — logged in as that Pangolin user, including the seeded tenant admin | +| B0i | `PermissionScope::Tenant` matched without comparing the grant's tenant to the resource's | A tenant-wide grant issued in one tenant satisfied authorization for another tenant's resources | +| B0c–B0f | `rename_table`, `update_namespace_properties`, view create/read and `perform_maintenance` performed no authorization check at all | Any tenant member could move any table (an effective delete), rewrite namespace properties including `location`, read any view's SQL, and trigger snapshot expiry and orphan-file deletion. `perform_maintenance` additionally ran against a hardcoded `"default"` catalog rather than the one addressed | +| B0m | `expires_in_hours` reached `chrono::Duration::hours` unclamped, and no catch-panic layer was installed | **Remote panic.** A large value aborted the connection task, taking other in-flight requests on that connection with it | +| B0o | A token whose `jti` was absent or unparseable skipped the revocation check entirely | Such tokens were unrevocable for their full lifetime | + +### Availability + +| ID | Issue | Impact | +|---|---|---| +| — | `SqliteStore` had no inherent `revoke_token`/`is_token_revoked`, so the trait implementations called themselves | **Remote crash on the SQLite backend.** Revoking a token — which logout does — recursed until the thread stack was exhausted and aborted the process | +| B2 | On MongoDB the revocation write and the revocation check used different field names *and* different types | Revocation could never match. Revoked tokens, including after logout, stayed valid | + +### Confidentiality and integrity of records + +| ID | Issue | Impact | +|---|---|---| +| B1 | MongoDB's `get_audit_event` discarded the `tenant_id` parameter | **Cross-tenant audit disclosure.** Any tenant holding an audit-event UUID could read another tenant's record: username, IP address, resource names and metadata | +| — | The SQLite `audit_logs` table declared different columns than the code inserted | **No audit trail at all on SQLite.** Every audit write failed at runtime. If you run SQLite, assume you have no audit history prior to 0.7.0 | +| — | The admin CLI and Python SDK wrote their stored bearer tokens at the process umask, typically `0644` | Any local account could read the token. Now `0600` under a `0700` directory | +| — | `generate-code` and `get-token` printed live JWTs into copy-paste output | Tokens landed in shell scrollback, session transcripts and pasted snippets | +| B44 | A live PyPI API publish token was committed to the repository working tree in `.env` | Never tracked by Git, but readable by any local tool and passed into containers by `docker compose`. **The token has been removed and must be treated as compromised** | + +### Data loss + +Not security boundaries, but silent corruption is worth the same attention: + +* **B5** MongoDB's `update_metadata_location` ignored the compare-and-swap + entirely. Two concurrent Iceberg commits both reported success and one + snapshot was lost. Memory, Postgres and SQLite all enforced it. +* **B3** SQLite's `delete_branch` referenced a column that does not exist and + was not transactional: the branch was committed away and its assets orphaned + permanently, while the caller received an error suggesting nothing happened. +* **B7** All three persistent backends stored the `Debug` spelling of + `AssetType` and parsed only two of seventeen variants. A `DeltaTable`, + `MlModel` or `Lance` asset round-tripped as an Iceberg table. +* **B4** Postgres decoded a `TEXT[]` column as `String`; `sqlx::Row::get` + panics on a decode failure, so any search returning at least one hit panicked + the request. + +### Credential vending could not be built + +The `aws-sts`, `azure-oauth` and `gcp-oauth` features - and the +`cloud-credentials` bundle that unions them - did not compile, at any version. +`cargo build` and `cargo test` run with default features and no job ever passed +`--features`, so the entire cloud-credential surface had rotted into code that +could not be built at all: parameters bound as `_name` and referenced as `name` +inside the `cfg` block, a missing macro import, and an STS expiry parsed from a +`DateTime` as though it were an RFC3339 string. + +This is not an exploitable defect - unbuildable code ships in no binary. It +matters because it means **STS-based credential vending was not running +anywhere**, so any deployment that believed it was handing out scoped, +time-limited credentials was not. Check what your warehouses are actually +configured with: if `use_sts` is set but the server was built without the +feature (which is to say, always), the static-credential fallback in +`S3Signer::generate_credentials` is what answered - vending your long-lived +warehouse keys, with no expiry, instead of a scoped session token. Where no +static keys were configured either, the call failed with "AWS credentials not +configured", which at least failed closed. + +Fixed in 0.7.0, with a CI job that builds every optional feature so it cannot +recur silently. + +### Upgrading to 0.7.0 + +1. **Rotate every issued token.** B0a means any account may have minted a + `Root` token, and B0j means logging out never invalidated anything. Rotating + `PANGOLIN_JWT_SECRET` invalidates all existing sessions at once and is the + fastest way to be sure. +2. **Rotate service-user API keys**, for the same reason: B0g allowed expired + keys to keep issuing fresh JWTs. +3. **Rotate any cloud storage credentials** reachable through a warehouse that + an untrusted tenant member could name. B0b vended them to anyone. +4. **Audit for the escalation.** `POST /api/v1/tokens` calls from non-admin + principals, and credential-vending calls for tables the caller had no grant + on, are the two signals. Note that on SQLite there is no audit history to + check, and on MongoDB action-filtered queries returned nothing (B23) — so a + clean audit log is not evidence of absence on those backends. +5. **Check `PANGOLIN_NO_AUTH` and `PANGOLIN_DEV_MODE`.** If both were set on a + non-loopback bind, treat the deployment as having been open to anonymous + tenant-admin access for that period. The server now refuses to start in that + configuration. +6. **Set `PANGOLIN_OAUTH_EMAIL_LINK_DOMAINS`** if you rely on OAuth accounts + being matched to existing local users by email. Without it, identity is + `(provider, subject)` only and an address never adopts an existing account — + which is the safe default, and a behaviour change. +7. **On SQLite, upgrading migrates the `audit_logs` table** to schema version 2. + The previous table is preserved as `audit_logs_pre_v2`; it is empty in + practice, because nothing could ever write to it. +8. **Third-party API clients may need changes.** Server request bodies now + reject unknown fields with `422` instead of silently ignoring them. If a + client sent a misspelled or obsolete field, it was already being discarded — + the request was never doing what it appeared to. ## Fixed in 0.6.0 @@ -126,5 +248,12 @@ Stated plainly, because a checklist that hides its limits is worse than none: * **Audit records are not tamper-evident.** They live in the same database as application data, with no hash chaining and no WORM option. * **No MFA, password policy, or account lockout.** +* **Eight dependency advisories are accepted rather than fixed**, each with its + reasoning recorded in `.cargo/audit.toml`. None is reachable in a default + build, but they are exceptions rather than absences: `rsa` (never compiled), + `quick-xml` (held back by `object_store` 0.11 and `azure_core` 0.20), + a second `rustls-webpki` arriving via the AWS SDK's `rustls` 0.21, + `http-types`, and `rand`'s custom-logger unsoundness. Re-check the list when + dependencies move. These are tracked in `AUDIT_EXECUTION_PLAN.md` (items C-2 through C-20). diff --git a/deployment_assets/demo/evaluate_single_tenant/docker-compose.yml b/deployment_assets/demo/evaluate_single_tenant/docker-compose.yml index e22d972..fede60c 100644 --- a/deployment_assets/demo/evaluate_single_tenant/docker-compose.yml +++ b/deployment_assets/demo/evaluate_single_tenant/docker-compose.yml @@ -44,7 +44,7 @@ services: - "8080:8080" environment: - RUST_LOG=info - - PANGOLIN_STORE_TYPE=memory + - PANGOLIN_STORAGE_TYPE=memory - PANGOLIN_NO_AUTH=true - PANGOLIN_ROOT_USER=root - PANGOLIN_ROOT_PASSWORD=rootpass diff --git a/deployment_assets/helm/pangolin/Chart.yaml b/deployment_assets/helm/pangolin/Chart.yaml index 87a62bc..9443db0 100644 --- a/deployment_assets/helm/pangolin/Chart.yaml +++ b/deployment_assets/helm/pangolin/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: pangolin description: A Helm chart for deploying the Pangolin Lakehouse Catalog type: application -version: 0.6.0 -appVersion: "0.6.0" +version: 0.7.0 +appVersion: "0.7.0" keywords: - iceberg - lakehouse diff --git a/deployment_assets/production/azure_mongo/docker-compose.yml b/deployment_assets/production/azure_mongo/docker-compose.yml index 4dbc972..25da39a 100644 --- a/deployment_assets/production/azure_mongo/docker-compose.yml +++ b/deployment_assets/production/azure_mongo/docker-compose.yml @@ -16,7 +16,7 @@ services: - "8080:8080" environment: - RUST_LOG=info - - PANGOLIN_STORE_TYPE=mongo + - PANGOLIN_STORAGE_TYPE=mongo - DATABASE_URL=mongodb://mongo:27017 - MONGO_DB_NAME=${MONGO_DB_NAME:-pangolin} - PANGOLIN_ROOT_USER=${PANGOLIN_ROOT_USER:-admin} diff --git a/deployment_assets/production/azure_postgres/docker-compose.yml b/deployment_assets/production/azure_postgres/docker-compose.yml index 50f21fe..d83a633 100644 --- a/deployment_assets/production/azure_postgres/docker-compose.yml +++ b/deployment_assets/production/azure_postgres/docker-compose.yml @@ -25,7 +25,7 @@ services: - "8080:8080" environment: - RUST_LOG=info - - PANGOLIN_STORE_TYPE=postgres + - PANGOLIN_STORAGE_TYPE=postgres - DATABASE_URL=postgres://${POSTGRES_USER:-pangolin}:${POSTGRES_PASSWORD:-password}@postgres:5432/${POSTGRES_DB:-pangolin} - PANGOLIN_ROOT_USER=${PANGOLIN_ROOT_USER:-admin} - PANGOLIN_ROOT_PASSWORD=${PANGOLIN_ROOT_PASSWORD:-change-me-in-prod} diff --git a/deployment_assets/production/gcp_mongo/docker-compose.yml b/deployment_assets/production/gcp_mongo/docker-compose.yml index 796b3f4..12679fe 100644 --- a/deployment_assets/production/gcp_mongo/docker-compose.yml +++ b/deployment_assets/production/gcp_mongo/docker-compose.yml @@ -16,7 +16,7 @@ services: - "8080:8080" environment: - RUST_LOG=info - - PANGOLIN_STORE_TYPE=mongo + - PANGOLIN_STORAGE_TYPE=mongo - DATABASE_URL=mongodb://mongo:27017 - MONGO_DB_NAME=${MONGO_DB_NAME:-pangolin} - PANGOLIN_ROOT_USER=${PANGOLIN_ROOT_USER:-admin} diff --git a/deployment_assets/production/gcp_postgres/docker-compose.yml b/deployment_assets/production/gcp_postgres/docker-compose.yml index 8499367..14bb187 100644 --- a/deployment_assets/production/gcp_postgres/docker-compose.yml +++ b/deployment_assets/production/gcp_postgres/docker-compose.yml @@ -25,7 +25,7 @@ services: - "8080:8080" environment: - RUST_LOG=info - - PANGOLIN_STORE_TYPE=postgres + - PANGOLIN_STORAGE_TYPE=postgres - DATABASE_URL=postgres://${POSTGRES_USER:-pangolin}:${POSTGRES_PASSWORD:-password}@postgres:5432/${POSTGRES_DB:-pangolin} - PANGOLIN_ROOT_USER=${PANGOLIN_ROOT_USER:-admin} - PANGOLIN_ROOT_PASSWORD=${PANGOLIN_ROOT_PASSWORD:-change-me-in-prod} diff --git a/deployment_assets/production/s3_mongo/docker-compose.yml b/deployment_assets/production/s3_mongo/docker-compose.yml index bce15dd..ba9a147 100644 --- a/deployment_assets/production/s3_mongo/docker-compose.yml +++ b/deployment_assets/production/s3_mongo/docker-compose.yml @@ -16,7 +16,7 @@ services: - "8080:8080" environment: - RUST_LOG=info - - PANGOLIN_STORE_TYPE=mongo + - PANGOLIN_STORAGE_TYPE=mongo - DATABASE_URL=mongodb://mongo:27017 - MONGO_DB_NAME=${MONGO_DB_NAME:-pangolin} - PANGOLIN_ROOT_USER=${PANGOLIN_ROOT_USER:-admin} diff --git a/deployment_assets/production/s3_postgres/docker-compose.yml b/deployment_assets/production/s3_postgres/docker-compose.yml index 11cd71a..b9b5378 100644 --- a/deployment_assets/production/s3_postgres/docker-compose.yml +++ b/deployment_assets/production/s3_postgres/docker-compose.yml @@ -25,7 +25,7 @@ services: - "8080:8080" environment: - RUST_LOG=info - - PANGOLIN_STORE_TYPE=postgres + - PANGOLIN_STORAGE_TYPE=postgres - DATABASE_URL=postgres://${POSTGRES_USER:-pangolin}:${POSTGRES_PASSWORD:-password}@postgres:5432/${POSTGRES_DB:-pangolin} - PANGOLIN_ROOT_USER=${PANGOLIN_ROOT_USER:-admin} - PANGOLIN_ROOT_PASSWORD=${PANGOLIN_ROOT_PASSWORD:-change-me-in-prod} diff --git a/deployment_assets/test/localstack/docker-compose.yml b/deployment_assets/test/localstack/docker-compose.yml index 92ec71e..40d87d0 100644 --- a/deployment_assets/test/localstack/docker-compose.yml +++ b/deployment_assets/test/localstack/docker-compose.yml @@ -21,7 +21,7 @@ services: - "8080:8080" environment: - RUST_LOG=info - - PANGOLIN_STORE_TYPE=memory + - PANGOLIN_STORAGE_TYPE=memory - PANGOLIN_NO_AUTH=true - AWS_ACCESS_KEY_ID=test - AWS_SECRET_ACCESS_KEY=test diff --git a/docker-compose.db-test.yml b/docker-compose.db-test.yml index 1dc184a..f8870d3 100644 --- a/docker-compose.db-test.yml +++ b/docker-compose.db-test.yml @@ -42,3 +42,48 @@ services: interval: 5s timeout: 5s retries: 5 + + # Object storage for the file-IO half of the store compliance suite. + # + # This file used to provide only the databases, so `test_*_store_compliance` + # had no S3 to talk to: it fell through to the EC2 instance-metadata endpoint, + # hung for eleven seconds, and failed with a credentials error that looked + # nothing like "you have no object store". CI has always had one, so the + # documented local workflow and CI disagreed about what it took to run the + # suite. + # + # `warehouse` is the bucket the application itself expects; `bucket` and + # `test-bucket` are fixtures the compliance tests write to. Creating only + # `warehouse` (as `docker-compose.yml` does, for the app) leaves the suite + # failing with `NoSuchBucket`. + # + # The obvious image for this is `bitnami/minio`, whose MINIO_DEFAULT_BUCKETS + # makes the init container unnecessary. Bitnami withdrew their public Docker + # Hub images, so `bitnami/minio:latest` no longer resolves at all - it fails + # with `manifest unknown` rather than anything that names the cause. + minio: + image: minio/minio + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + command: server /data --console-address ":9001" + ports: + - "9000:9000" + - "9001:9001" + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 20 + + createbuckets: + image: minio/mc + depends_on: + minio: + condition: service_healthy + entrypoint: > + /bin/sh -c " + /usr/bin/mc alias set m http://minio:9000 minioadmin minioadmin; + for b in warehouse bucket test-bucket; do /usr/bin/mc mb --ignore-existing m/$$b; done; + exit 0; + " diff --git a/docker-compose.mongo-rs.yml b/docker-compose.mongo-rs.yml new file mode 100644 index 0000000..0f9421b --- /dev/null +++ b/docker-compose.mongo-rs.yml @@ -0,0 +1,41 @@ +# A single-node MongoDB replica set, for exercising the paths a standalone +# `mongod` cannot reach. +# +# `docker-compose.db-test.yml` runs MongoDB standalone, which is the *degraded* +# topology: no transactions and no retryable writes. Everything verified against +# it therefore exercised the fallback paths only — and the fallback in +# `delete_catalog` was itself broken for as long as the code had existed, +# precisely because nothing ever ran the other branch. +# +# A single node is enough: a replica set of one supports transactions and +# retryable writes, which is what distinguishes the two code paths. It is not a +# model of production redundancy and is not meant to be. +# +# docker compose -f docker-compose.mongo-rs.yml up -d +# export PANGOLIN_TEST_MONGO_URL='mongodb://localhost:27018/?replicaSet=rs0&directConnection=true' +# cargo test -p pangolin_store +# +# Note the port: 27018, so this can run alongside the standalone on 27017 and +# the two topologies can be tested in one sitting. +# +# No authentication here. A replica set with auth needs a shared keyfile, which +# is meaningful setup for a throwaway test fixture; the standalone compose file +# covers the authenticated path. + +services: + mongo-rs: + image: mongo:7.0 + command: ["mongod", "--replSet", "rs0", "--bind_ip_all", "--port", "27018"] + ports: + - "27018:27018" + healthcheck: + # Health is "the replica set has a primary", not "the process started". + # `mongod` accepts connections long before the set is initiated, and a + # client that connects in between fails in confusing ways. + test: >- + mongosh --port 27018 --quiet --eval + "try { rs.status().ok } catch (e) { rs.initiate({_id:'rs0',members:[{_id:0,host:'localhost:27018'}]}).ok }" + interval: 5s + timeout: 10s + retries: 20 + start_period: 10s diff --git a/docker-compose.release.yml b/docker-compose.release.yml index e2b8cda..2a4cf9c 100644 --- a/docker-compose.release.yml +++ b/docker-compose.release.yml @@ -1,11 +1,23 @@ version: '3.8' +# An explicit project name, separate from every other compose file here. +# +# Compose derives the project name from the directory, so this file, +# docker-compose.yml and docker-compose.db-test.yml all shared the project +# `pangolin` - and `up`, `down -v` and orphan cleanup in one would stop and +# remove containers belonging to another. Verifying a release while a +# development database stack is running is a completely ordinary thing to do, +# and it made the API container exit mid-verification for reasons that had +# nothing to do with the release. +name: pangolin-release + services: minio: image: minio/minio - ports: - - "9000:9000" - - "9001:9001" + # No host port publication. The API and the smoke test both reach MinIO + # over the compose network, so claiming 9000/9001 on the host bought + # nothing and made this file unrunnable whenever a development stack was + # already using them - which is exactly when someone verifies a release. environment: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin @@ -34,18 +46,45 @@ services: " pangolin-api: - image: alexmerced/pangolin-api:0.2.0 + # B10: this pinned 0.2.0 - four releases behind the workspace - so the + # "release verification" compose file verified an image nobody ships. + # Parameterised so a release can be tested by setting PANGOLIN_VERSION, + # with the current version as the default. + image: alexmerced/pangolin-api:${PANGOLIN_VERSION:-0.7.0} + # Published so a human can poke at the server under test. Override + # RELEASE_API_PORT if 8080 is taken. ports: - - "8080:8080" + - "${RELEASE_API_PORT:-8080}:8080" environment: - RUST_LOG=info - - PANGOLIN_STORE_TYPE=memory + # B9: the server reads PANGOLIN_STORAGE_TYPE; PANGOLIN_STORE_TYPE was + # silently ignored. + - PANGOLIN_STORAGE_TYPE=memory + # B8: the server refuses to start without a JWT secret. + # RELEASE_* rather than PANGOLIN_*, deliberately. + # + # Docker Compose auto-loads `.env` from the project directory, so a + # developer's local PANGOLIN_ROOT_USER / PANGOLIN_ROOT_PASSWORD were + # silently feeding this file. In this working copy that meant the server + # refused to start at all - `.env` carries a placeholder password, and the + # config guard rejects it - so the release verification could not run, + # for a reason having nothing to do with the artifact being verified. + # + # A verification harness that inherits ambient local config is not + # reproducible. These names cannot collide with a real deployment's. + - PANGOLIN_JWT_SECRET=${RELEASE_JWT_SECRET:?set it first, e.g. export RELEASE_JWT_SECRET=$(openssl rand -base64 48)} - AWS_ACCESS_KEY_ID=minioadmin - AWS_SECRET_ACCESS_KEY=minioadmin - AWS_REGION=us-east-1 - AWS_ENDPOINT_URL=http://minio:9000 - AWS_ALLOW_HTTP=true - PANGOLIN_NO_AUTH=${PANGOLIN_NO_AUTH:-false} + # Root basic auth, so the smoke test can exercise a real authenticated + # write against the released image. `PANGOLIN_NO_AUTH` cannot be used for + # this: the server refuses to start with it on a non-loopback bind, which + # is every container, so that branch was unreachable here. + - PANGOLIN_ROOT_USER=${RELEASE_ROOT_USER:-releaseroot} + - PANGOLIN_ROOT_PASSWORD=${RELEASE_ROOT_PASSWORD:?set it first, e.g. export RELEASE_ROOT_PASSWORD=$(openssl rand -base64 24)} depends_on: minio: condition: service_healthy @@ -62,10 +101,23 @@ services: - AWS_ACCESS_KEY_ID=minioadmin - AWS_SECRET_ACCESS_KEY=minioadmin - AWS_REGION=us-east-1 - - TEST_MODE=${TEST_MODE:-no-auth} + # Mirrors the server's own setting rather than a separate TEST_MODE that + # nothing read, so the smoke test knows whether the authenticated surface + # is reachable. + - PANGOLIN_NO_AUTH=${PANGOLIN_NO_AUTH:-false} + - PANGOLIN_ROOT_USER=${RELEASE_ROOT_USER:-releaseroot} + - PANGOLIN_ROOT_PASSWORD=${RELEASE_ROOT_PASSWORD} depends_on: - pangolin-api - command: sh -c "pip install requests pyiceberg pyarrow && python scripts/test_release_v0.2.0.py" + # B10 pointed this at scripts/test_release_v0.2.0.py, which does not exist, + # then at scripts/integration_test.py - which builds and starts its own + # server with `cargo run`, and so cannot run in a python:3.11-slim container + # with only ./scripts mounted. Either way the release was never verified. + # + # release_smoke_test.py talks to the already-running container over HTTP, + # which is both the only thing possible here and the more useful check: it + # exercises the published image rather than a rebuild of the source. + command: sh -c "pip install requests && python scripts/release_smoke_test.py" volumes: minio_data_release: diff --git a/docker-compose.yml b/docker-compose.yml index 62c3d12..20e3a48 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,7 +40,17 @@ services: - "8080:8080" environment: - RUST_LOG=info - - PANGOLIN_STORE_TYPE=memory + # B8: since 0.6.0 the server refuses to start without a JWT secret, and + # refuses PANGOLIN_NO_AUTH on the default 0.0.0.0 bind - so the documented + # quick start produced a crash-looping container with no explanation. The + # `:?` form fails the `docker compose up` itself with this message, which + # is a far better failure than a restart loop. + - PANGOLIN_JWT_SECRET=${PANGOLIN_JWT_SECRET:?set it first, e.g. export PANGOLIN_JWT_SECRET=$(openssl rand -base64 48)} + # B9: the server reads PANGOLIN_STORAGE_TYPE. This said + # PANGOLIN_STORE_TYPE, which nothing reads - so anyone editing it to + # `postgres` silently kept the in-memory backend and lost their data on + # every restart. + - PANGOLIN_STORAGE_TYPE=memory - AWS_ACCESS_KEY_ID=minioadmin - AWS_SECRET_ACCESS_KEY=minioadmin - AWS_REGION=us-east-1 @@ -57,7 +67,11 @@ services: ports: - "3000:3000" environment: - - VITE_API_URL=http://localhost:8080 + # B31: SvelteKit's dynamic public env requires the PUBLIC_ prefix. The + # client reads PUBLIC_API_URL; this used to pass VITE_API_URL, which + # nothing read, so every deployed build fell back to the *end user's* + # localhost:8080. + - PUBLIC_API_URL=http://localhost:8080 - ORIGIN=http://localhost:3000 depends_on: - pangolin-api diff --git a/docs/best-practices/deployment.md b/docs/best-practices/deployment.md index e15c4dc..3c55224 100644 --- a/docs/best-practices/deployment.md +++ b/docs/best-practices/deployment.md @@ -101,7 +101,7 @@ export PANGOLIN_JWT_SECRET=$(aws secretsmanager get-secret-value \ **Production Dockerfile** ```dockerfile -FROM rust:1.92 as builder +FROM rust:1.94 as builder WORKDIR /app COPY . . RUN cargo build --release --bin pangolin_api diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 60ed622..08486da 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -1,6 +1,13 @@ # Environment Variables Reference -This document provides a comprehensive reference for all environment variables used by Pangolin API. +This document provides a comprehensive reference for all environment variables +used by the Pangolin API. + +> **This page is checked against the code.** `pangolin/scripts/check_env_var_docs.sh` +> re-derives the set of `PANGOLIN_*` variables the server actually reads and +> fails if this file documents one that nothing reads, or omits one that +> something does. It runs in CI. Before that check existed, this page listed +> three variables that did not exist and omitted thirty-four that did. ## Table of Contents @@ -234,34 +241,187 @@ export DATABASE_URL="/var/lib/pangolin/pangolin.db" ## Server Configuration -### `PANGOLIN_HOST` +### `PANGOLIN_BIND_ADDRESS` -**Required:** No -**Type:** String (IP address) -**Default:** `0.0.0.0` +**Required:** No +**Type:** String (IP address) +**Default:** `0.0.0.0` **Description:** IP address to bind the server to. +Note: if `PANGOLIN_NO_AUTH=true`, this **must** be a loopback address. The +server refuses to start otherwise, unconditionally - `PANGOLIN_DEV_MODE` does +not waive the check. + ```bash # Listen on all interfaces -export PANGOLIN_HOST="0.0.0.0" +export PANGOLIN_BIND_ADDRESS="0.0.0.0" -# Listen only on localhost -export PANGOLIN_HOST="127.0.0.1" +# Listen only on localhost (required when PANGOLIN_NO_AUTH=true) +export PANGOLIN_BIND_ADDRESS="127.0.0.1" ``` -### `PANGOLIN_PORT` or `PORT` +### `PORT` -**Required:** No -**Type:** Integer -**Default:** `8080` +**Required:** No +**Type:** Integer +**Default:** `8080` **Description:** Port number for the API server. ```bash -export PANGOLIN_PORT=3000 -# or export PORT=3000 ``` +### `PANGOLIN_MAX_BODY_BYTES` + +**Required:** No +**Type:** Integer (bytes) +**Default:** 10 MiB +**Description:** Largest request body the server will buffer. + +### `PANGOLIN_REQUEST_TIMEOUT_SECS` + +**Required:** No +**Type:** Integer (seconds) +**Default:** `30` +**Description:** Deadline for a request, **including** time spent queued behind +the concurrency limiter. + +### `PANGOLIN_MAX_CONCURRENT_REQUESTS` + +**Required:** No +**Type:** Integer +**Default:** `512` +**Description:** Requests admitted concurrently; the rest queue, bounded by the +request timeout above. + +### `PANGOLIN_SHUTDOWN_GRACE_SECS` + +**Required:** No +**Type:** Integer (seconds) +**Description:** Upper bound on the drain after SIGTERM. Readiness fails +immediately; in-flight requests then have this long to finish before the +process exits regardless. Set it below your orchestrator's termination grace +period. + +### `PANGOLIN_CORS_ALLOWED_ORIGINS` + +**Required:** No +**Type:** Comma-separated list of origins +**Default:** any origin +**Description:** Restricts CORS to the listed origins. + +### `PANGOLIN_WAREHOUSE_CACHE_TTL_SECS` + +**Required:** No +**Type:** Integer (seconds) +**Default:** `5` +**Description:** TTL of the warehouse cache. Entries hold cloud storage +credentials and the cache is node-local, so with more than one replica a +rotated credential can still be vended by peers for up to this long. + +### `PANGOLIN_METRICS_ENABLED` + +**Required:** No +**Type:** Boolean +**Description:** Serve Prometheus metrics at `/metrics`. + +--- + +## Identity and Access + +### `PANGOLIN_JWT_SECRET` + +**Required:** **Yes**, unless `PANGOLIN_DEV_MODE` or `PANGOLIN_NO_AUTH` is set +**Type:** String (32+ characters) +**Description:** Signing key for session JWTs. The server refuses to start +without it, and rejects known-weak values. + +```bash +export PANGOLIN_JWT_SECRET="$(openssl rand -base64 48)" +``` + +### `PANGOLIN_ROOT_USER` / `PANGOLIN_ROOT_PASSWORD` + +**Required:** No +**Type:** String +**Description:** Credentials for the root basic-auth principal. Compared in +constant time. A weak password is refused outside dev mode. + +### `PANGOLIN_SEED_ADMIN`, `PANGOLIN_ADMIN_USER`, `PANGOLIN_ADMIN_PASSWORD` + +**Required:** No +**Type:** Boolean / String / String +**Description:** Seed a first tenant administrator on an empty database. + +### `PANGOLIN_SESSION_TTL_SECS` + +**Required:** No +**Type:** Integer (seconds) +**Description:** Lifetime of an issued session token. + +### `PANGOLIN_DEV_MODE` + +**Required:** No +**Type:** Boolean +**Default:** `false` +**Description:** Relaxes *secret strength* requirements for local development. +It does **not** relax network exposure: the loopback requirement on +`PANGOLIN_NO_AUTH` applies regardless. + +### `PANGOLIN_ALLOW_LEGACY_API_KEYS` + +**Required:** No +**Type:** Boolean +**Default:** `false` +**Description:** Accept API keys minted before the key-id format. Off by +default because a legacy key costs a bcrypt verification per candidate. + +### `PANGOLIN_ALLOW_TOKENS_WITHOUT_JTI` + +**Required:** No +**Type:** Boolean +**Default:** `false` +**Description:** Accept JWTs carrying no `jti`. Such a token can never be +revoked, so this exists only as a migration window. + +--- + +## OAuth / OIDC + +Each provider is enabled by setting its client id and secret. The redirect URI +defaults to `/oauth/callback/`. + +| Provider | Variables | +|-----------|-----------| +| Google | `PANGOLIN_GOOGLE_CLIENT_ID`, `PANGOLIN_GOOGLE_CLIENT_SECRET`, `PANGOLIN_GOOGLE_REDIRECT_URI` | +| GitHub | `PANGOLIN_GITHUB_CLIENT_ID`, `PANGOLIN_GITHUB_CLIENT_SECRET`, `PANGOLIN_GITHUB_REDIRECT_URI` | +| Microsoft | `PANGOLIN_MICROSOFT_CLIENT_ID`, `PANGOLIN_MICROSOFT_CLIENT_SECRET`, `PANGOLIN_MICROSOFT_REDIRECT_URI`, `PANGOLIN_MICROSOFT_TENANT_ID` | +| Okta | `PANGOLIN_OKTA_CLIENT_ID`, `PANGOLIN_OKTA_CLIENT_SECRET`, `PANGOLIN_OKTA_REDIRECT_URI`, `PANGOLIN_OKTA_DOMAIN` | + +### `FRONTEND_URL` + +**Required:** No +**Type:** URL +**Default:** `http://localhost:5173` +**Description:** Where the UI lives. Always an acceptable OAuth landing page. + +### `PANGOLIN_OAUTH_REDIRECT_URIS` + +**Required:** No +**Type:** Comma-separated list of exact URLs +**Description:** Additional URLs an OAuth flow may hand control back to. +Anything not listed (and not `FRONTEND_URL`) is refused. + +### `PANGOLIN_OAUTH_EMAIL_LINK_DOMAINS` + +**Required:** No +**Type:** Comma-separated list of domains +**Default:** empty +**Description:** Domains whose **verified** addresses may adopt a pre-existing +local account. With no allowlist, identity is `(provider, subject)` only and an +email address never links an account - which is what stops someone setting a +matching address on any configured provider and logging in as that user. + --- ## Logging @@ -316,7 +476,7 @@ export AWS_ACCESS_KEY_ID="AKIAIOSFODNN7EXAMPLE" export AWS_SECRET_ACCESS_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" export AWS_REGION="us-east-1" export RUST_LOG=info -export PANGOLIN_PORT=8080 +export PORT=8080 ./pangolin_api ``` @@ -379,8 +539,10 @@ services: ### ❌ Using the wrong variable names ```bash -# WRONG - These are not used -export PANGOLIN_STORE_TYPE=mongo +# WRONG - This one is not read by anything. It appeared in the compose files +# and in this document for several releases, so editing it to `postgres` +# silently left you on the in-memory backend. +export PANGOLIN_STORE_TYPE=mongo # export MONGODB_URI=mongodb://localhost:27017 export MONGODB_DATABASE=pangolin @@ -433,7 +595,7 @@ When multiple variables could configure the same thing: 1. **Storage Backend:** `DATABASE_URL` (only this is used) 2. **MongoDB Database:** `MONGO_DB_NAME` (only this is used) 3. **S3 Endpoint:** `S3_ENDPOINT` or `AWS_ENDPOINT_URL` (both work) -4. **Server Port:** `PANGOLIN_PORT` or `PORT` (PANGOLIN_PORT takes precedence) +4. **Server Port:** `PORT` --- diff --git a/docs/getting-started/env_vars.md b/docs/getting-started/env_vars.md index 32ae9cf..b9ace1a 100644 --- a/docs/getting-started/env_vars.md +++ b/docs/getting-started/env_vars.md @@ -1,63 +1,12 @@ # Environment Variables -Pangolin is configured via environment variables. This guide lists all available options. +**This page has moved to [../environment-variables.md](../environment-variables.md).** -## 🚀 Core API Configuration +Two hand-maintained environment-variable references existed side by side and +both had drifted from the code (B43). Keeping one of them was not a fix - the +problem was that either could drift silently. -| Variable | Description | Default | -|----------|-------------|---------| -| `PORT` | The port the API server will listen on. | `8080` | -| `RUST_LOG` | Log level (`error`, `warn`, `info`, `debug`, `trace`). | `info` | - -## 💾 Metadata Persistence - -Pangolin stores its own metadata (tenants, users, catalogs) in a backend database. - -| Variable | Description | Default | -|----------|-------------|---------| -| `DATABASE_URL` | Connection string for Postgres, MongoDB, or SQLite. | (None) | -| `PANGOLIN_STORAGE_TYPE` | Storage driver if `DATABASE_URL` is missing. (`memory`, `postgres`, `mongo`, `sqlite`). | `memory` | -| `MONGO_DB_NAME` | Database name (when using MongoDB). | `pangolin` | - -> [!NOTE] -> If `DATABASE_URL` is provided, the driver is automatically inferred from the URI scheme (e.g., `postgresql://`, `mongodb://`, `sqlite://`). - -## 🛡️ Authentication & Security - -| Variable | Description | Default | -|----------|-------------|---------| -| `PANGOLIN_NO_AUTH` | **Evaluation Mode**. Auto-provisions a default tenant and admin user. Set to `"true"` to enable. | `false` | -| `PANGOLIN_JWT_SECRET` | Secret key for JWT signing. **MUST** be changed in production. | `default_secret` | -| `PANGOLIN_SEED_ADMIN` | Auto-provision a Tenant Admin even if `NO_AUTH` is false. Set to `"true"`. | `false` | -| `PANGOLIN_ADMIN_USER` | Username for seed admin. | `tenant_admin` | -| `PANGOLIN_ADMIN_PASSWORD` | Password for seed admin. | `password123` | -| `PANGOLIN_ROOT_USER` | Initial Root user for multi-tenant bootstrapping. | `admin` | -| `PANGOLIN_ROOT_PASSWORD` | Password for Root user. | `password` | - -## 🌐 OAuth 2.0 (External Providers) - -Required for enabling "Login with Google/GitHub/Microsoft" in the UI. - -| Variable | Provider | -|----------|----------| -| `OAUTH_GOOGLE_CLIENT_ID` / `_SECRET` | Google | -| `OAUTH_MICROSOFT_CLIENT_ID` / `_SECRET` | Microsoft | -| `OAUTH_GITHUB_CLIENT_ID` / `_SECRET` | GitHub | - -## ☁️ Cloud Provider Features - -When building with cloud features (`--features aws-sts`, etc.), these standard variables are used by the underlying SDKs for the **Signer** logic. - -| Variable | Description | -|----------|-------------| -| `AWS_ACCESS_KEY_ID` | AWS Credentials for STS / Signer. | -| `AWS_SECRET_ACCESS_KEY` | AWS Credentials for STS / Signer. | -| `AWS_REGION` | Default AWS region. | -| `AWS_ENDPOINT_URL` | Override for MinIO or custom S3 backends. | - -## 🚦 Security Checklist - -1. **Disable NO_AUTH**: Ensure `PANGOLIN_NO_AUTH` is unset or `false` in production. -2. **Rotate JWT Secret**: Use a 32+ character random string. -3. **Strong Root Password**: Change the default `admin/password` immediately. -4. **Use DATABASE_URL**: Avoid the `memory` store for any non-trivial use case. +The surviving reference is checked against the source in CI by +`pangolin/scripts/check_env_var_docs.sh`, which fails the build if it documents +a `PANGOLIN_*` variable nothing reads, or omits one something does. This page is +a redirect so existing links keep working. diff --git a/docs/operations/backend-parity.md b/docs/operations/backend-parity.md index 5a498e8..0bfc63a 100644 --- a/docs/operations/backend-parity.md +++ b/docs/operations/backend-parity.md @@ -8,6 +8,13 @@ on which backend. **Recommended for production: PostgreSQL.** +Every ✅ below is now backed by the cross-backend parity suite +(`cargo test -p pangolin_store --test store_integration`) run against a live +instance of each backend, not by inspection. That distinction matters: until +0.7.0 the Postgres and MongoDB tests had never actually been executed, and this +table asserted several capabilities that failed at the first request — see +"What the first live run found" below. + | Capability | Memory | SQLite | PostgreSQL | MongoDB | |---|:--:|:--:|:--:|:--:| | Tenants, catalogs, namespaces, assets | ✅ | ✅ | ✅ | ✅ | @@ -52,15 +59,107 @@ a `pg_advisory_lock` so that concurrent replicas do not race. > also carried a shape the code could not write to, so every audit write failed. > Both are fixed in 0.6.0; see `migrations/20260809000000_repair_orphaned_schema.sql`. -**MongoDB.** Functional for core catalog operations. Known gaps: +**MongoDB.** Functional for core catalog operations. The RBAC and audit-log +failures listed here before 0.7.0 are fixed, along with four more found by +auditing every collection at once: they were all one bug wearing eight hats, and +"The MongoDB UUID encoding rule" below is what it takes not to add a ninth. Note +in particular that the ✅ against *service users and API keys* was wrong until +0.7.0 — every method in that module was a silent no-op, so an API key could not +authenticate on MongoDB at all. Remaining gaps: + +* No schema or index management beyond a handful of indexes created at startup. +* **A replica set is strongly recommended.** On a standalone `mongod`: + * transactions are unavailable, so `delete_catalog` degrades to a sequential, + non-atomic cascade. It now degrades with a warning; before 0.7.0 the + degradation path was unreachable and the delete failed outright. + * retryable writes are unavailable — add `retryWrites=false` to your + connection string or single-document writes will be rejected. + + Both topologies are tested. CI runs the MongoDB suites twice: once against a + standalone (the `guardrails` job) and once against a single-node replica set + (`mongo-replica-set`), because the two exercise *different branches* of + `delete_catalog` and testing one says nothing about the other. Locally: + + ```bash + # standalone — the degraded paths + docker compose -f docker-compose.db-test.yml up -d mongo + export PANGOLIN_TEST_MONGO_URL='mongodb://testuser:testpass@localhost:27017/?retryWrites=false' + + # replica set — transactions and retryable writes + docker compose -f docker-compose.mongo-rs.yml up -d + export PANGOLIN_TEST_MONGO_URL='mongodb://localhost:27018/?replicaSet=rs0&directConnection=true' + ``` + + They use different ports so both can run at once. +* Multi-statement operations other than `delete_catalog` are still not atomic. + +## What the first live run found + +The parity suite was written against the memory and SQLite backends, which are +the two that CI could run without a service container. The first time it was +pointed at a live PostgreSQL and MongoDB it failed on both, for reasons no +amount of code reading had surfaced: + +| Backend | Defect | +|---|---| +| PostgreSQL | **`business_metadata` was never created by any migration**, while `search_assets` joined it. Every asset search failed with `relation "business_metadata" does not exist` — a hard SQL error, not an empty result. The three CRUD methods were unimplemented, so the trait's "not supported" default answered them. | +| MongoDB | **`get_metadata_location` had no fallback to the asset's own `location`**, unlike the other three backends. A table created with a location but no explicit metadata-location property reported none, so its metadata could not be loaded and its commits compared against a different value than the read path returned. | +| MongoDB | **Role assignments were written by serde and queried as BSON Binary.** `bson::to_document` writes a `Uuid` as a string; the deserializer expects Binary. So `get_user_roles` never matched, every role-derived permission silently vanished, and a user holding an admin role was authorized as though they held none. The same asymmetry caused B1 (audit) and B2 (token revocation). | +| MongoDB | **`delete_catalog`'s "fall back when transactions are unavailable" path was unreachable.** `start_transaction` is a local call in the Rust driver, so it cannot fail for want of a replica set; the error arrives on the first operation *inside* the transaction and was propagated instead of caught. | +| MongoDB | Once that fallback *did* run, it carried B21 — the cascade deleted every matching child row before checking the catalog existed, then reported "not found" to a caller with every reason to believe nothing had happened. The same shape had been fixed for SQLite during the roadmap work; it survived here because this branch had never executed. | + +Postgres, notably, is the only backend that makes the orphaned-child state +*unreachable*: it carries foreign keys from namespaces to catalogs and from +assets to namespaces, so a mis-ordered cascade has nothing to destroy. The other +three permit orphans, which is why the parity suite asserts the ordering there +and records the asymmetry rather than skipping quietly. + +None of these were regressions. They had been present for as long as the code +had, and a per-backend test could not have found the first three: each is one +backend disagreeing with the others, which is only visible when something asserts +they agree. -* No schema or index management at all — indexes must be created by hand. -* No transactions: `MongoStore` opens no sessions, so multi-statement - operations are not atomic. -* RBAC aggregation and audit-log filtering have failing tests - (`test_mongo_rbac_operations`, `test_mongo_list_user_permissions_aggregation`, - `test_mongo_audit_log_filtering`, `test_mongo_store_regression`). Treat - MongoDB as beta. +## The MongoDB UUID encoding rule + +If you write a MongoDB backend method, this is the one thing to get right. + +There are three ways this codebase converts a `Uuid` into BSON, and they produce +three different values: + +| Route | Produces | +|---|---| +| `to_bson_uuid(id)` | `Binary`, generic subtype | +| `doc! { "k": id }` | `Binary`, **UUID** subtype | +| `bson::to_document(&value)` | a `String` | + +Reads disagree as well. A typed `Collection` deserializes non-human-readably +and requires binary; `bson::from_bson` requires a string. So a write and a read +chosen independently agree only by luck. + +When they disagree the code does not fail loudly. The document is written, the +filter matches nothing, and the caller gets an empty result that is +indistinguishable from "there is nothing there". That is how a user holding an +admin role came to be authorized as though they held none, how token revocation +became a silent no-op, and how every service-user method became an unremarked +no-op including the API-key lookup. + +The rule: + +* **Write** UUIDs with `to_bson_uuid`, or with `with_binary_uuids` when the + document came from `bson::to_document`. Pass the *serialized* field names — + most of these structs are `rename_all = "kebab-case"`, so it is `"tenant-id"`, + not `"tenant_id"`. Getting that wrong is its own silent no-op. +* **Read** UUIDs with `from_bson_uuid` (or `read_optional_uuid` for an optional + field), which accepts all three encodings so records already in a deployed + database still load. +* **Add a case to `mongo_uuid_round_trip_tests.rs`** for any new collection. It + runs against both topologies in CI and is the only thing that checks the write + and the read agree. + +Timestamps have the same shape of problem in the opposite direction: serde +writes a `DateTime` as an RFC3339 string, `doc!` writes a BSON DateTime, +and chrono's `Deserialize` accepts only the string. Match whatever the +collection already uses. ## Transactions @@ -75,7 +174,10 @@ rollback and no repair tooling. What exists today: -* SQLite uses a transaction in one place. +* SQLite uses transactions for branch deletion and the catalog cascade. +* MongoDB uses a transaction for the catalog cascade where the deployment + supports one, and degrades to a sequential cascade with a warning where it + does not. * PostgreSQL serialises schema setup with an advisory lock, so concurrent replicas cannot corrupt the schema. * The Iceberg table-commit path uses compare-and-swap on the metadata pointer diff --git a/docs/upgrading/0.6-to-0.7.md b/docs/upgrading/0.6-to-0.7.md new file mode 100644 index 0000000..5388d27 --- /dev/null +++ b/docs/upgrading/0.6-to-0.7.md @@ -0,0 +1,226 @@ +# Upgrading from 0.6.x to 0.7.0 + +**0.7.0 is a security release.** 0.6.0 is affected by a privilege escalation +that any authenticated principal could exploit. Read +[SECURITY.md](../../SECURITY.md#fixed-in-070) first, then come back here for the +compatibility detail. + +This page covers only what changes for *you*. The full list of fixes is in the +[changelog](../../CHANGELOG.md). + +--- + +## Do these before anything else + +1. **Rotate `PANGOLIN_JWT_SECRET`.** Any account could have minted a `Root` + token (B0a), and logging out never invalidated anything (B0j). Rotating the + signing secret invalidates every outstanding session in one step, which is + the only way to be certain none survive. +2. **Rotate service-user API keys.** An expired key could keep exchanging + itself for fresh JWTs (B0g). +3. **Rotate cloud storage credentials** on any warehouse an untrusted tenant + member could reach. Credential vending had no authorization at all (B0b). + +--- + +## Breaking changes + +### Unknown request fields are now rejected + +Server request bodies carry `#[serde(deny_unknown_fields)]`. A field the server +does not recognise is a `422` naming the field, where it used to be silently +discarded and the request would return `200`. + +**Nothing that was working stops working.** If a client sent a field the server +did not have, that field was already being thrown away — the request was never +doing what it appeared to. What changes is that you now find out. + +Real examples from Pangolin's own clients, all of which looked like they worked: + +| Sent | Server field | What actually happened | +|---|---|---| +| `warehouse` | `warehouse_name` | The catalog was created with **no warehouse**, and the existence check was skipped | +| `type: "pangea"` | `catalog_type` | Defaulted to `Local` | +| `expires_in_days: 30` | `expires_in_hours` | Every token was 24 hours | +| `motivation` | `reason` | Access requests reached reviewers with no justification | + +If you maintain a third-party client, run it against 0.7.0 in a staging +environment and fix whatever 422s. Each one is a call that was not doing what +you thought. + +### A commit to a non-`main` branch no longer moves `main` + +Previously, committing to any branch set `current_snapshot_id` and fabricated a +`main` ref pointing at the new snapshot. A `dev`-branch commit therefore changed +what `main` readers resolved (B16). + +`current_snapshot_id` and `snapshot_log` now describe `main` alone, and a +feature-branch commit updates only its own ref. + +**If you depended on the old behaviour** — for instance a pipeline that wrote to +a branch and expected `main` to follow — you now need an explicit merge or a +`set-snapshot-ref` on `main`. + +### Tokens without a `jti` are rejected + +A JWT carrying no `jti` cannot be revoked, and was previously exempt from the +revocation check entirely (B0o). Such tokens are now refused. + +Set `PANGOLIN_ALLOW_TOKENS_WITHOUT_JTI=true` for a migration window if you have +long-lived tokens minted before this. Re-issue them and unset it. + +### OAuth no longer links accounts by email + +Identity is `(provider, subject)`. An email address adopts a pre-existing local +account only when the provider reports it as **verified** *and* its domain is +listed in `PANGOLIN_OAUTH_EMAIL_LINK_DOMAINS` (B0l). + +**If your users sign in with OAuth against accounts created locally**, set that +variable or those users will get a *new* account on next login: + +```bash +export PANGOLIN_OAUTH_EMAIL_LINK_DOMAINS=yourcompany.com,subsidiary.example +``` + +Leaving it unset is the safe default and the right choice if you do not rely on +this. + +### `PANGOLIN_DEV_MODE` no longer waives the `NO_AUTH` bind guard + +`PANGOLIN_NO_AUTH=true` now requires a loopback bind address unconditionally. +Dev mode relaxes secret *strength*, never network exposure (B0h). + +A deployment setting both flags on `0.0.0.0` **will now refuse to start**. That +configuration served every anonymous request as `TenantAdmin`. + +### `PANGOLIN_STORE_TYPE` is gone from the compose files + +It never did anything — the server reads `PANGOLIN_STORAGE_TYPE` (B9). If you +copied a compose file and edited `PANGOLIN_STORE_TYPE` to `postgres`, **you have +been running the in-memory backend**, losing all catalog metadata on every +restart. Check before you upgrade. + +### The UI reads `PUBLIC_API_URL` + +`VITE_API_URL` was never read by the client (B31). If you set it, rename it: + +```diff +- VITE_API_URL=https://pangolin.internal ++ PUBLIC_API_URL=https://pangolin.internal +``` + +Leave it empty for a same-origin reverse-proxy deployment. + +--- + +## Python SDK + +Four signatures changed, each because the old one could not work: + +```python +# permissions.grant - the old scope shape had no matching server field +- client.permissions.grant(user_id, action, scope_type, scope_id) ++ client.permissions.grant( ++ user_id=..., actions=["read"], scope_type="catalog", catalog_id=... ++ ) + +# business_metadata.delete - the `key` was ignored; this deletes everything +- client.business_metadata.delete(asset_id, key) ++ client.business_metadata.delete(asset_id) + +# request_access - `motivation` was dropped on the floor +- client.business_metadata.request_access(asset_id, motivation="...") ++ client.business_metadata.request_access(asset_id, reason="...") + +# branches.rebase - omitted a required field and always 422'd +- client.branches.rebase(branch_name, base_branch, catalog_name=...) ++ client.branches.rebase(branch_name, catalog_name=...) + +# tokens.generate - three of its four arguments were ignored +- client.tokens.generate(name, user_id, tenant_id, expires_in_days=30) ++ client.tokens.generate(username=..., tenant_id=..., expires_in_hours=720) +``` + +Also: + +* **`pyiceberg` is now an optional extra.** Install `pypangolin[iceberg]` if you + use `get_iceberg_catalog`. Importing the package no longer pulls in the whole + Iceberg stack (B41). +* **Minimum Python is 3.9.** The declared `>=3.8` was unsatisfiable with the + package's own dependencies (B40). +* **Every CLI command now exits non-zero on failure.** Scripts that relied on a + zero exit code will start failing — correctly. Commands were reporting + success for calls that raised `TypeError` internally. +* **`pangolin` is now on `PATH`** after install, via a real console script. + +## Rust CLIs + +Roughly 35 call sites targeted endpoints that do not exist. Commands that +previously printed an error and exited `0` now exit non-zero. If you have +scripts that ignored the exit code, they were ignoring failures. + +`pangolin-admin grant-permission` changed shape, because the old arguments +matched no server field: + +```diff +- pangolin-admin grant-permission --username alice --action read --resource cat ++ pangolin-admin grant-permission --user-id --action read \ ++ --scope-type catalog --catalog-id +``` + +`pangolin-admin update-user --username` now errors instead of silently doing +nothing: the server has no rename operation. + +--- + +## Storage backends + +### SQLite + +Upgrading migrates `audit_logs` to schema version 2 automatically on startup. +The previous table is preserved as `audit_logs_pre_v2` and can be dropped once +you are satisfied — it is empty in practice, because the old column set meant +**every audit write failed**. Assume you have no SQLite audit history before +0.7.0. + +If you keep backups, take one before upgrading, as with any schema change. + +### MongoDB + +**A replica set is strongly recommended.** On a standalone `mongod`: + +* transactions are unavailable, so `delete_catalog` falls back to a + sequential, non-atomic cascade (it now degrades with a warning instead of + failing outright, which it did before); +* retryable writes are unavailable — add `retryWrites=false` to your connection + string. + +Revocation records written before 0.7.0 are unreadable by the fixed code path, +because the old write used a different field name *and* type (B2). They never +worked as revocations either. Rotating the signing secret, which you should do +anyway, makes this moot. + +### Postgres + +No schema change. `list_catalogs` now returns the real `catalog_type` and +`federated_config` instead of hardcoding `Local`/`None` (B24) — if you have code +branching on a listing's `catalog_type`, it will now take the federated path +where it should have all along. + +--- + +## Verifying the upgrade + +```bash +# every artifact carries the same version +scripts/bump_version.sh 0.7.0 --check + +# the authorization matrix and cross-backend parity suites +cd pangolin +cargo test -p pangolin_api --lib authz_matrix_tests +cargo test -p pangolin_store --test store_integration +``` + +The second command runs against whichever backends you configure via +`PANGOLIN_TEST_POSTGRES_URL` and `PANGOLIN_TEST_MONGO_URL`, and skips the rest +with a note rather than passing silently. diff --git a/pangolin/.cargo/audit.toml b/pangolin/.cargo/audit.toml new file mode 100644 index 0000000..3ccadb4 --- /dev/null +++ b/pangolin/.cargo/audit.toml @@ -0,0 +1,64 @@ +# Advisories accepted deliberately, each with the reason it cannot be fixed by +# upgrading and what the exposure actually is. +# +# This file is not a way to make `cargo audit` quiet. Every entry is a specific +# advisory ID, so a *new* advisory - including a new one against these same +# crates - still fails CI. Re-check this list whenever dependencies move; the +# right outcome for most of these is deletion once upstream releases. +# +# Reviewed: 2026-08-10, against Cargo.lock at pangolin 0.7.0. + +[advisories] +ignore = [ + # ---- No fixed version exists at any release ---- + + # RUSTSEC-2023-0071 - Marvin Attack: RSA key recovery through timing side + # channels. The `rsa` crate has no fixed release; the maintainers' guidance + # is that a constant-time rewrite is required. + # + # It is never compiled into Pangolin. Its only route in is `sqlx-mysql`, + # which is an *optional* dependency of `sqlx`: this workspace builds sqlx + # with `postgres` and `sqlite` only, and there is no MySQL `CatalogStore`. + # `cargo tree -i rsa --target all` finds no path to it. It appears here + # solely because `cargo audit` scans Cargo.lock, which records optional + # dependencies whether or not their feature is enabled. + "RUSTSEC-2023-0071", + + # ---- Held back by an upstream that has not released ---- + + # RUSTSEC-2026-0194 / -0195 - quick-xml quadratic attribute checking and + # unbounded namespace allocation, both DoS. Two copies are in the tree and + # neither is ours to move: + # * 0.31.0 via azure_core 0.20, which is behind the OPTIONAL `azure` + # feature and absent from a default build; + # * 0.37.5 via object_store 0.11, which is not optional. + # object_store 0.14 carries a fixed quick-xml but is three minor versions + # ahead with breaking API changes across the IO layer. That upgrade is + # worth doing on its own terms, not as a drive-by inside a security fix. + # Exposure: both parse XML from the object store's own responses, so an + # attacker needs control of the storage endpoint's replies. + "RUSTSEC-2026-0194", + "RUSTSEC-2026-0195", + + # RUSTSEC-2026-0098 / -0099 / -0104 - rustls-webpki name-constraint and CRL + # defects. The fixed 0.103.13 IS in the tree and is what rustls 0.23 uses. + # The flagged 0.101.7 comes in alongside it via rustls 0.21, pulled by + # aws-smithy-http-client 1.2.0. No release of the AWS SDK has dropped that + # second TLS stack yet. Pangolin's own TLS goes through the 0.23 path. + "RUSTSEC-2026-0098", + "RUSTSEC-2026-0099", + "RUSTSEC-2026-0104", + + # RUSTSEC-2026-0174 - http-types can build Authorization and + # WwwAuthenticate header values that violate ASCII invariants. Via + # azure_core 0.20, so behind the optional `azure` feature. http-types is + # unmaintained and azure_core 0.20 is the last release of that line before + # the Azure SDK restructure; moving off it is an Azure-support project. + "RUSTSEC-2026-0174", + + # RUSTSEC-2026-0097 - `rand` is unsound when a custom logger calls + # `rand::rng()` re-entrantly. Pangolin installs no custom logger that draws + # randomness; `tracing-subscriber`'s formatters do not. Both 0.8 and 0.9 are + # in the tree transitively and neither has a fixed release. + "RUSTSEC-2026-0097", +] diff --git a/pangolin/.test_output.txt b/pangolin/.test_output.txt deleted file mode 100644 index 643cc43..0000000 --- a/pangolin/.test_output.txt +++ /dev/null @@ -1,391 +0,0 @@ -warning: unused import: `std::collections::HashMap` - --> pangolin_core/src/business_metadata.rs:3:5 - | -3 | use std::collections::HashMap; - | ^^^^^^^^^^^^^^^^^^^^^^^^^ - | - = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default - -warning: `pangolin_core` (lib) generated 1 warning (run `cargo fix --lib -p pangolin_core` to apply 1 suggestion) -warning: unused import: `CatalogType` - --> pangolin_store/src/memory.rs:7:14 - | -7 | ...g, CatalogType, N... - | ^^^^^^^^^^^ - | - = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default - -warning: unused import: `BusinessMetadata` - --> pangolin_store/src/memory.rs:16:40 - | -16 | ...::{BusinessMetadata, A... - | ^^^^^^^^^^^^^^^^ - -warning: unused import: `PermissionGrant` - --> pangolin_store/src/postgres.rs:9:51 - | -9 | ...n, PermissionGrant, U... - | ^^^^^^^^^^^^^^^ - -warning: unused import: `DateTime` - --> pangolin_store/src/postgres.rs:16:14 - | -16 | ...::{DateTime, U... - | ^^^^^^^^ - -warning: unused imports: `OAuthProvider` and `UserRole as CoreUserRole` - --> pangolin_store/src/mongo.rs:13:33 - | -13 | ...r, UserRole as CoreUserRole, OAuthProvider}; - | ^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^ - -warning: unused import: `PermissionGrant` - --> pangolin_store/src/mongo.rs:14:51 - | -14 | ...n, PermissionGrant, U... - | ^^^^^^^^^^^^^^^ - -warning: unused import: `RequestStatus` - --> pangolin_store/src/mongo.rs:15:55 - | -15 | ...t, RequestStatus}; - | ^^^^^^^^^^^^^ - -warning: unused import: `PermissionGrant` - --> pangolin_store/src/sqlite.rs:5:51 - | -5 | ...n, PermissionGrant, U... - | ^^^^^^^^^^^^^^^ - -warning: unused import: `DateTime` - --> pangolin_store/src/sqlite.rs:12:14 - | -12 | ...::{DateTime, U... - | ^^^^^^^^ - -warning: unused import: `crate::memory::MemoryStore` - --> pangolin_store/src/tests/multi_cloud.rs:1:5 - | -1 | use crate::memory::MemoryStore; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unused import: `crate::CatalogStore` - --> pangolin_store/src/tests/multi_cloud.rs:2:5 - | -2 | use crate::CatalogStore; - | ^^^^^^^^^^^^^^^^^^^ - -warning: unused imports: `Credentials` and `Signer` - --> pangolin_store/src/tests/multi_cloud.rs:3:21 - | -3 | ...::{Signer, Credentials}; - | ^^^^^^ ^^^^^^^^^^^ - -warning: unused imports: `Tenant`, `VendingStrategy`, and `Warehouse` - --> pangolin_store/src/tests/multi_cloud.rs:4:28 - | -4 | ...::{Warehouse, VendingStrategy, Tenant}; - | ^^^^^^^^^ ^^^^^^^^^^^^^^^ ^^^^^^ - -warning: unused import: `std::collections::HashMap` - --> pangolin_store/src/tests/multi_cloud.rs:5:5 - | -5 | use std::collections::HashMap; - | ^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unused import: `uuid::Uuid` - --> pangolin_store/src/tests/multi_cloud.rs:6:5 - | -6 | use uuid::Uuid; - | ^^^^^^^^^^ - -warning: unused imports: `AuditAction`, `AuditLogEntry`, `AuditLogFilter`, `AuditResult`, and `ResourceType` - --> pangolin_store/src/tests/audit_tests.rs:1:28 - | - 1 | ...::{AuditAction, AuditLogEntry, AuditLogFilter, AuditResult, ResourceType}; - | ^^^^^^^^^^^ ^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ ^^^^^^^^^^^ ^^^^^^^^^^^^ - | -help: if this is a test module, consider adding a `#[cfg(test)]` to the containing module - --> pangolin_store/src/tests/mod.rs:154:1 - | -154 | pub mod audit_tests; - | ^^^^^^^^^^^^^^^^^^^^ - -warning: unused imports: `CatalogStore` and `MemoryStore` - --> pangolin_store/src/tests/audit_tests.rs:2:13 - | - 2 | ...::{CatalogStore, MemoryStore}; - | ^^^^^^^^^^^^ ^^^^^^^^^^^ - | -help: if this is a test module, consider adding a `#[cfg(test)]` to the containing module - --> pangolin_store/src/tests/mod.rs:154:1 - | -154 | pub mod audit_tests; - | ^^^^^^^^^^^^^^^^^^^^ - -warning: unused import: `uuid::Uuid` - --> pangolin_store/src/tests/audit_tests.rs:3:5 - | - 3 | use uuid::Uuid; - | ^^^^^^^^^^ - | -help: if this is a test module, consider adding a `#[cfg(test)]` to the containing module - --> pangolin_store/src/tests/mod.rs:154:1 - | -154 | pub mod audit_tests; - | ^^^^^^^^^^^^^^^^^^^^ - -warning: unused import: `chrono::Utc` - --> pangolin_store/src/tests/audit_tests.rs:4:5 - | - 4 | use chrono::Utc; - | ^^^^^^^^^^^ - | -help: if this is a test module, consider adding a `#[cfg(test)]` to the containing module - --> pangolin_store/src/tests/mod.rs:154:1 - | -154 | pub mod audit_tests; - | ^^^^^^^^^^^^^^^^^^^^ - -warning: unused imports: `Duration` and `Utc` - --> pangolin_store/src/azure_signer.rs:2:14 - | -2 | ...::{Duration, Utc}; - | ^^^^^^^^ ^^^ - -warning: unused import: `azure_storage_blobs::prelude::*` - --> pangolin_store/src/azure_signer.rs:4:5 - | -4 | use azure_storage_blobs::prelude::*; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unused imports: `DateTime` and `Utc` - --> pangolin_store/src/gcp_signer.rs:2:14 - | -2 | ...::{DateTime, Utc}; - | ^^^^^^^^ ^^^ - -warning: unused import: `std::sync::Arc` - --> pangolin_store/src/object_store_factory.rs:4:5 - | -4 | use std::sync::Arc; - | ^^^^^^^^^^^^^^ - -warning: unused import: `TimeZone` - --> pangolin_store/src/postgres.rs:16:29 - | -16 | ...c, TimeZone}; - | ^^^^^^^^ - -warning: unused variable: `now` - --> pangolin_store/src/memory.rs:1184:13 - | -1184 | ...et now = ... - | ^^^ help: if this is intentional, prefix it with an underscore: `_now` - | - = note: `#[warn(unused_variables)]` (part of `#[warn(unused)]`) on by default - -warning: unused variable: `client` - --> pangolin_store/src/gcp_signer.rs:52:13 - | -52 | ...et client = ... - | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_client` - -warning: value assigned to `param_count` is never read - --> pangolin_store/src/postgres.rs:1168:17 - | -1168 | ... param_count += 1; - | ^^^^^^^^^^^^^^^^ - | - = help: maybe it is overwritten before being read? - = note: `#[warn(unused_assignments)]` (part of `#[warn(unused)]`) on by default - -warning: value assigned to `param_count` is never read - --> pangolin_store/src/postgres.rs:1312:17 - | -1312 | ... param_count += 1; - | ^^^^^^^^^^^^^^^^ - | - = help: maybe it is overwritten before being read? - -warning: unused variable: `tenant_id` - --> pangolin_store/src/mongo.rs:178:38 - | -178 | ...f, tenant_id: U... - | ^^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_tenant_id` - -warning: variable does not need to be mutable - --> pangolin_store/src/mongo.rs:357:13 - | -357 | ...et mut filter = ... - | ----^^^^^^ - | | - | help: remove this `mut` - | - = note: `#[warn(unused_mut)]` (part of `#[warn(unused)]`) on by default - -warning: unused variable: `p` - --> pangolin_store/src/mongo.rs:362:21 - | -362 | ...me(p) = pa... - | ^ help: if this is intentional, prefix it with an underscore: `_p` - -warning: unused variable: `s3_nested` - --> pangolin_store/src/mongo.rs:1662:18 - | -1662 | ...et s3_nested = ... - | ^^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_s3_nested` - -warning: field `signer` is never read - --> pangolin_store/src/memory.rs:37:5 - | -22 | pub struct MemoryStore { - | ----------- field in this struct -... -37 | signer: crate::signer... - | ^^^^^^ - | - = note: `MemoryStore` has a derived impl for the trait `Clone`, but this is intentionally ignored during dead code analysis - = note: `#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default - -warning: field `key` is never read - --> pangolin_store/src/signer.rs:41:5 - | -40 | pub struct SignerImpl { - | ---------- field in this struct -41 | key: String, - | ^^^ - | - = note: `SignerImpl` has a derived impl for the trait `Clone`, but this is intentionally ignored during dead code analysis - -warning: methods `revoke_token`, `is_token_revoked`, and `cleanup_expired_tokens` are never used - --> pangolin_store/src/postgres.rs:1809:14 - | -1750 | impl PostgresStore { - | ------------------ methods in this implementation -... -1809 | async fn revoke_token(&self, token... - | ^^^^^^^^^^^^ -... -1821 | async fn is_token_revoked(&self, t... - | ^^^^^^^^^^^^^^^^ -... -1831 | async fn cleanup_expired_tokens(&s... - | ^^^^^^^^^^^^^^^^^^^^^^ - -warning: field `client` is never read - --> pangolin_store/src/mongo.rs:27:5 - | -26 | pub struct MongoStore { - | ---------- field in this struct -27 | client: Client, - | ^^^^^^ - | - = note: `MongoStore` has a derived impl for the trait `Clone`, but this is intentionally ignored during dead code analysis - -warning: methods `catalogs`, `branches`, `tags`, `commits`, and `audit_logs` are never used - --> pangolin_store/src/mongo.rs:63:8 - | -31 | impl MongoStore { - | --------------- methods in this implementation -... -63 | fn catalogs(&sel... - | ^^^^^^^^ -... -75 | fn branches(&sel... - | ^^^^^^^^ -... -79 | fn tags(&self) -... - | ^^^^ -... -83 | fn commits(&self... - | ^^^^^^^ -... -87 | fn audit_logs(&s... - | ^^^^^^^^^^ - -warning: methods `revoke_token`, `is_token_revoked`, and `cleanup_expired_tokens` are never used - --> pangolin_store/src/sqlite.rs:2003:14 - | -1941 | impl SqliteStore { - | ---------------- methods in this implementation -... -2003 | async fn revoke_token(&self, token... - | ^^^^^^^^^^^^ -... -2018 | async fn is_token_revoked(&self, t... - | ^^^^^^^^^^^^^^^^ -... -2029 | async fn cleanup_expired_tokens(&s... - | ^^^^^^^^^^^^^^^^^^^^^^ - -warning: function `test_azure_path_parsing` is never used - --> pangolin_store/src/tests/multi_cloud.rs:129:4 - | -129 | fn test_azure_path_parsing() { - | ^^^^^^^^^^^^^^^^^^^^^^^ - -warning: fields `account_name` and `account_key` are never read - --> pangolin_store/src/azure_signer.rs:7:5 - | -6 | pub struct AzureSigner { - | ----------- fields in this struct -7 | account_name: String, - | ^^^^^^^^^^^^ -8 | account_key: String, - | ^^^^^^^^^^^ - -warning: `pangolin_store` (lib) generated 40 warnings (run `cargo fix --lib -p pangolin_store` to apply 29 suggestions) -warning: unused import: `CatalogStore` - --> pangolin_store/tests/store_integration.rs:2:58 - | -2 | ...e, CatalogStore, - | ^^^^^^^^^^^^ - | - = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default - -warning: `pangolin_store` (test "store_integration") generated 1 warning (run `cargo fix --test "store_integration" -p pangolin_store` to apply 1 suggestion) - Finished `test` profile [unoptimized + debuginfo] target(s) in 0.32s - Running tests/store_integration.rs (target/debug/deps/store_integration-9aac1c19d52cf927) - -running 4 tests -test test_memory_store_regression ... ok -test test_sqlite_store_regression ... FAILED -test test_postgres_store_regression ... FAILED -test test_mongo_store_regression ... FAILED - -failures: - ----- test_sqlite_store_regression stdout ---- - -thread 'test_sqlite_store_regression' (314071) panicked at /home/alexmerced/development/personal/Personal/2026/pangolin/pangolin/pangolin_store/src/tests/mod.rs:51:90: -Failed to create asset: error returned from database: (code: 1) no such table: assets - -Caused by: - (code: 1) no such table: assets -note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace - ----- test_postgres_store_regression stdout ---- - -thread 'test_postgres_store_regression' (314070) panicked at /home/alexmerced/development/personal/Personal/2026/pangolin/pangolin/pangolin_store/src/tests/mod.rs:51:90: -Failed to create asset: error returned from database: insert or update on table "assets" violates foreign key constraint "assets_tenant_id_fkey" - -Caused by: - insert or update on table "assets" violates foreign key constraint "assets_tenant_id_fkey" - ----- test_mongo_store_regression stdout ---- - -thread 'test_mongo_store_regression' (314069) panicked at /home/alexmerced/development/personal/Personal/2026/pangolin/pangolin/pangolin_store/src/tests/mod.rs:56:5: -assertion `left == right` failed: Initial metadata location mismatch - left: None - right: Some("s3://bucket/path/v1.json") - - -failures: - test_mongo_store_regression - test_postgres_store_regression - test_sqlite_store_regression - -test result: FAILED. 1 passed; 3 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.87s - -error: test failed, to rerun pass `-p pangolin_store --test store_integration` diff --git a/pangolin/Cargo.lock b/pangolin/Cargo.lock index 57cd327..8d0dc11 100644 --- a/pangolin/Cargo.lock +++ b/pangolin/Cargo.lock @@ -29,9 +29,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -44,18 +44,18 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android_system_properties" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" dependencies = [ "libc", ] [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -68,15 +68,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -103,9 +103,18 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.100" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] [[package]] name = "assert-json-diff" @@ -160,11 +169,11 @@ dependencies = [ [[package]] name = "async-lock" -version = "3.4.1" +version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd03604047cee9b6ce9de9f70c6cd540a0520c813cbd49bae61f33ab80ed1dc" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" dependencies = [ - "event-listener 5.4.1", + "event-listener 5.4.2", "event-listener-strategy", "pin-project-lite", ] @@ -182,16 +191,16 @@ dependencies = [ "async-task", "blocking", "cfg-if", - "event-listener 5.4.1", + "event-listener 5.4.2", "futures-lite 2.6.1", "rustix", ] [[package]] name = "async-signal" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c070bbf59cd3570b6b2dd54cd772527c7c3620fce8be898406dd3ed6adc64c" +checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485" dependencies = [ "async-io", "async-lock", @@ -224,7 +233,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -235,13 +244,13 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 3.0.3", ] [[package]] @@ -261,15 +270,15 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.5.10" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b49afaa341e8dd8577e1a2200468f98956d6eda50bcf4a53246cc00174ba924" +checksum = "1b180a3c8b55960db3426d8964b8745e652466a1a49fe1a2eda828046d30b5e4" dependencies = [ "aws-credential-types", "aws-runtime", @@ -277,17 +286,18 @@ dependencies = [ "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", - "aws-smithy-http 0.60.12", - "aws-smithy-json 0.60.7", + "aws-smithy-http", + "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.3.0", + "fastrand 2.5.0", "hex", - "http 0.2.12", - "ring", + "http 1.5.0", + "sha1 0.10.7", "time", "tokio", "tracing", @@ -297,9 +307,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.11" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cd362783681b15d136480ad555a099e82ecd8e2d10a841e14dfd0078d67fee3" +checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -309,9 +319,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.15.2" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a88aab2464f1f25453baa7a07c84c5b7684e274054ba06817f382357f77a288" +checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" dependencies = [ "aws-lc-sys", "zeroize", @@ -319,35 +329,39 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.35.0" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45afffdee1e7c9126814751f88dddc747f41d91da16c9551a0f1e8a11e788a1" +checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] name = "aws-runtime" -version = "1.5.17" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d81b5b2898f6798ad58f484856768bca817e3cd9de0974c24ae0f1113fe88f1b" +checksum = "c9007227e10b5fed2f3e0a2beff489211e2b5604c400b7a9d5d81ca9d64c24bb" dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", "aws-smithy-eventstream", - "aws-smithy-http 0.62.6", + "aws-smithy-http", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.3.0", + "bytes-utils", + "fastrand 2.5.0", "http 0.2.12", + "http 1.5.0", "http-body 0.4.6", + "http-body 1.1.0", "percent-encoding", "pin-project-lite", "tracing", @@ -356,127 +370,141 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.118.0" +version = "1.141.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e6b7079f85d9ea9a70643c9f89f50db70f5ada868fa9cfe08c1ffdf51abc13" +checksum = "d9f9420d3a2467eed22ed3635ca653653162c386a0b0f65c78189f9bd3c1379e" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-sigv4", "aws-smithy-async", "aws-smithy-checksums", "aws-smithy-eventstream", - "aws-smithy-http 0.62.6", - "aws-smithy-json 0.61.9", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-smithy-xml", "aws-types", "bytes", - "fastrand 2.3.0", + "fastrand 2.5.0", "hex", - "hmac", + "hmac 0.13.0", "http 0.2.12", - "http 1.4.0", - "http-body 0.4.6", + "http 1.5.0", + "http-body 1.1.0", "lru", "percent-encoding", "regex-lite", - "sha2", + "sha2 0.11.0", "tracing", "url", ] [[package]] name = "aws-sdk-sso" -version = "1.91.0" +version = "1.105.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee6402a36f27b52fe67661c6732d684b2635152b676aa2babbfb5204f99115d" +checksum = "6ffd0fbe7873cb548a7aa60f9573c268fff94155397fd4f14dc9f1ecaaab8516" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.62.6", - "aws-smithy-json 0.61.9", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.3.0", + "fastrand 2.5.0", "http 0.2.12", + "http 1.5.0", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-ssooidc" -version = "1.93.0" +version = "1.107.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a45a7f750bbd170ee3677671ad782d90b894548f4e4ae168302c57ec9de5cb3e" +checksum = "175763eb222a46377df7aa257a3bca980ab3e96703fefc8f4d0b8da6ad2e254c" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.62.6", - "aws-smithy-json 0.61.9", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.3.0", + "fastrand 2.5.0", "http 0.2.12", + "http 1.5.0", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-sts" -version = "1.50.0" +version = "1.110.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ada54e5f26ac246dc79727def52f7f8ed38915cb47781e2a72213957dc3a7d5" +checksum = "dd8b14781dfbff48984017d57167b6ea0b6471c6920ec52b44a2677c7feb3c13" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.60.12", - "aws-smithy-json 0.60.7", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-smithy-xml", "aws-types", + "fastrand 2.5.0", "http 0.2.12", - "once_cell", + "http 1.5.0", "regex-lite", "tracing", ] [[package]] name = "aws-sigv4" -version = "1.3.7" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e523e1c4e8e7e8ff219d732988e22bfeae8a1cafdbe6d9eca1546fa080be7c" +checksum = "723c2234ad7511ceef63eab016b7ba6ff7c55590fefb96fa8467af014a07309f" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", - "aws-smithy-http 0.62.6", + "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "crypto-bigint 0.5.5", + "crypto-bigint", "form_urlencoded", "hex", - "hmac", + "hmac 0.13.0", "http 0.2.12", - "http 1.4.0", + "http 1.5.0", "p256", "percent-encoding", - "ring", - "sha2", + "sha2 0.11.0", "subtle", "time", "tracing", @@ -485,9 +513,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.7" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ee19095c7c4dda59f1697d028ce704c24b2d33c6718790c7f1d5a3015b4107c" +checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316" dependencies = [ "futures-util", "pin-project-lite", @@ -496,29 +524,30 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.63.12" +version = "0.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87294a084b43d649d967efe58aa1f9e0adc260e13a6938eb904c0ae9b45824ae" +checksum = "b67ecd999972b58e67cab052f5129906c08c25883bd0788ceefc55ef97d61307" dependencies = [ - "aws-smithy-http 0.62.6", + "aws-smithy-http", "aws-smithy-types", "bytes", "crc-fast", "hex", - "http 0.2.12", - "http-body 0.4.6", - "md-5", + "http 1.5.0", + "http-body 1.1.0", + "http-body-util", + "md-5 0.11.0", "pin-project-lite", - "sha1", - "sha2", + "sha1 0.11.0", + "sha2 0.11.0", "tracing", ] [[package]] name = "aws-smithy-eventstream" -version = "0.60.14" +version = "0.61.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc12f8b310e38cad85cf3bef45ad236f470717393c613266ce0a89512286b650" +checksum = "5a9381123ab62d20c13082b151f30f962a3b112b727345394536dfa39a482944" dependencies = [ "aws-smithy-types", "bytes", @@ -527,29 +556,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.12" +version = "0.64.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7809c27ad8da6a6a68c454e651d4962479e81472aa19ae99e59f9aba1f9713cc" -dependencies = [ - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "bytes-utils", - "futures-core", - "http 0.2.12", - "http-body 0.4.6", - "once_cell", - "percent-encoding", - "pin-project-lite", - "pin-utils", - "tracing", -] - -[[package]] -name = "aws-smithy-http" -version = "0.62.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826141069295752372f8203c17f28e30c464d22899a43a0c9fd9c458d469c88b" +checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -558,9 +567,9 @@ dependencies = [ "bytes-utils", "futures-core", "futures-util", - "http 0.2.12", - "http 1.4.0", - "http-body 0.4.6", + "http 1.5.0", + "http-body 1.1.0", + "http-body-util", "percent-encoding", "pin-project-lite", "pin-utils", @@ -569,89 +578,87 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.1.5" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59e62db736db19c488966c8d787f52e6270be565727236fd5579eaa301e7bc4a" +checksum = "635d23afda0a6ab48d666c4d447c4873e8d1e83518a2be2093122397e50b838e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", "h2 0.3.27", - "h2 0.4.12", + "h2 0.4.15", "http 0.2.12", - "http 1.4.0", + "http 1.5.0", "http-body 0.4.6", "hyper 0.14.32", - "hyper 1.8.1", + "hyper 1.11.0", "hyper-rustls 0.24.2", - "hyper-rustls 0.27.7", + "hyper-rustls 0.27.9", "hyper-util", "pin-project-lite", "rustls 0.21.12", - "rustls 0.23.35", + "rustls 0.23.43", "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", - "tower 0.5.2", + "tower 0.5.3", "tracing", ] [[package]] name = "aws-smithy-json" -version = "0.60.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4683df9469ef09468dad3473d129960119a0d3593617542b7d52086c8486f2d6" -dependencies = [ - "aws-smithy-types", -] - -[[package]] -name = "aws-smithy-json" -version = "0.61.9" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49fa1213db31ac95288d981476f78d05d9cbb0353d22cdf3472cc05bb02f6551" +checksum = "3dc65a121adb4b33729919fcfa14fa36fb33c1555a8f06bb0e2188dbfdc1d9ef" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", ] [[package]] name = "aws-smithy-observability" -version = "0.1.5" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f616c3f2260612fe44cede278bafa18e73e6479c4e393e2c4518cf2a9a228a" +checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c" dependencies = [ "aws-smithy-runtime-api", ] [[package]] name = "aws-smithy-query" -version = "0.60.9" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae5d689cf437eae90460e944a58b5668530d433b4ff85789e69d2f2a556e057d" +checksum = "512346c7212ab7436df2d77a16d976a468ae44a418835511d2a69269810aaf62" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", + "aws-smithy-xml", "urlencoding", ] [[package]] name = "aws-smithy-runtime" -version = "1.9.6" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65fda37911905ea4d3141a01364bc5509a0f32ae3f3b22d6e330c0abfb62d247" +checksum = "07505b34e8f4b3591a4fa69e9792b52289b95488dbbc68c3c0075b7bedb245e1" dependencies = [ "aws-smithy-async", - "aws-smithy-http 0.62.6", + "aws-smithy-http", "aws-smithy-http-client", "aws-smithy-observability", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "bytes", - "fastrand 2.3.0", + "fastrand 2.5.0", "http 0.2.12", - "http 1.4.0", + "http 1.5.0", "http-body 0.4.6", - "http-body 1.0.1", + "http-body 1.1.0", + "http-body-util", "pin-project-lite", "pin-utils", "tokio", @@ -660,35 +667,58 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.9.3" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0d43d899f9e508300e587bf582ba54c27a452dd0a9ea294690669138ae14a2" +checksum = "3b98f2e1fd67ec06618f9c291e5e495a468e60519e44c9c1979cd0521f3affdb" dependencies = [ "aws-smithy-async", + "aws-smithy-runtime-api-macros", "aws-smithy-types", "bytes", "http 0.2.12", - "http 1.4.0", + "http 1.5.0", "pin-project-lite", "tokio", "tracing", "zeroize", ] +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "aws-smithy-schema" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.5.0", +] + [[package]] name = "aws-smithy-types" -version = "1.3.5" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "905cb13a9895626d49cf2ced759b062d913834c7482c38e49557eac4e6193f01" +checksum = "d6dc683efb34b9e755675b37fedbe0103141e5b6df7bdc9eb6967756a8c167d8" dependencies = [ "base64-simd", "bytes", "bytes-utils", "futures-core", "http 0.2.12", - "http 1.4.0", + "http 1.5.0", "http-body 0.4.6", - "http-body 1.0.1", + "http-body 1.1.0", "http-body-util", "itoa", "num-integer", @@ -703,22 +733,26 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.13" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11b2f670422ff42bf7065031e72b45bc52a3508bd089f743ea90731ca2b6ea57" +checksum = "ce84f71c72fee2cbbadde6e7d082f5fb466e3a84733855295fa7aafd1b31b7d8" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", "xmlparser", ] [[package]] name = "aws-types" -version = "1.3.11" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d980627d2dd7bfc32a3c025685a033eeab8d365cc840c631ef59d1b8f428164" +checksum = "eec1cd5469f328c782dc3e33d4153cf118a54e33cbb3356d60d16f89883e1f94" dependencies = [ "aws-credential-types", "aws-smithy-async", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "rustc_version", "tracing", @@ -734,10 +768,10 @@ dependencies = [ "axum-core", "bytes", "futures-util", - "http 1.4.0", - "http-body 1.0.1", + "http 1.5.0", + "http-body 1.1.0", "http-body-util", - "hyper 1.8.1", + "hyper 1.11.0", "hyper-util", "itoa", "matchit", @@ -752,7 +786,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.2", "tokio", - "tower 0.5.2", + "tower 0.5.3", "tower-layer", "tower-service", "tracing", @@ -767,8 +801,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.4.0", - "http-body 1.0.1", + "http 1.5.0", + "http-body 1.1.0", "http-body-util", "mime", "pin-project-lite", @@ -790,19 +824,19 @@ dependencies = [ "bytes", "dyn-clone", "futures", - "getrandom 0.2.16", - "hmac", + "getrandom 0.2.17", + "hmac 0.12.1", "http-types", "once_cell", "paste", "pin-project", "quick-xml 0.31.0", - "rand 0.8.5", - "reqwest 0.12.26", + "rand 0.8.7", + "reqwest 0.12.28", "rustc_version", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "time", "tracing", "url", @@ -888,9 +922,9 @@ dependencies = [ [[package]] name = "base16ct" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" [[package]] name = "base64" @@ -922,9 +956,9 @@ dependencies = [ [[package]] name = "base64ct" -version = "1.6.0" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "bcrypt" @@ -934,7 +968,7 @@ checksum = "e65938ed058ef47d92cf8b346cc76ef48984572ade631927e9937b5ffc7662c7" dependencies = [ "base64 0.22.1", "blowfish", - "getrandom 0.2.16", + "getrandom 0.2.17", "subtle", "zeroize", ] @@ -956,18 +990,18 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.10.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" dependencies = [ "serde_core", ] [[package]] name = "bitvec" -version = "1.0.1" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" dependencies = [ "funty", "radium", @@ -984,6 +1018,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "blocking" version = "1.6.2" @@ -1017,13 +1060,13 @@ dependencies = [ "base64 0.22.1", "bitvec", "chrono", - "getrandom 0.2.16", + "getrandom 0.2.17", "getrandom 0.3.4", "hex", "indexmap", "js-sys", "once_cell", - "rand 0.9.2", + "rand 0.9.5", "serde", "serde_bytes", "serde_json", @@ -1033,9 +1076,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.19.1" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "byteorder" @@ -1045,9 +1088,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "bytes-utils" @@ -1061,9 +1104,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.49" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -1085,15 +1128,26 @@ checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" [[package]] name = "cfg_aliases" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] [[package]] name = "chrono" -version = "0.4.42" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -1109,15 +1163,15 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] [[package]] name = "clap" -version = "4.5.53" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ "clap_builder", "clap_derive", @@ -1125,9 +1179,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.53" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" dependencies = [ "anstream", "anstyle", @@ -1137,21 +1191,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.49" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.111", + "syn 3.0.3", ] [[package]] name = "clap_lex" -version = "0.7.6" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "clipboard-win" @@ -1164,18 +1218,34 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.56" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b042e5d8a74ae91bb0961acd039822472ec99f8ab0948cbf6d1369588f8be586" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "colorchoice" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] [[package]] name = "concurrent-queue" @@ -1205,6 +1275,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -1220,16 +1296,16 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "once_cell", "tiny-keccak", ] [[package]] name = "const_fn" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f8a2ca5ac02d09563609681103aada9e1777d54fc57a5acd7a41404f9c93b6e" +checksum = "413d67b29ef1021b4d60f4aa1e925ca031751e213832b4b1d588fae623c05c60" [[package]] name = "convert_case" @@ -1275,6 +1351,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.4.0" @@ -1286,21 +1371,18 @@ dependencies = [ [[package]] name = "crc-catalog" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "crc-fast" -version = "1.6.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ddc2d09feefeee8bd78101665bd8645637828fa9317f9f292496dbbd8c65ff3" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" dependencies = [ - "crc", - "digest", - "rand 0.9.2", - "regex", - "rustversion", + "digest 0.10.7", + "spin 0.10.1", ] [[package]] @@ -1320,36 +1402,36 @@ checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" [[package]] name = "crossbeam-channel" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "crunchy" @@ -1359,9 +1441,9 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto-bigint" -version = "0.4.9" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ "generic-array", "rand_core 0.6.4", @@ -1370,23 +1452,22 @@ dependencies = [ ] [[package]] -name = "crypto-bigint" -version = "0.5.5" +name = "crypto-common" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ - "rand_core 0.6.4", - "subtle", + "generic-array", + "typenum", ] [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "generic-array", - "typenum", + "hybrid-array", ] [[package]] @@ -1410,11 +1491,20 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "darling" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ "darling_core", "darling_macro", @@ -1422,27 +1512,26 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" dependencies = [ - "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "darling_macro" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -1460,9 +1549,9 @@ dependencies = [ [[package]] name = "dashmap" -version = "6.1.0" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -1474,9 +1563,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.9.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" [[package]] name = "deadpool" @@ -1496,34 +1585,23 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" -[[package]] -name = "der" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" -dependencies = [ - "const-oid", - "zeroize", -] - [[package]] name = "der" version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "pem-rfc7468", "zeroize", ] [[package]] name = "deranged" -version = "0.5.5" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "powerfmt", "serde_core", ] @@ -1535,40 +1613,40 @@ checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "derive-where" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef941ded77d15ca19b40374869ac6000af1c9f2a4c0f3d4c70926287e6364a8f" +checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "derive_more" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10b768e943bed7bf2cab53df09f4bc34bfd217cdb57d971e769874c9a6710618" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" dependencies = [ "derive_more-impl", ] [[package]] name = "derive_more-impl" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d286bfdaf75e988b4a78e013ecd79c581e06399ab53fbacd2d916c2f904f30b" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" dependencies = [ "convert_case", "proc-macro2", "quote", "rustc_version", - "syn 2.0.111", + "syn 2.0.119", "unicode-xid", ] @@ -1591,12 +1669,24 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "directories" version = "5.0.1" @@ -1641,13 +1731,13 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 3.0.3", ] [[package]] @@ -1670,39 +1760,41 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "ecdsa" -version = "0.14.8" +version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der 0.6.1", + "der", + "digest 0.10.7", "elliptic-curve", "rfc6979", - "signature 1.6.4", + "signature", + "spki", ] [[package]] name = "either" -version = "1.15.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" dependencies = [ "serde", ] [[package]] name = "elliptic-curve" -version = "0.12.3" +version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ "base16ct", - "crypto-bigint 0.4.9", - "der 0.6.1", - "digest", + "crypto-bigint", + "digest 0.10.7", "ff", "generic-array", "group", - "pkcs8 0.9.0", + "pem-rfc7468", + "pkcs8", "rand_core 0.6.4", "sec1", "subtle", @@ -1730,18 +1822,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" -[[package]] -name = "enum-as-inner" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.111", -] - [[package]] name = "env_logger" version = "0.10.2" @@ -1796,11 +1876,10 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "5.4.1" +version = "5.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" dependencies = [ - "concurrent-queue", "parking", "pin-project-lite", ] @@ -1811,7 +1890,7 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" dependencies = [ - "event-listener 5.4.1", + "event-listener 5.4.2", "pin-project-lite", ] @@ -1826,9 +1905,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.3.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "fd-lock" @@ -1843,9 +1922,9 @@ dependencies = [ [[package]] name = "ff" -version = "0.12.1" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" dependencies = [ "rand_core 0.6.4", "subtle", @@ -1853,15 +1932,15 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.5" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "flate2" -version = "1.1.5" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", @@ -1875,7 +1954,7 @@ checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", - "spin", + "spin 0.9.9", ] [[package]] @@ -1890,6 +1969,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -1928,9 +2013,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" dependencies = [ "futures-channel", "futures-core", @@ -1943,9 +2028,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" dependencies = [ "futures-core", "futures-sink", @@ -1953,15 +2038,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-executor" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" dependencies = [ "futures-core", "futures-task", @@ -1981,9 +2066,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" [[package]] name = "futures-lite" @@ -2006,7 +2091,7 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" dependencies = [ - "fastrand 2.3.0", + "fastrand 2.5.0", "futures-core", "futures-io", "parking", @@ -2015,32 +2100,32 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "futures-sink" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-channel", "futures-core", @@ -2050,30 +2135,30 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] [[package]] name = "gcp_auth" -version = "0.12.5" +version = "0.12.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d24fd9357c4d0ae97d90b38e8fabeb07f9590f6944f4f65f50b5110d7c3882b6" +checksum = "26d27dbcc645b60b8e7f6e2868a9d7102ece97d1bb49c1288b5321fcc67f7260" dependencies = [ "async-trait", "base64 0.22.1", "bytes", "chrono", - "http 1.4.0", + "http 1.5.0", "http-body-util", - "hyper 1.8.1", - "hyper-rustls 0.27.7", + "hyper 1.11.0", + "hyper-rustls 0.27.9", "hyper-util", "ring", + "rustls 0.23.43", "rustls-pki-types", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.20", "tokio", "tracing", "tracing-futures", @@ -2088,6 +2173,7 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] @@ -2103,9 +2189,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", "js-sys", @@ -2123,11 +2209,25 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + [[package]] name = "google-cloud-auth" version = "0.16.0" @@ -2140,7 +2240,7 @@ dependencies = [ "google-cloud-token", "home", "jsonwebtoken", - "reqwest 0.12.26", + "reqwest 0.12.28", "serde", "serde_json", "thiserror 1.0.69", @@ -2156,7 +2256,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d901aeb453fd80e51d64df4ee005014f6cf39f2d736dd64f7239c132d9d39a6a" dependencies = [ - "reqwest 0.12.26", + "reqwest 0.12.28", "thiserror 1.0.69", "tokio", ] @@ -2179,14 +2279,14 @@ dependencies = [ "hex", "once_cell", "percent-encoding", - "pkcs8 0.10.2", + "pkcs8", "regex", - "reqwest 0.12.26", + "reqwest 0.12.28", "reqwest-middleware", "ring", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "thiserror 1.0.69", "time", "tokio", @@ -2205,9 +2305,9 @@ dependencies = [ [[package]] name = "group" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ "ff", "rand_core 0.6.4", @@ -2235,16 +2335,16 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.12" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http 1.4.0", + "http 1.5.0", "indexmap", "slab", "tokio", @@ -2266,7 +2366,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -2274,6 +2374,17 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "hashlink" @@ -2303,47 +2414,71 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] -name = "hickory-proto" -version = "0.25.2" +name = "hickory-net" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" +checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" dependencies = [ "async-trait", "cfg-if", "data-encoding", - "enum-as-inner", "futures-channel", "futures-io", "futures-util", + "hickory-proto", + "idna", + "ipnet", + "jni", + "rand 0.10.2", + "thiserror 2.0.20", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", "idna", "ipnet", + "jni", "once_cell", - "rand 0.9.2", + "prefix-trie", + "rand 0.10.2", "ring", - "thiserror 2.0.17", + "thiserror 2.0.20", "tinyvec", - "tokio", "tracing", "url", ] [[package]] name = "hickory-resolver" -version = "0.25.2" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" +checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" dependencies = [ "cfg-if", "futures-util", + "hickory-net", "hickory-proto", "ipconfig", + "ipnet", + "jni", "moka", + "ndk-context", "once_cell", "parking_lot", - "rand 0.9.2", + "rand 0.10.2", "resolv-conf", "smallvec", - "thiserror 2.0.17", + "system-configuration 0.7.0", + "thiserror 2.0.20", "tokio", "tracing", ] @@ -2354,7 +2489,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", ] [[package]] @@ -2363,16 +2498,25 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", ] [[package]] name = "home" -version = "0.5.9" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2388,9 +2532,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -2409,24 +2553,24 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", - "http 1.4.0", + "http 1.5.0", ] [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", - "http 1.4.0", - "http-body 1.0.1", + "http 1.5.0", + "http-body 1.1.0", "pin-project-lite", ] @@ -2464,9 +2608,18 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] [[package]] name = "hyper" @@ -2494,22 +2647,21 @@ dependencies = [ [[package]] name = "hyper" -version = "1.8.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", - "h2 0.4.12", - "http 1.4.0", - "http-body 1.0.1", + "h2 0.4.15", + "http 1.5.0", + "http-body 1.1.0", "httparse", "httpdate", "itoa", "pin-project-lite", - "pin-utils", "smallvec", "tokio", "want", @@ -2532,16 +2684,15 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.7" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http 1.4.0", - "hyper 1.8.1", + "http 1.5.0", + "hyper 1.11.0", "hyper-util", - "rustls 0.23.35", + "rustls 0.23.43", "rustls-native-certs", - "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", "tower-service", @@ -2568,7 +2719,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.8.1", + "hyper 1.11.0", "hyper-util", "native-tls", "tokio", @@ -2578,24 +2729,23 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ "base64 0.22.1", "bytes", "futures-channel", - "futures-core", "futures-util", - "http 1.4.0", - "http-body 1.0.1", - "hyper 1.8.1", + "http 1.5.0", + "http-body 1.1.0", + "hyper 1.11.0", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.1", - "system-configuration", + "socket2 0.6.5", + "system-configuration 0.7.0", "tokio", "tower-service", "tracing", @@ -2604,9 +2754,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.64" +version = "0.1.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -2628,12 +2778,13 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", @@ -2641,9 +2792,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -2654,9 +2805,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -2668,15 +2819,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ "icu_collections", "icu_locale_core", @@ -2688,15 +2839,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -2726,9 +2877,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", @@ -2736,12 +2887,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -2772,29 +2923,23 @@ dependencies = [ [[package]] name = "ipconfig" -version = "0.3.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" dependencies = [ - "socket2 0.5.10", + "socket2 0.6.5", "widestring", - "windows-sys 0.48.0", - "winreg", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", ] [[package]] name = "ipnet" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" - -[[package]] -name = "iri-string" -version = "0.7.9" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" dependencies = [ - "memchr", "serde", ] @@ -2826,27 +2971,77 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.15" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jni" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.20", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] [[package]] name = "jobserver" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.3", "libc", ] [[package]] name = "js-sys" -version = "0.3.83" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" dependencies = [ - "once_cell", + "cfg-if", + "futures-util", "wasm-bindgen", ] @@ -2871,30 +3066,31 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin", + "spin 0.9.9", ] [[package]] name = "libc" -version = "0.2.178" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libm" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.11" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df15f6eac291ed1cf25865b1ee60399f57e7c227e7f51bdbd4c5270396a9ed50" +checksum = "2026a5056764a10b2bf5d56488cba40da507f5493a6a429340e2004d9ed085fa" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "libc", - "redox_syscall 0.6.0", + "plain", + "redox_syscall 0.9.1", ] [[package]] @@ -2916,15 +3112,15 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "lock_api" @@ -2937,17 +3133,17 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "lru" -version = "0.12.5" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" dependencies = [ - "hashbrown 0.15.5", + "hashbrown 0.16.1", ] [[package]] @@ -2965,7 +3161,7 @@ dependencies = [ "macro_magic_core", "macro_magic_macros", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -2979,7 +3175,7 @@ dependencies = [ "macro_magic_core_macros", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -2990,7 +3186,7 @@ checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -3001,7 +3197,7 @@ checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" dependencies = [ "macro_magic_core", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -3026,14 +3222,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", ] [[package]] name = "memchr" -version = "2.7.6" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "mime" @@ -3063,9 +3269,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -3074,16 +3280,16 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.12" +version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3dec6bd31b08944e08b58fd99373893a6c17054d6f3ea5006cc894f4f4eee2a" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" dependencies = [ "async-lock", "crossbeam-channel", "crossbeam-epoch", "crossbeam-utils", "equivalent", - "event-listener 5.4.1", + "event-listener 5.4.2", "futures-util", "parking_lot", "portable-atomic", @@ -3094,9 +3300,9 @@ dependencies = [ [[package]] name = "mongocrypt" -version = "0.3.2" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da0cd419a51a5fb44819e290fbdb0665a54f21dead8923446a799c7f4d26ad9" +checksum = "8426a875ded61430d4a811dbfda7633b6b8af0225c547fc6c28b8b0aa7d79a13" dependencies = [ "bson", "mongocrypt-sys", @@ -3106,18 +3312,18 @@ dependencies = [ [[package]] name = "mongocrypt-sys" -version = "0.1.5+1.15.1" +version = "0.1.6+1.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224484c5d09285a7b8cb0a0c117e847ebd14cb6e4470ecf68cdb89c503b0edb9" +checksum = "851fac73f7fe22f6a3ab87f720ce509cae7c9fd08e7dd27866cc232dee07ccf4" [[package]] name = "mongodb" -version = "3.4.1" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f5c20217413bed97c613714e6d6dfe39ef59dd79a68999f1043b0566192975" +checksum = "b814038f367d212f55de0a630cb35102a9b8ca23785a86955d62c0087c93846d" dependencies = [ "base64 0.22.1", - "bitflags 2.10.0", + "bitflags 2.13.1", "bson", "derive-where", "derive_more", @@ -3125,54 +3331,54 @@ dependencies = [ "futures-io", "futures-util", "hex", + "hickory-net", "hickory-proto", "hickory-resolver", - "hmac", + "hmac 0.13.0", "macro_magic", - "md-5", + "md-5 0.11.0", "mongocrypt", "mongodb-internal-macros", "pbkdf2", "percent-encoding", - "rand 0.9.2", + "rand 0.9.5", "rustc_version_runtime", - "rustls 0.23.35", - "rustversion", + "rustls 0.23.43", "serde", "serde_bytes", "serde_with", - "sha1", - "sha2", - "socket2 0.6.1", + "sha1 0.11.0", + "sha2 0.11.0", + "socket2 0.6.5", "stringprep", "strsim", "take_mut", - "thiserror 2.0.17", + "thiserror 2.0.20", "tokio", "tokio-rustls 0.26.4", "tokio-util", "typed-builder", "uuid", - "webpki-roots 1.0.4", + "webpki-roots 1.0.9", ] [[package]] name = "mongodb-internal-macros" -version = "3.4.1" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20033442aa13664e70bc9f8be1bacabebf6a31b6d4bb5608ceb99c4ec96e9951" +checksum = "f736d2fbc56e0011a341fbb9172bd822fda75c5f93b82fae1c7aab1e2613c810" dependencies = [ "macro_magic", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "native-tls" -version = "0.2.14" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" dependencies = [ "libc", "log", @@ -3180,11 +3386,17 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "security-framework 2.11.1", + "security-framework", "security-framework-sys", "tempfile", ] +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "nibble_vec" version = "0.1.0" @@ -3200,7 +3412,7 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "cfg-if", "cfg_aliases 0.1.1", "libc", @@ -3217,9 +3429,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.6" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", @@ -3236,16 +3448,16 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.5", + "rand 0.8.7", "smallvec", "zeroize", ] [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -3258,11 +3470,10 @@ dependencies = [ [[package]] name = "num-iter" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" dependencies = [ - "autocfg", "num-integer", "num-traits", ] @@ -3304,13 +3515,13 @@ checksum = "c38841cdd844847e3e7c8d29cef9dcfed8877f8f56f9071f77843ecf3baf937f" dependencies = [ "base64 0.13.1", "chrono", - "getrandom 0.2.16", + "getrandom 0.2.17", "http 0.2.12", - "rand 0.8.5", + "rand 0.8.7", "serde", "serde_json", "serde_path_to_error", - "sha2", + "sha2 0.10.9", "thiserror 1.0.69", "url", ] @@ -3328,14 +3539,14 @@ dependencies = [ "futures", "httparse", "humantime", - "hyper 1.8.1", + "hyper 1.11.0", "itertools", - "md-5", + "md-5 0.10.6", "parking_lot", "percent-encoding", "quick-xml 0.37.5", - "rand 0.8.5", - "reqwest 0.12.26", + "rand 0.8.7", + "reqwest 0.12.28", "ring", "rustls-pemfile 2.2.0", "serde", @@ -3349,9 +3560,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" dependencies = [ "critical-section", "portable-atomic", @@ -3365,11 +3576,11 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "onig" -version = "6.5.1" +version = "6.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" +checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "libc", "once_cell", "onig_sys", @@ -3377,9 +3588,9 @@ dependencies = [ [[package]] name = "onig_sys" -version = "69.9.1" +version = "69.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc" +checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" dependencies = [ "cc", "pkg-config", @@ -3387,15 +3598,14 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.75" +version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "cfg-if", "foreign-types", "libc", - "once_cell", "openssl-macros", "openssl-sys", ] @@ -3408,20 +3618,20 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "openssl-probe" -version = "0.1.6" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.111" +version = "0.9.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" dependencies = [ "cc", "libc", @@ -3443,18 +3653,19 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "p256" -version = "0.11.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" dependencies = [ "ecdsa", "elliptic-curve", - "sha2", + "primeorder", + "sha2 0.10.9", ] [[package]] name = "pangolin_api" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "async-trait", @@ -3471,20 +3682,20 @@ dependencies = [ "chrono", "dotenvy", "gcp_auth", - "hmac", - "hyper 1.8.1", + "hmac 0.12.1", + "hyper 1.11.0", "jsonwebtoken", "moka", "pangolin_api", "pangolin_core", "pangolin_store", - "rand 0.8.5", + "rand 0.8.7", "reqwest 0.11.27", "serde", "serde_json", "serde_yaml", "serial_test", - "sha2", + "sha2 0.10.9", "thiserror 1.0.69", "tokio", "tower 0.4.13", @@ -3500,7 +3711,7 @@ dependencies = [ [[package]] name = "pangolin_cli_admin" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "async-trait", @@ -3522,7 +3733,7 @@ dependencies = [ [[package]] name = "pangolin_cli_common" -version = "0.6.0" +version = "0.7.0" dependencies = [ "async-trait", "chrono", @@ -3540,7 +3751,7 @@ dependencies = [ [[package]] name = "pangolin_cli_user" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "clap", @@ -3558,7 +3769,7 @@ dependencies = [ [[package]] name = "pangolin_core" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "chrono", @@ -3573,7 +3784,7 @@ dependencies = [ [[package]] name = "pangolin_store" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "async-trait", @@ -3586,7 +3797,7 @@ dependencies = [ "bson", "bytes", "chrono", - "dashmap 6.1.0", + "dashmap 6.2.1", "futures", "google-cloud-auth", "google-cloud-storage", @@ -3596,7 +3807,7 @@ dependencies = [ "pangolin_core", "pangolin_store", "regex", - "reqwest 0.12.26", + "reqwest 0.12.28", "serde", "serde_json", "sqlx", @@ -3643,11 +3854,11 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "pbkdf2" -version = "0.12.2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" dependencies = [ - "digest", + "digest 0.11.3", ] [[package]] @@ -3677,29 +3888,29 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pin-project" -version = "1.1.10" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.10" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "pin-project-lite" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pin-utils" @@ -3709,34 +3920,24 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "piper" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" dependencies = [ "atomic-waker", - "fastrand 2.3.0", + "fastrand 2.5.0", "futures-io", ] [[package]] -name = "pkcs1" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" -dependencies = [ - "der 0.7.10", - "pkcs8 0.10.2", - "spki 0.7.3", -] - -[[package]] -name = "pkcs8" -version = "0.9.0" +name = "pkcs1" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der 0.6.1", - "spki 0.6.0", + "der", + "pkcs8", + "spki", ] [[package]] @@ -3745,25 +3946,31 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der 0.7.10", - "spki 0.7.3", + "der", + "spki", ] [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "plain" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" [[package]] name = "plist" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "740ebea15c5d1428f910cd1a5f52cebf8d25006245ed8ade92702f4943d91e07" +checksum = "7da1d65da6dd5d1e44199ac0f58712d241c0f439f80adea8924d832384087f85" dependencies = [ "base64 0.22.1", "indexmap", - "quick-xml 0.38.4", + "quick-xml 0.41.0", "serde", "time", ] @@ -3784,15 +3991,15 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.11.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "potential_utf" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -3812,6 +4019,17 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prefix-trie" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" +dependencies = [ + "either", + "ipnet", + "num-traits", +] + [[package]] name = "prettytable-rs" version = "0.10.0" @@ -3826,6 +4044,15 @@ dependencies = [ "unicode-width 0.1.14", ] +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -3852,9 +4079,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.103" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -3881,28 +4108,28 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.38.4" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", ] [[package]] name = "quinn" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ "bytes", - "cfg_aliases 0.2.1", + "cfg_aliases 0.2.2", "pin-project-lite", "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.35", - "socket2 0.6.1", - "thiserror 2.0.17", + "rustls 0.23.43", + "socket2 0.6.5", + "thiserror 2.0.20", "tokio", "tracing", "web-time", @@ -3910,20 +4137,21 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.13" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "bytes", - "getrandom 0.3.4", + "getrandom 0.4.3", "lru-slab", - "rand 0.9.2", + "rand 0.10.2", + "rand_pcg", "ring", "rustc-hash", - "rustls 0.23.35", + "rustls 0.23.43", "rustls-pki-types", "slab", - "thiserror 2.0.17", + "thiserror 2.0.20", "tinyvec", "tracing", "web-time", @@ -3931,23 +4159,23 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.14" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" dependencies = [ - "cfg_aliases 0.2.1", + "cfg_aliases 0.2.2", "libc", "once_cell", - "socket2 0.6.1", + "socket2 0.6.5", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "quote" -version = "1.0.42" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -3958,6 +4186,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "radium" version = "0.7.0" @@ -3989,9 +4223,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -4000,12 +4234,23 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.3", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -4035,7 +4280,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.3", + "rand_core 0.9.5", ] [[package]] @@ -4053,18 +4298,24 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", ] [[package]] name = "rand_core" -version = "0.9.3" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_hc" version = "0.2.0" @@ -4074,22 +4325,31 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", ] [[package]] name = "redox_syscall" -version = "0.6.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec96166dafa0886eb81fe1c0a388bece180fbef2135f97c1e2cf8302e74b43b5" +checksum = "07507be7b4a5f9f26eeb41eeaebb1f5a7ff29dfb29739facc21d35bf8b11c21e" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", ] [[package]] @@ -4098,16 +4358,16 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "libredox", "thiserror 1.0.69", ] [[package]] name = "regex" -version = "1.12.2" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -4117,9 +4377,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -4128,15 +4388,15 @@ dependencies = [ [[package]] name = "regex-lite" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "reqwest" @@ -4167,7 +4427,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 0.1.2", - "system-configuration", + "system-configuration 0.5.1", "tokio", "tokio-native-tls", "tower-service", @@ -4180,21 +4440,21 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.26" +version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b4c14b2d9afca6a60277086b0cc6a6ae0b568f6f7916c943a8cdc79f8be240f" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64 0.22.1", "bytes", "encoding_rs", "futures-core", "futures-util", - "h2 0.4.12", - "http 1.4.0", - "http-body 1.0.1", + "h2 0.4.15", + "http 1.5.0", + "http-body 1.1.0", "http-body-util", - "hyper 1.8.1", - "hyper-rustls 0.27.7", + "hyper 1.11.0", + "hyper-rustls 0.27.9", "hyper-tls 0.6.0", "hyper-util", "js-sys", @@ -4205,7 +4465,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.35", + "rustls 0.23.43", "rustls-native-certs", "rustls-pki-types", "serde", @@ -4216,8 +4476,8 @@ dependencies = [ "tokio-native-tls", "tokio-rustls 0.26.4", "tokio-util", - "tower 0.5.2", - "tower-http 0.6.8", + "tower 0.5.3", + "tower-http 0.6.11", "tower-service", "url", "wasm-bindgen", @@ -4234,8 +4494,8 @@ checksum = "562ceb5a604d3f7c885a792d42c199fd8af239d0a51b2fa6a78aafa092452b04" dependencies = [ "anyhow", "async-trait", - "http 1.4.0", - "reqwest 0.12.26", + "http 1.5.0", + "reqwest 0.12.28", "serde", "thiserror 1.0.69", "tower-service", @@ -4249,13 +4509,12 @@ checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "rfc6979" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "crypto-bigint 0.4.9", - "hmac", - "zeroize", + "hmac 0.12.1", + "subtle", ] [[package]] @@ -4266,7 +4525,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.16", + "getrandom 0.2.17", "libc", "untrusted", "windows-sys 0.52.0", @@ -4274,29 +4533,29 @@ dependencies = [ [[package]] name = "rsa" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40a0376c50d0358279d9d643e4bf7b7be212f1f4ff1da9070a7b54d22ef75c88" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid", - "digest", + "const-oid 0.9.6", + "digest 0.10.7", "num-bigint-dig", "num-integer", "num-traits", "pkcs1", - "pkcs8 0.10.2", + "pkcs8", "rand_core 0.6.4", - "signature 2.2.0", - "spki 0.7.3", + "signature", + "spki", "subtle", "zeroize", ] [[package]] name = "rust-embed" -version = "8.9.0" +version = "8.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "947d7f3fad52b283d261c4c99a084937e2fe492248cb9a68a8435a861b8798ca" +checksum = "e9e7760e252aaba7b09f4be00e36476cf585bdb68a53552ac954cdf504ab4bc9" dependencies = [ "rust-embed-impl", "rust-embed-utils", @@ -4305,32 +4564,33 @@ dependencies = [ [[package]] name = "rust-embed-impl" -version = "8.9.0" +version = "8.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fa2c8c9e8711e10f9c4fd2d64317ef13feaab820a4c51541f1a8c8e2e851ab2" +checksum = "3bcfc4d6f53af43755f7a723e4b6b8794fcce052a178dd8c6c1dadc5f5343097" dependencies = [ + "mime_guess", "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.111", + "syn 2.0.119", "walkdir", ] [[package]] name = "rust-embed-utils" -version = "8.9.0" +version = "8.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b161f275cb337fe0a44d924a5f4df0ed69c2c39519858f931ce61c779d3475" +checksum = "42ffa149f6aa81b58a5b3011d01a857c4ed12c7a732d2c51947a4c7c692185f0" dependencies = [ - "sha2", + "sha2 0.11.0", "walkdir", ] [[package]] name = "rustc-hash" -version = "2.1.1" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustc_version" @@ -4353,11 +4613,11 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.2" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys", @@ -4378,30 +4638,30 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.35" +version = "0.23.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" dependencies = [ "aws-lc-rs", "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.8", + "rustls-webpki 0.103.13", "subtle", "zeroize", ] [[package]] name = "rustls-native-certs" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.5.1", + "security-framework", ] [[package]] @@ -4424,9 +4684,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.13.1" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "708c0f9d5f54ba0272468c1d306a52c495b31fa155e91bc25371e6df7996908c" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "web-time", "zeroize", @@ -4444,9 +4704,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.8" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", "ring", @@ -4456,9 +4716,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "rustyline" @@ -4466,7 +4726,7 @@ version = "14.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7803e8936da37efd9b6d4478277f4b2b9bb5cdb37a113e8d63222e58da647e63" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "cfg-if", "clipboard-win", "fd-lock", @@ -4484,9 +4744,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.20" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "same-file" @@ -4499,9 +4759,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] @@ -4524,38 +4784,25 @@ dependencies = [ [[package]] name = "sec1" -version = "0.3.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct", - "der 0.6.1", + "der", "generic-array", - "pkcs8 0.9.0", + "pkcs8", "subtle", "zeroize", ] [[package]] name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags 2.10.0", - "core-foundation 0.9.4", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework" -version = "3.5.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -4564,9 +4811,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.15.0" +version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ "core-foundation-sys", "libc", @@ -4574,15 +4821,15 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -4600,36 +4847,36 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 3.0.3", ] [[package]] name = "serde_json" -version = "1.0.145" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "indexmap", "itoa", "memchr", - "ryu", "serde", "serde_core", + "zmij", ] [[package]] @@ -4668,9 +4915,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.16.1" +version = "3.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7" +checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a" dependencies = [ "serde_core", "serde_with_macros", @@ -4678,14 +4925,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.16.1" +version = "3.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c" +checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -4723,18 +4970,29 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "sha1" -version = "0.10.6" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -4744,8 +5002,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -4765,68 +5034,75 @@ checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "signal-hook-registry" -version = "1.4.7" +version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" dependencies = [ + "errno", "libc", ] [[package]] name = "signature" -version = "1.6.4" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", + "digest 0.10.7", "rand_core 0.6.4", ] [[package]] -name = "signature" -version = "2.2.0" +name = "simd-adler32" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" dependencies = [ - "digest", - "rand_core 0.6.4", + "rustc_version", + "simdutf8", ] [[package]] -name = "simd-adler32" -version = "0.3.8" +name = "simdutf8" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "simple_asn1" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" +checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.17", + "thiserror 2.0.20", "time", ] [[package]] name = "slab" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" dependencies = [ "serde", ] @@ -4849,7 +5125,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -4864,32 +5140,28 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.1" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" dependencies = [ "lock_api", ] [[package]] -name = "spki" -version = "0.6.0" +name = "spin" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" -dependencies = [ - "base64ct", - "der 0.6.1", -] +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" [[package]] name = "spki" @@ -4898,7 +5170,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der 0.7.10", + "der", ] [[package]] @@ -4926,7 +5198,7 @@ dependencies = [ "crc", "crossbeam-queue", "either", - "event-listener 5.4.1", + "event-listener 5.4.2", "futures-core", "futures-intrusive", "futures-io", @@ -4938,12 +5210,12 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rustls 0.23.35", + "rustls 0.23.43", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", - "thiserror 2.0.17", + "thiserror 2.0.20", "tokio", "tokio-stream", "tracing", @@ -4962,7 +5234,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -4980,12 +5252,12 @@ dependencies = [ "quote", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "sqlx-core", "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.111", + "syn 2.0.119", "tokio", "url", ] @@ -4998,12 +5270,12 @@ checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ "atoi", "base64 0.22.1", - "bitflags 2.10.0", + "bitflags 2.13.1", "byteorder", "bytes", "chrono", "crc", - "digest", + "digest 0.10.7", "dotenvy", "either", "futures-channel", @@ -5013,22 +5285,22 @@ dependencies = [ "generic-array", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "once_cell", "percent-encoding", - "rand 0.8.5", + "rand 0.8.7", "rsa", "serde", - "sha1", - "sha2", + "sha1 0.10.7", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.17", + "thiserror 2.0.20", "tracing", "uuid", "whoami", @@ -5042,7 +5314,7 @@ checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", "base64 0.22.1", - "bitflags 2.10.0", + "bitflags 2.13.1", "byteorder", "chrono", "crc", @@ -5053,21 +5325,21 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "home", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "once_cell", - "rand 0.8.5", + "rand 0.8.7", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.17", + "thiserror 2.0.20", "tracing", "uuid", "whoami", @@ -5093,7 +5365,7 @@ dependencies = [ "serde", "serde_urlencoded", "sqlx-core", - "thiserror 2.0.17", + "thiserror 2.0.20", "tracing", "url", "uuid", @@ -5140,9 +5412,20 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.111" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -5172,7 +5455,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -5191,7 +5474,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.20", "walkdir", "yaml-rust", ] @@ -5204,7 +5487,18 @@ checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", "core-foundation 0.9.4", - "system-configuration-sys", + "system-configuration-sys 0.5.0", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.13.1", + "core-foundation 0.9.4", + "system-configuration-sys 0.6.0", ] [[package]] @@ -5217,6 +5511,16 @@ dependencies = [ "libc", ] +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -5237,12 +5541,12 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.23.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ - "fastrand 2.3.0", - "getrandom 0.3.4", + "fastrand 2.5.0", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -5279,11 +5583,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.17", + "thiserror-impl 2.0.20", ] [[package]] @@ -5294,58 +5598,57 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 3.0.3", ] [[package]] name = "thread_local" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" dependencies = [ "cfg-if", ] [[package]] name = "time" -version = "0.3.44" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", - "itoa", "js-sys", "libc", "num-conv", "num_threads", "powerfmt", - "serde", + "serde_core", "time-core", "time-macros", ] [[package]] name = "time-core" -version = "0.1.6" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.24" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" dependencies = [ "num-conv", "time-core", @@ -5362,9 +5665,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -5372,9 +5675,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.10.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" dependencies = [ "tinyvec_macros", ] @@ -5387,9 +5690,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.48.0" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -5397,20 +5700,20 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.1", + "socket2 0.6.5", "tokio-macros", "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.6.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 3.0.3", ] [[package]] @@ -5439,15 +5742,15 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls 0.23.35", + "rustls 0.23.43", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.17" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -5456,12 +5759,10 @@ dependencies = [ [[package]] name = "tokio-test" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2468baabc3311435b55dd935f702f42cd1b8abb7e754fb7dfb16bd36aa88f9f7" +checksum = "3f6d24790a10a7af737693a3e8f1d03faef7e6ca0cc99aae5066f533766de545" dependencies = [ - "async-stream", - "bytes", "futures-core", "tokio", "tokio-stream", @@ -5469,14 +5770,16 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.17" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-io", "futures-sink", + "futures-util", + "libc", "pin-project-lite", "tokio", ] @@ -5500,9 +5803,9 @@ dependencies = [ [[package]] name = "tower" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", @@ -5520,10 +5823,11 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "bytes", - "http 1.4.0", - "http-body 1.0.1", + "futures-util", + "http 1.5.0", + "http-body 1.1.0", "http-body-util", "pin-project-lite", "tokio", @@ -5534,20 +5838,20 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "bytes", "futures-util", - "http 1.4.0", - "http-body 1.0.1", - "iri-string", + "http 1.5.0", + "http-body 1.1.0", "pin-project-lite", - "tower 0.5.2", + "tower 0.5.3", "tower-layer", "tower-service", + "url", ] [[package]] @@ -5564,9 +5868,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "log", "pin-project-lite", @@ -5582,14 +5886,14 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "tracing-core" -version = "0.1.35" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", "valuable", @@ -5628,9 +5932,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ "matchers", "nu-ansi-term", @@ -5670,14 +5974,14 @@ checksum = "0e48cea23f68d1f78eb7bc092881b6bb88d3d6b5b7e6234f6f9c911da1ffb221" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "tz-rs" @@ -5690,9 +5994,9 @@ dependencies = [ [[package]] name = "unicase" -version = "2.8.1" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" [[package]] name = "unicode-bidi" @@ -5702,9 +6006,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.22" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-normalization" @@ -5723,9 +6027,9 @@ checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" [[package]] name = "unicode-segmentation" -version = "1.12.0" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-width" @@ -5759,14 +6063,15 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.7" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -5809,7 +6114,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.111", + "syn 2.0.119", "uuid", ] @@ -5831,11 +6136,11 @@ dependencies = [ [[package]] name = "uuid" -version = "1.19.0" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.3", "js-sys", "serde_core", "wasm-bindgen", @@ -5904,9 +6209,9 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.1+wasi-0.2.4" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ "wit-bindgen", ] @@ -5919,9 +6224,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.106" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" dependencies = [ "cfg-if", "once_cell", @@ -5932,22 +6237,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.56" +version = "0.4.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" dependencies = [ - "cfg-if", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.106" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5955,22 +6257,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.106" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.106" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" dependencies = [ "unicode-ident", ] @@ -5990,9 +6292,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.83" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" dependencies = [ "js-sys", "wasm-bindgen", @@ -6014,14 +6316,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.4", + "webpki-roots 1.0.9", ] [[package]] name = "webpki-roots" -version = "1.0.4" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" dependencies = [ "rustls-pki-types", ] @@ -6094,7 +6396,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -6105,7 +6407,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -6170,15 +6472,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -6212,30 +6505,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -6248,12 +6524,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -6266,12 +6536,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -6284,24 +6548,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -6314,12 +6566,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -6332,12 +6578,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -6350,12 +6590,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -6368,12 +6602,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winreg" version = "0.50.0" @@ -6394,9 +6622,9 @@ dependencies = [ "base64 0.22.1", "deadpool", "futures", - "http 1.4.0", + "http 1.5.0", "http-body-util", - "hyper 1.8.1", + "hyper 1.11.0", "hyper-util", "log", "once_cell", @@ -6409,15 +6637,15 @@ dependencies = [ [[package]] name = "wit-bindgen" -version = "0.46.0" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "wyz" @@ -6445,9 +6673,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -6456,68 +6684,68 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.31" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.31" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] name = "zerofrom" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", "synstructure", ] [[package]] name = "zeroize" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -6526,9 +6754,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -6537,13 +6765,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.119", ] [[package]] @@ -6557,3 +6785,9 @@ dependencies = [ "crossbeam-utils", "flate2", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/pangolin/Cargo.toml b/pangolin/Cargo.toml index 1ca930d..efa54be 100644 --- a/pangolin/Cargo.toml +++ b/pangolin/Cargo.toml @@ -16,9 +16,9 @@ members = [ # UI 0.5.0 and the Helm chart 0.1.0 - with no way to tell which combination was # ever tested together (C-23). Every crate now inherits from here. [workspace.package] -version = "0.6.0" +version = "0.7.0" edition = "2021" -rust-version = "1.92" +rust-version = "1.94" license = "MIT" repository = "https://github.com/AlexMercedCoder/pangolin" authors = ["Alex Merced"] @@ -63,7 +63,7 @@ async-trait = "0.1" dashmap = "6.0" futures = "0.3" tower = { version = "0.4", features = ["limit", "util"] } -tower-http = { version = "0.5", features = ["cors", "trace", "timeout", "limit"] } +tower-http = { version = "0.5", features = ["cors", "trace", "timeout", "limit", "catch-panic"] } reqwest = { version = "0.11", features = ["json"] } url = "2.5" bytes = "1.5" diff --git a/pangolin/Dockerfile b/pangolin/Dockerfile index 476a30e..83780d8 100644 --- a/pangolin/Dockerfile +++ b/pangolin/Dockerfile @@ -4,8 +4,9 @@ # Build stage # --------------------------------------------------------------------------- # The base image is pinned to the Rust version the README requires. It used to -# be 1.88 while the README asked for 1.92+ (A-36). -FROM rust:1.92-slim-bookworm AS builder +# be 1.88 while the README asked for 1.92+ (A-36). Kept in step with +# `rust-version` in Cargo.toml, which the `msrv` CI job verifies. +FROM rust:1.94-slim-bookworm AS builder WORKDIR /usr/src/pangolin diff --git a/pangolin/clippy-warning-budget.txt b/pangolin/clippy-warning-budget.txt index 7facc89..e85087a 100644 --- a/pangolin/clippy-warning-budget.txt +++ b/pangolin/clippy-warning-budget.txt @@ -1 +1 @@ -36 +31 diff --git a/pangolin/logs/api_log.txt b/pangolin/logs/api_log.txt deleted file mode 100644 index 5e1615a..0000000 --- a/pangolin/logs/api_log.txt +++ /dev/null @@ -1,417 +0,0 @@ -warning: unused import: `std::collections::HashMap` - --> pangolin_store/src/postgres.rs:10:5 - | -10 | use std::collections::HashMap; - | ^^^^^^^^^^^^^^^^^^^^^^^^^ - | - = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default - -warning: unused variable: `target` - --> pangolin_store/src/postgres.rs:637:13 - | -637 | let target = self.get_branch(tenant_id, catalog_name, target_branch.clone()).await? - | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_target` - | - = note: `#[warn(unused_variables)]` (part of `#[warn(unused)]`) on by default - -warning: unused variable: `location` - --> pangolin_store/src/postgres.rs:697:31 - | -697 | async fn read_file(&self, location: &str) -> Result> { - | ^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_location` - -warning: unused variable: `bytes` - --> pangolin_store/src/postgres.rs:703:48 - | -703 | async fn write_file(&self, location: &str, bytes: Vec) -> Result<()> { - | ^^^^^ help: if this is intentional, prefix it with an underscore: `_bytes` - -warning: unused variable: `location` - --> pangolin_store/src/postgres.rs:703:32 - | -703 | async fn write_file(&self, location: &str, bytes: Vec) -> Result<()> { - | ^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_location` - -warning: unused variable: `branch` - --> pangolin_store/src/postgres.rs:820:83 - | -820 | ...d: Uuid, catalog_name: &str, branch: Option, namespace: Vec, table: String, expected_location: Option, new_loc... - | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_branch` - -warning: unused variable: `tenant_id` - --> pangolin_store/src/mongo.rs:121:38 - | -121 | async fn create_warehouse(&self, tenant_id: Uuid, warehouse: Warehouse) -> Result<()> { - | ^^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_tenant_id` - -warning: variable does not need to be mutable - --> pangolin_store/src/mongo.rs:284:13 - | -284 | let mut filter = doc! { - | ----^^^^^^ - | | - | help: remove this `mut` - | - = note: `#[warn(unused_mut)]` (part of `#[warn(unused)]`) on by default - -warning: unused variable: `p` - --> pangolin_store/src/mongo.rs:289:21 - | -289 | if let Some(p) = parent { - | ^ help: if this is intentional, prefix it with an underscore: `_p` - -warning: unused variable: `branch` - --> pangolin_store/src/mongo.rs:364:68 - | -364 | ...d: Uuid, catalog_name: &str, branch: Option, namespace: Vec, name: String) -> Result> { - | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_branch` - -warning: unused variable: `branch` - --> pangolin_store/src/mongo.rs:401:70 - | -401 | async fn list_assets(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec) -> Result> { - | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_branch` - -warning: unused variable: `branch` - --> pangolin_store/src/mongo.rs:432:71 - | -432 | ...d: Uuid, catalog_name: &str, branch: Option, namespace: Vec, name: String) -> Result<()> { - | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_branch` - -warning: unused variable: `branch` - --> pangolin_store/src/mongo.rs:443:71 - | -443 | ...d: Uuid, catalog_name: &str, branch: Option, source_namespace: Vec, source_name: String, dest_namespace: Vec, ... - | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_branch` - -warning: unused variable: `branch` - --> pangolin_store/src/mongo.rs:714:83 - | -714 | ...d: Uuid, catalog_name: &str, branch: Option, namespace: Vec, table: String, expected_location: Option, new_loc... - | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_branch` - -warning: field `signer` is never read - --> pangolin_store/src/memory.rs:32:5 - | -17 | pub struct MemoryStore { - | ----------- field in this struct -... -32 | signer: crate::signer::SignerImpl, - | ^^^^^^ - | - = note: `MemoryStore` has a derived impl for the trait `Clone`, but this is intentionally ignored during dead code analysis - = note: `#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default - -warning: field `key` is never read - --> pangolin_store/src/signer.rs:25:5 - | -24 | pub struct SignerImpl { - | ---------- field in this struct -25 | key: String, - | ^^^ - | - = note: `SignerImpl` has a derived impl for the trait `Clone`, but this is intentionally ignored during dead code analysis - -warning: field `client` is never read - --> pangolin_store/src/mongo.rs:18:5 - | -17 | pub struct MongoStore { - | ---------- field in this struct -18 | client: Client, - | ^^^^^^ - | - = note: `MongoStore` has a derived impl for the trait `Clone`, but this is intentionally ignored during dead code analysis - -warning: multiple methods are never used - --> pangolin_store/src/mongo.rs:38:8 - | -22 | impl MongoStore { - | --------------- methods in this implementation -... -38 | fn catalogs(&self) -> Collection { - | ^^^^^^^^ -... -42 | fn namespaces(&self) -> Collection { - | ^^^^^^^^^^ -... -46 | fn assets(&self) -> Collection { - | ^^^^^^ -... -50 | fn branches(&self) -> Collection { - | ^^^^^^^^ -... -54 | fn tags(&self) -> Collection { - | ^^^^ -... -58 | fn commits(&self) -> Collection { - | ^^^^^^^ -... -62 | fn audit_logs(&self) -> Collection { - | ^^^^^^^^^^ - -warning: `pangolin_store` (lib) generated 18 warnings (run `cargo fix --lib -p pangolin_store` to apply 14 suggestions) -warning: unused import: `pangolin_store::memory::MemoryStore` - --> pangolin_api/src/lib.rs:4:5 - | -4 | use pangolin_store::memory::MemoryStore; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - | - = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default - -warning: unused import: `Any` - --> pangolin_api/src/lib.rs:6:35 - | -6 | use tower_http::cors::{CorsLayer, Any}; - | ^^^ - -warning: unused imports: `Request` and `body::Body` - --> pangolin_api/src/iceberg_handlers.rs:5:43 - | -5 | http::{StatusCode, HeaderMap, Method, Request}, - | ^^^^^^^ -6 | body::Body, - | ^^^^^^^^^^ - -warning: unused import: `pangolin_store::memory::MemoryStore` - --> pangolin_api/src/iceberg_handlers.rs:12:5 - | -12 | use pangolin_store::memory::MemoryStore; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unused import: `UserRole` - --> pangolin_api/src/iceberg_handlers.rs:1021:27 - | -1021 | use pangolin_core::user::{UserRole, UserSession}; - | ^^^^^^^^ - -warning: unused import: `std::sync::Arc` - --> pangolin_api/src/tenant_handlers.rs:8:5 - | -8 | use std::sync::Arc; - | ^^^^^^^^^^^^^^ - -warning: unused import: `std::collections::HashMap` - --> pangolin_api/src/tenant_handlers.rs:9:5 - | -9 | use std::collections::HashMap; - | ^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unused import: `std::sync::Arc` - --> pangolin_api/src/warehouse_handlers.rs:8:5 - | -8 | use std::sync::Arc; - | ^^^^^^^^^^^^^^ - -warning: unused import: `std::sync::Arc` - --> pangolin_api/src/asset_handlers.rs:8:5 - | -8 | use std::sync::Arc; - | ^^^^^^^^^^^^^^ - -warning: unused import: `HeaderMap` - --> pangolin_api/src/auth.rs:3:24 - | -3 | http::{StatusCode, HeaderMap}, - | ^^^^^^^^^ - -warning: unused import: `std::sync::Arc` - --> pangolin_api/src/signing_handlers.rs:8:5 - | -8 | use std::sync::Arc; - | ^^^^^^^^^^^^^^ - -warning: unused import: `Json` - --> pangolin_api/src/oauth_handlers.rs:5:5 - | -5 | Json, - | ^^^^ - -warning: unused import: `Serialize` - --> pangolin_api/src/oauth_handlers.rs:7:26 - | -7 | use serde::{Deserialize, Serialize}; - | ^^^^^^^^^ - -warning: unused import: `std::sync::Arc` - --> pangolin_api/src/business_metadata_handlers.rs:8:5 - | -8 | use std::sync::Arc; - | ^^^^^^^^^^^^^^ - -warning: unused imports: `MergeConflict` and `MergeOperation` - --> pangolin_api/src/merge_handlers.rs:7:48 - | -7 | use pangolin_core::model::{ConflictResolution, MergeConflict, MergeOperation, ResolutionStrategy}; - | ^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ - -warning: unused import: `Serialize` - --> pangolin_api/src/permission_handlers.rs:8:26 - | -8 | use serde::{Deserialize, Serialize}; - | ^^^^^^^^^ - -warning: unused import: `Serialize` - --> pangolin_api/src/service_user_handlers.rs:10:26 - | -10 | use serde::{Deserialize, Serialize}; - | ^^^^^^^^^ - -warning: variable does not need to be mutable - --> pangolin_api/src/auth.rs:92:21 - | -92 | let mut validation = Validation::new(Algorithm::HS256); - | ----^^^^^^^^^^ - | | - | help: remove this `mut` - | - = note: `#[warn(unused_mut)]` (part of `#[warn(unused)]`) on by default - -warning: variable does not need to be mutable - --> pangolin_api/src/oauth_handlers.rs:110:14 - | -110 | Some(mut u) => { - | ----^ - | | - | help: remove this `mut` - -warning: variable does not need to be mutable - --> pangolin_api/src/permission_handlers.rs:100:10 - | -100 | Json(mut role): Json, - | ----^^^^ - | | - | help: remove this `mut` - -warning: unused import: `pangolin_store::CatalogStore` - --> pangolin_api/src/business_metadata_handlers.rs:9:5 - | -9 | use pangolin_store::CatalogStore; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unused import: `pangolin_store::CatalogStore` - --> pangolin_api/src/tenant_handlers.rs:10:5 - | -10 | use pangolin_store::CatalogStore; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unused import: `pangolin_store::CatalogStore` - --> pangolin_api/src/warehouse_handlers.rs:9:5 - | -9 | use pangolin_store::CatalogStore; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unused import: `pangolin_store::CatalogStore` - --> pangolin_api/src/asset_handlers.rs:9:5 - | -9 | use pangolin_store::CatalogStore; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unused import: `pangolin_store::CatalogStore` - --> pangolin_api/src/signing_handlers.rs:9:5 - | -9 | use pangolin_store::CatalogStore; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -warning: unused variable: `prefix` - --> pangolin_api/src/iceberg_handlers.rs:286:5 - | -286 | prefix: Option>, - | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_prefix` - | - = note: `#[warn(unused_variables)]` (part of `#[warn(unused)]`) on by default - -warning: unused variable: `timestamp` - --> pangolin_api/src/iceberg_handlers.rs:656:24 - | -656 | if let Ok(timestamp) = timestamp_str.parse::() { - | ^^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_timestamp` - -warning: unused variable: `session` - --> pangolin_api/src/iceberg_handlers.rs:854:15 - | -854 | Extension(session): Extension, - | ^^^^^^^ help: if this is intentional, prefix it with an underscore: `_session` - -warning: unused variable: `session` - --> pangolin_api/src/iceberg_handlers.rs:982:15 - | -982 | Extension(session): Extension, - | ^^^^^^^ help: if this is intentional, prefix it with an underscore: `_session` - -warning: unused variable: `params` - --> pangolin_api/src/pangolin_handlers.rs:66:11 - | -66 | Query(params): Query, - | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_params` - -warning: unused variable: `tenant_uuid` - --> pangolin_api/src/token_handlers.rs:34:9 - | -34 | let tenant_uuid = match Uuid::parse_str(&payload.tenant_id) { - | ^^^^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_tenant_uuid` - -warning: fields `identifier` and `requirements` are never read - --> pangolin_api/src/iceberg_handlers.rs:175:5 - | -174 | pub struct CommitTableRequest { - | ------------------ fields in this struct -175 | identifier: Option, - | ^^^^^^^^^^ -176 | requirements: Vec, - | ^^^^^^^^^^^^ - | - = note: `#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default - -warning: field `removals` is never read - --> pangolin_api/src/iceberg_handlers.rs:968:5 - | -967 | pub struct UpdateNamespacePropertiesRequest { - | -------------------------------- field in this struct -968 | removals: Option>, - | ^^^^^^^^ - -warning: fields `name` and `catalog` are never read - --> pangolin_api/src/pangolin_handlers.rs:30:5 - | -29 | pub struct ListBranchParams { - | ---------------- fields in this struct -30 | name: Option, - | ^^^^ -31 | catalog: Option, - | ^^^^^^^ - -warning: `pangolin_api` (lib) generated 34 warnings (run `cargo fix --lib -p pangolin_api` to apply 26 suggestions) -warning: unused import: `std::env` - --> pangolin_api/src/main.rs:3:5 - | -3 | use std::env; - | ^^^^^^^^ - | - = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default - -warning: unused imports: `Any` and `CorsLayer` - --> pangolin_api/src/main.rs:4:24 - | -4 | use tower_http::cors::{CorsLayer, Any}; - | ^^^^^^^^^ ^^^ - -warning: unused imports: `HeaderValue` and `Method` - --> pangolin_api/src/main.rs:5:18 - | -5 | use axum::http::{HeaderValue, Method}; - | ^^^^^^^^^^^ ^^^^^^ - -warning: unused variable: `storage_type` - --> pangolin_api/src/main.rs:17:9 - | -17 | let storage_type = std::env::var("PANGOLIN_STORAGE_TYPE").unwrap_or_else(|_| "memory".to_string()); - | ^^^^^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_storage_type` - | - = note: `#[warn(unused_variables)]` (part of `#[warn(unused)]`) on by default - -warning: `pangolin_api` (bin "pangolin_api") generated 4 warnings (run `cargo fix --bin "pangolin_api" -p pangolin_api` to apply 4 suggestions) - Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.25s - Running `target/debug/pangolin_api` -2025-12-14T22:11:31.649619Z  INFO pangolin_api: Using Memory Storage -2025-12-14T22:11:31.650363Z  INFO pangolin_api: Created default tenant for testing: 00000000-0000-0000-0000-000000000000 -2025-12-14T22:11:31.653384Z  INFO pangolin_api: listening on 0.0.0.0:8080 diff --git a/pangolin/pangolin_api/Cargo.toml b/pangolin/pangolin_api/Cargo.toml index 1d2ae27..d0db34a 100644 --- a/pangolin/pangolin_api/Cargo.toml +++ b/pangolin/pangolin_api/Cargo.toml @@ -15,8 +15,8 @@ categories = ["database"] workspace = true [dependencies] -pangolin_core = { path = "../pangolin_core", version = "0.6.0" } -pangolin_store = { path = "../pangolin_store", version = "0.6.0" } +pangolin_core = { path = "../pangolin_core", version = "0.7.0" } +pangolin_store = { path = "../pangolin_store", version = "0.7.0" } axum = { workspace = true } tokio = { workspace = true } serde = { workspace = true } @@ -46,7 +46,7 @@ sha2 = "0.10" # Cloud provider SDKs (optional features) aws-config = { version = "1.0", optional = true } -aws-sdk-sts = { version = "=1.50.0", optional = true } +aws-sdk-sts = { version = "1.109", optional = true } aws-credential-types = { version = "1.0", optional = true } aws-smithy-types = { version = "1.0", optional = true } azure_identity = { version = "0.20", optional = true } @@ -68,6 +68,6 @@ cloud-credentials = ["aws-sts", "azure-oauth", "gcp-oauth"] [dev-dependencies] hyper = { version = "1.0", features = ["full"] } pangolin_api = { path = ".", features = ["test-fixtures"] } -pangolin_store = { path = "../pangolin_store", version = "0.6.0" } +pangolin_store = { path = "../pangolin_store", version = "0.7.0" } serial_test = "2.0" wiremock = "0.6" diff --git a/pangolin/pangolin_api/src/asset_handlers.rs b/pangolin/pangolin_api/src/asset_handlers.rs index 9e378f5..874dd80 100644 --- a/pangolin/pangolin_api/src/asset_handlers.rs +++ b/pangolin/pangolin_api/src/asset_handlers.rs @@ -1,7 +1,7 @@ use crate::auth::TenantId; use crate::authz::check_permission; -use crate::iceberg::parse_table_identifier; use crate::iceberg::AppState; +use crate::iceberg::{parse_namespace, parse_table_identifier}; use axum::{ extract::{Extension, Path, Query, State}, http::StatusCode, @@ -17,6 +17,7 @@ use utoipa::ToSchema; use uuid::Uuid; #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CreateViewRequest { pub name: String, pub sql: String, @@ -66,6 +67,7 @@ impl From for ViewResponse { pub async fn create_view( State(store): State, Extension(tenant): Extension, + Extension(session): Extension, Path((prefix, namespace)): Path<(String, String)>, Json(payload): Json, ) -> impl IntoResponse { @@ -73,10 +75,35 @@ pub async fn create_view( let catalog_name = prefix; let (view_name, branch_from_name) = parse_table_identifier(&payload.name); - let branch = branch_from_name.unwrap_or("main".to_string()); + let (namespace_parts, branch_from_ns) = parse_namespace(&namespace); + let branch = branch_from_name + .or(branch_from_ns) + .unwrap_or_else(|| "main".to_string()); - // Parse namespace - let namespace_parts: Vec = namespace.split('\x1F').map(|s| s.to_string()).collect(); + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), + Err(e) => { + tracing::error!(error = %e, "create_view: failed to load catalog"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(); + } + }; + + // B0e: this handler took no session and performed no authorization at all, + // so any tenant member could create a view in any namespace. It now mirrors + // `create_table`'s namespace-scoped Create check. + let scope = PermissionScope::Namespace { + catalog_id: catalog.id, + namespace: namespace_parts.join("."), + }; + match check_permission(&store, &session, &Action::Create, &scope).await { + Ok(true) => (), + Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Err(e) => { + tracing::error!(error = %e, "create_view: permission check failed"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Permission check failed").into_response(); + } + } let mut properties = payload.properties.unwrap_or_default(); properties.insert("sql".to_string(), payload.sql); @@ -104,7 +131,10 @@ pub async fn create_view( .await { Ok(_) => (StatusCode::CREATED, Json(ViewResponse::from(asset))).into_response(), - Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(), + Err(e) => { + tracing::error!(error = %e, "create_view: failed to create asset"); + (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + } } } @@ -128,39 +158,69 @@ pub async fn create_view( pub async fn get_view( State(store): State, Extension(tenant): Extension, + Extension(session): Extension, Path((prefix, namespace, view)): Path<(String, String, String)>, ) -> impl IntoResponse { let tenant_id = tenant.0; let catalog_name = prefix; let (view_name, branch_from_name) = parse_table_identifier(&view); - let branch = branch_from_name.unwrap_or("main".to_string()); + let (namespace_parts, branch_from_ns) = parse_namespace(&namespace); + let branch = branch_from_name + .or(branch_from_ns) + .unwrap_or_else(|| "main".to_string()); - let namespace_parts: Vec = namespace.split('\x1F').map(|s| s.to_string()).collect(); + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), + Err(e) => { + tracing::error!(error = %e, "get_view: failed to load catalog"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(); + } + }; - match store + let asset = match store .get_asset( tenant_id, &catalog_name, Some(branch), - namespace_parts, + namespace_parts.clone(), view_name, ) .await { - Ok(Some(asset)) => { - if asset.kind == AssetType::View { - (StatusCode::OK, Json(ViewResponse::from(asset))).into_response() - } else { - (StatusCode::NOT_FOUND, "Asset is not a view").into_response() - } + Ok(Some(a)) => a, + Ok(None) => return (StatusCode::NOT_FOUND, "View not found").into_response(), + Err(e) => { + tracing::error!(error = %e, "get_view: failed to load asset"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(); + } + }; + + if asset.kind != AssetType::View { + return (StatusCode::NOT_FOUND, "Asset is not a view").into_response(); + } + + // B0e: a view's `properties["sql"]` is its whole definition - business + // logic, column names, sometimes filter predicates that reveal the data. + // Reading it used to require nothing beyond being authenticated. + let scope = PermissionScope::Asset { + catalog_id: catalog.id, + namespace: namespace_parts.join("."), + asset_id: asset.id, + }; + match check_permission(&store, &session, &Action::Read, &scope).await { + Ok(true) => (StatusCode::OK, Json(ViewResponse::from(asset))).into_response(), + Ok(false) => (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Err(e) => { + tracing::error!(error = %e, "get_view: permission check failed"); + (StatusCode::INTERNAL_SERVER_ERROR, "Permission check failed").into_response() } - Ok(None) => (StatusCode::NOT_FOUND, "View not found").into_response(), - Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(), } } #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct RegisterAssetRequest { pub name: String, pub kind: AssetType, @@ -487,6 +547,7 @@ pub async fn list_assets( // Filter assets based on permissions let filtered = crate::authz_utils::filter_assets( + tenant_id, assets_with_metadata, &permissions, session.role, @@ -613,6 +674,7 @@ mod tests { tenant_id: Some(tenant_id), expires_at: chrono::Utc::now() + chrono::Duration::hours(1), issued_at: chrono::Utc::now(), + token_id: None, }; let payload = RegisterAssetRequest { @@ -646,6 +708,7 @@ mod tests { tenant_id: Some(tenant_id), expires_at: chrono::Utc::now() + chrono::Duration::hours(1), issued_at: chrono::Utc::now(), + token_id: None, }; let payload = RegisterAssetRequest { @@ -680,6 +743,7 @@ mod tests { tenant_id: Some(tenant_id), expires_at: chrono::Utc::now() + chrono::Duration::hours(1), issued_at: chrono::Utc::now(), + token_id: None, }; let payload = RegisterAssetRequest { @@ -713,6 +777,7 @@ mod tests { tenant_id: Some(tenant_id), expires_at: chrono::Utc::now() + chrono::Duration::hours(1), issued_at: chrono::Utc::now(), + token_id: None, }; let asset_types = vec![ diff --git a/pangolin/pangolin_api/src/audit_tests.rs b/pangolin/pangolin_api/src/audit_tests.rs index cbe6a5b..11981a9 100644 --- a/pangolin/pangolin_api/src/audit_tests.rs +++ b/pangolin/pangolin_api/src/audit_tests.rs @@ -77,7 +77,8 @@ mod tests { .header("Authorization", format!("Bearer {}", root_token)) .header("Content-Type", "application/json") .body(Body::from( - json!({"name":"audit-tenant", "organization":"org"}).to_string(), + json!({"name":"audit-tenant", "properties": {"organization":"org"}}) + .to_string(), )) .unwrap(), ) @@ -106,7 +107,8 @@ mod tests { .header("Authorization", format!("Bearer {}", user_token)) .header("Content-Type", "application/json") .body(Body::from( - json!({"name":"cat","warehouse_name":"wh","type":"pangolin"}).to_string(), + json!({"name":"cat","warehouse_name":"wh","catalog_type":"Local"}) + .to_string(), )) .unwrap(), ) diff --git a/pangolin/pangolin_api/src/auth.rs b/pangolin/pangolin_api/src/auth.rs index 50592a3..b13700f 100644 --- a/pangolin/pangolin_api/src/auth.rs +++ b/pangolin/pangolin_api/src/auth.rs @@ -47,6 +47,14 @@ impl Claims { .ok_or("Invalid issued_at timestamp")?, expires_at: DateTime::from_timestamp(self.exp, 0) .ok_or("Invalid expires_at timestamp")?, + // Carried through so logout can revoke the token that is actually + // presented rather than the user id (B0j). + token_id: self + .jti + .as_ref() + .map(|id| Uuid::parse_str(id)) + .transpose() + .map_err(|e| e.to_string())?, }) } } diff --git a/pangolin/pangolin_api/src/auth_middleware.rs b/pangolin/pangolin_api/src/auth_middleware.rs index b8f3a65..4ca76eb 100644 --- a/pangolin/pangolin_api/src/auth_middleware.rs +++ b/pangolin/pangolin_api/src/auth_middleware.rs @@ -57,6 +57,12 @@ pub fn create_session( role, issued_at: now, expires_at, + // Sessions built here are not derived from a presented JWT: they are + // either about to have a token minted *from* them (login), or backed by + // an API key / root basic auth, which have nothing to revoke. Sessions + // that do come from a bearer token get their `jti` in + // `Claims::to_session`. + token_id: None, } } @@ -223,6 +229,18 @@ pub async fn auth_middleware( // tests and tools can exercise a specific identity even in NO_AUTH mode. } + // Public endpoints, matched structurally rather than by suffix (A-11). + // + // B0o: this check used to sit *below* the API-key branch, which returned + // unconditionally. A client that sets `X-API-Key` globally - the normal way + // to configure an HTTP client - therefore could not reach `/v1/config`, + // `/health` or the OAuth token endpoint at all, the opposite of the + // documented ordering. Resolving public paths first also keeps an + // unauthenticated `/health` probe away from bcrypt. + if crate::public_paths::is_public_path(&path) { + return next.run(req).await; + } + // Service-user API key. if let Some(api_key_header) = req.headers().get("X-API-Key") { let Ok(api_key) = api_key_header.to_str() else { @@ -271,11 +289,6 @@ pub async fn auth_middleware( } } - // Public endpoints, matched structurally rather than by suffix (A-11). - if crate::public_paths::is_public_path(&path) { - return next.run(req).await; - } - let auth_header = req .headers() .get(header::AUTHORIZATION) @@ -369,8 +382,19 @@ pub async fn auth_middleware( // Revocation must fail *closed*. Previously an error from the store was // logged and ignored, so during a database blip every revoked token was // accepted again (A-13). - if let Some(ref jti_str) = claims.jti { - if let Ok(token_id) = uuid::Uuid::parse_str(jti_str) { + // + // B0o: the two nested `if let`s also failed open on the *shape* of the + // claim. A token whose `jti` was present but not a UUID skipped the + // revocation check entirely and was unrevocable for its whole lifetime, and + // a token minted with no `jti` at all was likewise exempt. A malformed `jti` + // is now a hard rejection, and the no-`jti` case is accepted only when the + // operator has explicitly opted into legacy tokens. + match claims.jti.as_deref() { + Some(jti_str) => { + let Ok(token_id) = uuid::Uuid::parse_str(jti_str) else { + tracing::warn!(jti = %jti_str, "rejected a token with a malformed jti"); + return (StatusCode::UNAUTHORIZED, "Invalid token").into_response(); + }; match store.is_token_revoked(token_id).await { Ok(true) => { tracing::warn!(jti = %jti_str, "revoked token presented"); @@ -390,6 +414,15 @@ pub async fn auth_middleware( } } } + None => { + if !crate::config::allow_tokens_without_jti() { + tracing::warn!( + "rejected a token with no jti; it could never be revoked. Re-issue the \ + token, or set PANGOLIN_ALLOW_TOKENS_WITHOUT_JTI=true during migration" + ); + return (StatusCode::UNAUTHORIZED, "Invalid token").into_response(); + } + } } let session = match claims.to_session() { diff --git a/pangolin/pangolin_api/src/authz.rs b/pangolin/pangolin_api/src/authz.rs index adcc747..a3397f0 100644 --- a/pangolin/pangolin_api/src/authz.rs +++ b/pangolin/pangolin_api/src/authz.rs @@ -183,6 +183,7 @@ mod tests { username: "test_user".to_string(), issued_at: chrono::Utc::now(), expires_at: chrono::Utc::now() + chrono::Duration::hours(1), + token_id: None, }; // Target Action/Scope @@ -267,6 +268,7 @@ mod tests { username: "test_user".to_string(), issued_at: chrono::Utc::now(), expires_at: chrono::Utc::now() + chrono::Duration::hours(1), + token_id: None, }; // --- Branch Permission Test --- diff --git a/pangolin/pangolin_api/src/authz_matrix_tests.rs b/pangolin/pangolin_api/src/authz_matrix_tests.rs new file mode 100644 index 0000000..e20e4d0 --- /dev/null +++ b/pangolin/pangolin_api/src/authz_matrix_tests.rs @@ -0,0 +1,686 @@ +//! Permission matrix: who is allowed to call what. +//! +//! Roadmap improvement #0, the highest-leverage addition in the August audit. +//! +//! Every finding in the B0a-B0m cluster was invisible to CI. The code compiled, +//! it was formatted, it was lint-clean, and the tests passed - because nothing +//! asserted *who is allowed to call what*. A handler that simply forgot to call +//! `check_permission` looks identical, to every existing check, to one that +//! calls it correctly. `POST /api/v1/tokens` minting a `Root` JWT for any +//! authenticated caller sat there through a full security release. +//! +//! This module drives each sensitive route as several principals and asserts +//! the outcome for each. A missing authorization check fails here as a `200` +//! where a `403` was expected, which is the only signal that distinguishes the +//! two shapes of handler from the outside. +//! +//! Deliberately end-to-end through the real router: a unit test of +//! `check_permission` proves the function works, not that the handler calls it, +//! and "the handler does not call it" is the entire bug class. + +use crate::tests_common::EnvGuard; +use axum::{ + body::{to_bytes, Body}, + http::{Request, StatusCode}, + Router, +}; +use pangolin_store::memory::MemoryStore; +use serde_json::json; +use serial_test::serial; +use std::sync::Arc; +use tower::ServiceExt; +use uuid::Uuid; + +fn app() -> Router { + let store = Arc::new(MemoryStore::new()); + crate::app(store) +} + +/// The principals every route is driven as. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Principal { + /// Global superuser. + Root, + /// Administrator of the tenant that owns the resource. + TenantAdmin, + /// Ordinary member of the tenant with no explicit grants. + TenantUser, + /// Administrator of a *different* tenant. + ForeignTenantAdmin, +} + +struct Fixture { + app: Router, + root_token: String, + admin_token: String, + user_token: String, + foreign_admin_token: String, + tenant_id: Uuid, + foreign_tenant_id: Uuid, +} + +impl Fixture { + fn token(&self, who: Principal) -> &str { + match who { + Principal::Root => &self.root_token, + Principal::TenantAdmin => &self.admin_token, + Principal::TenantUser => &self.user_token, + Principal::ForeignTenantAdmin => &self.foreign_admin_token, + } + } + + fn tenant_header(&self, who: Principal) -> Uuid { + match who { + Principal::ForeignTenantAdmin => self.foreign_tenant_id, + _ => self.tenant_id, + } + } + + async fn request( + &self, + who: Principal, + method: &str, + uri: &str, + body: Option, + ) -> StatusCode { + let mut builder = Request::builder() + .method(method) + .uri(uri) + .header("Authorization", format!("Bearer {}", self.token(who))) + .header("Content-Type", "application/json") + .header("X-Pangolin-Tenant", self.tenant_header(who).to_string()); + + // Root has no tenant of its own, so it addresses one explicitly. + if who == Principal::Root { + builder = builder.header("X-Pangolin-Tenant", self.tenant_id.to_string()); + } + + let body = match body { + Some(v) => Body::from(v.to_string()), + None => Body::empty(), + }; + + self.app + .clone() + .oneshot(builder.body(body).unwrap()) + .await + .unwrap() + .status() + } +} + +async fn login(app: &Router, username: &str, password: &str, tenant_id: Option) -> String { + let mut body = json!({ "username": username, "password": password }); + if let Some(tid) = tenant_id { + body.as_object_mut() + .unwrap() + .insert("tenant-id".to_string(), json!(tid)); + } + + let response = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/v1/users/login") + .header("Content-Type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::OK, + "login for {username} should succeed" + ); + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let json: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + json["token"].as_str().unwrap().to_string() +} + +async fn create_tenant(app: &Router, root_token: &str, name: &str) -> Uuid { + let response = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/v1/tenants") + .header("Authorization", format!("Bearer {root_token}")) + .header("Content-Type", "application/json") + .body(Body::from(json!({ "name": name }).to_string())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::CREATED); + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let json: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + Uuid::parse_str(json["id"].as_str().unwrap()).unwrap() +} + +async fn create_user( + app: &Router, + token: &str, + tenant_id: Uuid, + username: &str, + password: &str, + role: &str, +) { + let response = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/v1/users") + .header("Authorization", format!("Bearer {token}")) + .header("X-Pangolin-Tenant", tenant_id.to_string()) + .header("Content-Type", "application/json") + .body(Body::from( + json!({ + "username": username, + "password": password, + "email": format!("{username}@example.test"), + "role": role, + "tenant_id": tenant_id, + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::CREATED, + "creating {username} should succeed" + ); +} + +/// Two tenants, each with an admin, plus one ordinary user with no grants. +async fn fixture() -> Fixture { + let app = app(); + let root_token = login(&app, "admin", "password", None).await; + + let tenant_id = create_tenant(&app, &root_token, "matrix-tenant").await; + let foreign_tenant_id = create_tenant(&app, &root_token, "matrix-other-tenant").await; + + // Root creates the tenant admins; root is deliberately *not* allowed to + // create tenant users (it has no tenant context of its own), so the admin + // creates those. + create_user( + &app, + &root_token, + tenant_id, + "matrix_admin", + "password", + "tenant-admin", + ) + .await; + create_user( + &app, + &root_token, + foreign_tenant_id, + "matrix_foreign_admin", + "password", + "tenant-admin", + ) + .await; + + let admin_token = login(&app, "matrix_admin", "password", Some(tenant_id)).await; + create_user( + &app, + &admin_token, + tenant_id, + "matrix_user", + "password", + "tenant-user", + ) + .await; + + Fixture { + user_token: login(&app, "matrix_user", "password", Some(tenant_id)).await, + admin_token, + foreign_admin_token: login( + &app, + "matrix_foreign_admin", + "password", + Some(foreign_tenant_id), + ) + .await, + app, + root_token, + tenant_id, + foreign_tenant_id, + } +} + +fn root_env() -> (EnvGuard, EnvGuard, EnvGuard) { + ( + EnvGuard::new("PANGOLIN_ROOT_USER", "admin"), + EnvGuard::new("PANGOLIN_ROOT_PASSWORD", "password"), + EnvGuard::new( + "PANGOLIN_JWT_SECRET", + "authz-matrix-test-secret-of-adequate-length-000000", + ), + ) +} + +/// **B0a.** Token minting is the single most dangerous endpoint in the API: a +/// `Root` token is a global bypass, because `check_permission` short-circuits +/// for `Root`. This endpoint took no session at all, so any authenticated +/// principal - down to a `TenantUser` - could mint one for any tenant. +#[tokio::test] +#[serial] +async fn token_minting_is_not_open_to_everyone() { + let _env = root_env(); + let f = fixture().await; + + // A TenantUser must not be able to mint anything at all. + let status = f + .request( + Principal::TenantUser, + "POST", + "/api/v1/tokens", + Some(json!({ "tenant_id": f.tenant_id, "roles": ["Root"] })), + ) + .await; + assert_eq!( + status, + StatusCode::FORBIDDEN, + "a TenantUser must not be able to mint a Root token (B0a)" + ); + + // Nor a token for itself. + let status = f + .request( + Principal::TenantUser, + "POST", + "/api/v1/tokens", + Some(json!({ "tenant_id": f.tenant_id })), + ) + .await; + assert_eq!( + status, + StatusCode::FORBIDDEN, + "a TenantUser must not be able to mint tokens (B0a)" + ); + + // A TenantAdmin may mint within its own tenant, but not above its rank. + let status = f + .request( + Principal::TenantAdmin, + "POST", + "/api/v1/tokens", + Some(json!({ "tenant_id": f.tenant_id, "roles": ["Root"] })), + ) + .await; + assert_eq!( + status, + StatusCode::FORBIDDEN, + "a TenantAdmin must not be able to mint a Root token (B0a)" + ); + + // ...and not for somebody else's tenant. + let status = f + .request( + Principal::TenantAdmin, + "POST", + "/api/v1/tokens", + Some(json!({ "tenant_id": f.foreign_tenant_id, "roles": ["tenant-admin"] })), + ) + .await; + assert_eq!( + status, + StatusCode::FORBIDDEN, + "a TenantAdmin must not be able to mint a token for another tenant (B0a)" + ); + + // The legitimate case still works. + let status = f + .request( + Principal::TenantAdmin, + "POST", + "/api/v1/tokens", + Some(json!({ "tenant_id": f.tenant_id, "roles": ["tenant-user"] })), + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "a TenantAdmin must still be able to mint a lesser token for its own tenant" + ); +} + +/// **B0m.** `expires_in_hours` is caller-controlled and used to reach +/// `chrono::Duration::hours` unclamped, which panics on a large enough value - +/// aborting the connection task, since nothing caught it. +#[tokio::test] +#[serial] +async fn an_absurd_token_lifetime_does_not_take_the_server_down() { + let _env = root_env(); + let f = fixture().await; + + let status = f + .request( + Principal::TenantAdmin, + "POST", + "/api/v1/tokens", + Some(json!({ + "tenant_id": f.tenant_id, + "roles": ["tenant-user"], + "expires_in_hours": u64::MAX, + })), + ) + .await; + + // Clamped rather than panicking; either a clamp (200) or a refusal (400) is + // acceptable, a dropped connection is not. + assert!( + status == StatusCode::OK || status == StatusCode::BAD_REQUEST, + "an absurd expires_in_hours must be handled, not panic (B0m); got {status}" + ); +} + +/// **Improvement #0.** Request structs reject fields they do not have. +/// +/// This is what turns client drift from a silent no-op into an error. Before +/// it, a CLI sending `warehouse` instead of `warehouse_name` got a `201` for a +/// catalog created without the warehouse it named. +#[tokio::test] +#[serial] +async fn unknown_request_fields_are_rejected() { + let _env = root_env(); + let f = fixture().await; + + let status = f + .request( + Principal::TenantAdmin, + "POST", + "/api/v1/catalogs", + Some(json!({ "name": "drift", "warehouse": "wh", "type": "pangea" })), + ) + .await; + + assert_eq!( + status, + StatusCode::UNPROCESSABLE_ENTITY, + "a request naming fields the server does not have must be refused, not \ + silently emptied (improvement #0)" + ); +} + +/// **B0e.** View creation and reading had no authorization at all. A view's +/// `properties["sql"]` is its whole definition. +#[tokio::test] +#[serial] +async fn view_endpoints_require_permission() { + let _env = root_env(); + let f = fixture().await; + + // Setup: an admin creates the catalog and namespace. + assert_eq!( + f.request( + Principal::TenantAdmin, + "POST", + "/api/v1/warehouses", + Some(json!({ + "name": "wh", + "use_sts": false, + "storage_config": { "type": "filesystem", "root": "/tmp/pangolin-authz" } + })), + ) + .await, + StatusCode::CREATED + ); + assert_eq!( + f.request( + Principal::TenantAdmin, + "POST", + "/api/v1/catalogs", + Some(json!({ "name": "cat", "warehouse_name": "wh", "catalog_type": "Local" })), + ) + .await, + StatusCode::CREATED + ); + assert_eq!( + f.request( + Principal::TenantAdmin, + "POST", + "/v1/cat/namespaces", + Some(json!({ "namespace": ["ns"] })), + ) + .await, + StatusCode::OK + ); + + // A TenantUser with no grants must not be able to create a view. + let status = f + .request( + Principal::TenantUser, + "POST", + "/v1/cat/namespaces/ns/views", + Some(json!({ "name": "v", "sql": "SELECT 1" })), + ) + .await; + assert_eq!( + status, + StatusCode::FORBIDDEN, + "creating a view must require Create on the namespace (B0e)" + ); +} + +/// **B0f.** Maintenance expires snapshots and deletes orphan files. It had no +/// session and no check, and ran against a hardcoded `"default"` catalog. +#[tokio::test] +#[serial] +async fn maintenance_requires_permission() { + let _env = root_env(); + let f = fixture().await; + + let status = f + .request( + Principal::TenantUser, + "POST", + "/v1/cat/namespaces/ns/tables/t/maintenance", + Some(json!({ "job_type": "expire_snapshots" })), + ) + .await; + + assert!( + status == StatusCode::FORBIDDEN || status == StatusCode::NOT_FOUND, + "destructive maintenance must never be reachable by an ungranted user \ + (B0f); got {status}" + ); + assert_ne!( + status, + StatusCode::OK, + "maintenance ran for a user with no grants (B0f)" + ); +} + +/// **B0b.** Credential vending hands out real cloud-storage credentials. It +/// performed no authorization, never looked the table up, and hardcoded +/// read+write. +#[tokio::test] +#[serial] +async fn credential_vending_requires_permission() { + let _env = root_env(); + let f = fixture().await; + + let status = f + .request( + Principal::TenantUser, + "GET", + "/v1/cat/namespaces/ns/tables/does_not_exist/credentials", + None, + ) + .await; + + assert_ne!( + status, + StatusCode::OK, + "credentials were vended for a table that does not exist, to a user \ + with no grants (B0b)" + ); +} + +/// **B0i.** A tenant admin must not reach another tenant's resources, even +/// though `check_permission` short-circuits on the role. +#[tokio::test] +#[serial] +async fn a_tenant_admin_cannot_reach_another_tenants_catalogs() { + let _env = root_env(); + let f = fixture().await; + + // Tenant A's admin creates a catalog. + assert_eq!( + f.request( + Principal::TenantAdmin, + "POST", + "/api/v1/warehouses", + Some(json!({ + "name": "wh", + "use_sts": false, + "storage_config": { "type": "filesystem", "root": "/tmp/pangolin-authz-2" } + })), + ) + .await, + StatusCode::CREATED + ); + assert_eq!( + f.request( + Principal::TenantAdmin, + "POST", + "/api/v1/catalogs", + Some(json!({ "name": "private", "warehouse_name": "wh", "catalog_type": "Local" })), + ) + .await, + StatusCode::CREATED + ); + + // Tenant B's admin must not see it. Their requests are scoped to their own + // tenant, so the catalog must simply not be there. + let status = f + .request( + Principal::ForeignTenantAdmin, + "GET", + "/api/v1/catalogs/private", + None, + ) + .await; + + assert_ne!( + status, + StatusCode::OK, + "a tenant admin reached another tenant's catalog (B0i)" + ); +} + +/// **B0j.** Logout must actually revoke: the token has to stop working. +/// +/// It used to revoke `session.user_id`, which no token carries as its `jti`, so +/// the revocation check could never match and logout was cosmetic. +#[tokio::test] +#[serial] +async fn logout_revokes_the_presented_token() { + let _env = root_env(); + let f = fixture().await; + + // The token works before logout. + assert_eq!( + f.request(Principal::TenantAdmin, "GET", "/api/v1/catalogs", None) + .await, + StatusCode::OK + ); + + assert_eq!( + f.request( + Principal::TenantAdmin, + "POST", + "/api/v1/auth/revoke", + Some(json!({ "reason": "logout" })), + ) + .await, + StatusCode::OK + ); + + // And must not afterwards. + let status = f + .request(Principal::TenantAdmin, "GET", "/api/v1/catalogs", None) + .await; + assert_eq!( + status, + StatusCode::UNAUTHORIZED, + "a revoked token kept working after logout (B0j)" + ); +} + +/// **B0k.** The OAuth code-exchange endpoint must be reachable without a +/// bearer token: it is the endpoint whose job is to issue the first one. +#[tokio::test] +#[serial] +async fn the_oauth_exchange_endpoint_is_reachable_unauthenticated() { + let _env = root_env(); + let app = app(); + + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/api/v1/oauth/exchange") + .header("Content-Type", "application/json") + .body(Body::from(json!({ "code": "nonexistent" }).to_string())) + .unwrap(), + ) + .await + .unwrap(); + + assert_ne!( + response.status(), + StatusCode::UNAUTHORIZED, + "the OAuth code-exchange endpoint demanded the token it exists to \ + issue, making the login flow unreachable (B0k)" + ); +} + +/// Unauthenticated requests are refused across the board. +/// +/// The floor under everything above: if this regresses, the per-principal +/// assertions stop meaning anything. +#[tokio::test] +#[serial] +async fn protected_routes_reject_anonymous_callers() { + let _env = root_env(); + let app = app(); + + for (method, uri) in [ + ("GET", "/api/v1/catalogs"), + ("POST", "/api/v1/catalogs"), + ("GET", "/api/v1/warehouses"), + ("POST", "/api/v1/tokens"), + ("GET", "/api/v1/users"), + ("GET", "/api/v1/tenants"), + ("POST", "/api/v1/permissions"), + ] { + let response = app + .clone() + .oneshot( + Request::builder() + .method(method) + .uri(uri) + .header("Content-Type", "application/json") + .body(Body::from("{}")) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::UNAUTHORIZED, + "{method} {uri} served an anonymous caller" + ); + } +} diff --git a/pangolin/pangolin_api/src/authz_utils.rs b/pangolin/pangolin_api/src/authz_utils.rs index 999f5fd..c2abfec 100644 --- a/pangolin/pangolin_api/src/authz_utils.rs +++ b/pangolin/pangolin_api/src/authz_utils.rs @@ -3,12 +3,26 @@ use pangolin_core::permission::{Action, Permission, PermissionScope}; use pangolin_core::user::UserRole; use uuid::Uuid; +/// Does a `Tenant`-scoped grant apply to a resource in `resource_tenant_id`? +/// +/// B0i: all three access checks below used a bare `PermissionScope::Tenant => +/// true`, never comparing the grant's own `tenant_id` against the resource's. +/// A tenant-wide grant issued in tenant A therefore satisfied access checks for +/// resources in tenant B. Nothing exploited it today only because callers +/// pre-scope their store queries to one tenant - but every path that can +/// surface cross-tenant rows (root impersonation, search, dashboards) leaked +/// through it, and the invariant was one refactor away from mattering. +fn tenant_grant_applies(perm: &Permission, resource_tenant_id: Uuid) -> bool { + perm.tenant_id == resource_tenant_id +} + /// Check if a user has access to a catalog based on their permissions /// /// Checks for Read or Discoverable actions on: /// - Exact catalog scope -/// - Tenant-wide scope +/// - Tenant-wide scope, when the grant belongs to the resource's tenant pub fn has_catalog_access( + resource_tenant_id: Uuid, catalog_id: Uuid, permissions: &[Permission], required_actions: &[Action], @@ -18,7 +32,8 @@ pub fn has_catalog_access( let scope_matches = matches!( &perm.scope, PermissionScope::Catalog { catalog_id: cid } if *cid == catalog_id - ) || matches!(&perm.scope, PermissionScope::Tenant); + ) || (matches!(&perm.scope, PermissionScope::Tenant) + && tenant_grant_applies(perm, resource_tenant_id)); // Check if permission has any of the required actions let has_action = required_actions @@ -34,8 +49,9 @@ pub fn has_catalog_access( /// Checks for Read or Discoverable actions on: /// - Exact namespace scope /// - Parent catalog scope -/// - Tenant-wide scope +/// - Tenant-wide scope, when the grant belongs to the resource's tenant pub fn has_namespace_access( + resource_tenant_id: Uuid, catalog_id: Uuid, namespace: &str, permissions: &[Permission], @@ -49,7 +65,7 @@ pub fn has_namespace_access( namespace: ns, } => *cid == catalog_id && ns == namespace, PermissionScope::Catalog { catalog_id: cid } => *cid == catalog_id, - PermissionScope::Tenant => true, + PermissionScope::Tenant => tenant_grant_applies(perm, resource_tenant_id), _ => false, }; @@ -68,8 +84,9 @@ pub fn has_namespace_access( /// - Exact asset scope /// - Parent namespace scope /// - Parent catalog scope -/// - Tenant-wide scope +/// - Tenant-wide scope, when the grant belongs to the resource's tenant pub fn has_asset_access( + resource_tenant_id: Uuid, catalog_id: Uuid, namespace: &str, asset_id: Uuid, @@ -89,7 +106,7 @@ pub fn has_asset_access( namespace: ns, } => *cid == catalog_id && ns == namespace, PermissionScope::Catalog { catalog_id: cid } => *cid == catalog_id, - PermissionScope::Tenant => true, + PermissionScope::Tenant => tenant_grant_applies(perm, resource_tenant_id), _ => false, }; @@ -107,6 +124,7 @@ pub fn has_asset_access( /// Returns only catalogs the user has Read or Discoverable access to. /// Root and TenantAdmin users bypass filtering. pub fn filter_catalogs( + resource_tenant_id: Uuid, catalogs: Vec, permissions: &[Permission], user_role: UserRole, @@ -120,7 +138,14 @@ pub fn filter_catalogs( catalogs .into_iter() - .filter(|catalog| has_catalog_access(catalog.id, permissions, &required_actions)) + .filter(|catalog| { + has_catalog_access( + resource_tenant_id, + catalog.id, + permissions, + &required_actions, + ) + }) .collect() } @@ -129,6 +154,7 @@ pub fn filter_catalogs( /// Returns only namespaces the user has Read or Discoverable access to. /// Root and TenantAdmin users bypass filtering. pub fn filter_namespaces( + resource_tenant_id: Uuid, namespaces: Vec<(Namespace, String)>, permissions: &[Permission], user_role: UserRole, @@ -147,7 +173,13 @@ pub fn filter_namespaces( // Get catalog ID from the map if let Some(&catalog_id) = catalog_id_map.get(catalog_name) { let namespace_str = namespace.name.join("."); - has_namespace_access(catalog_id, &namespace_str, permissions, &required_actions) + has_namespace_access( + resource_tenant_id, + catalog_id, + &namespace_str, + permissions, + &required_actions, + ) } else { false } @@ -160,6 +192,7 @@ pub fn filter_namespaces( /// Returns only assets the user has Read or Discoverable access to. /// Root and TenantAdmin users bypass filtering. pub fn filter_assets( + resource_tenant_id: Uuid, assets: Vec<( Asset, Option, @@ -196,6 +229,7 @@ pub fn filter_assets( if let Some(&catalog_id) = catalog_id_map.get(catalog_name) { let namespace_str = namespace.join("."); has_asset_access( + resource_tenant_id, catalog_id, &namespace_str, asset.id, @@ -215,23 +249,29 @@ mod tests { use chrono::Utc; use std::collections::HashSet; - #[test] - fn test_has_catalog_access_with_catalog_permission() { - let catalog_id = Uuid::new_v4(); + /// Build a permission with a single `Read` action. + fn read_permission(tenant_id: Uuid, scope: PermissionScope) -> Permission { let mut actions = HashSet::new(); actions.insert(Action::Read); - - let permission = Permission { + Permission { id: Uuid::new_v4(), user_id: Uuid::new_v4(), - tenant_id: Uuid::new_v4(), - scope: PermissionScope::Catalog { catalog_id }, + tenant_id, + scope, actions, granted_by: Uuid::new_v4(), granted_at: Utc::now(), - }; + } + } + + #[test] + fn test_has_catalog_access_with_catalog_permission() { + let tenant_id = Uuid::new_v4(); + let catalog_id = Uuid::new_v4(); + let permission = read_permission(tenant_id, PermissionScope::Catalog { catalog_id }); assert!(has_catalog_access( + tenant_id, catalog_id, &[permission], &[Action::Read] @@ -240,47 +280,70 @@ mod tests { #[test] fn test_has_catalog_access_with_tenant_permission() { + let tenant_id = Uuid::new_v4(); let catalog_id = Uuid::new_v4(); - let mut actions = HashSet::new(); - actions.insert(Action::Read); - - let permission = Permission { - id: Uuid::new_v4(), - user_id: Uuid::new_v4(), - tenant_id: Uuid::new_v4(), - scope: PermissionScope::Tenant, - actions, - granted_by: Uuid::new_v4(), - granted_at: Utc::now(), - }; + let permission = read_permission(tenant_id, PermissionScope::Tenant); assert!(has_catalog_access( + tenant_id, catalog_id, &[permission], &[Action::Read] )); } + /// Regression test for B0i: a `Tenant`-scoped grant issued in tenant A must + /// not satisfy access for a resource in tenant B. + #[test] + fn tenant_scoped_grant_does_not_cross_tenants() { + let tenant_a = Uuid::new_v4(); + let tenant_b = Uuid::new_v4(); + let catalog_id = Uuid::new_v4(); + let asset_id = Uuid::new_v4(); + let permission = read_permission(tenant_a, PermissionScope::Tenant); + let grants = [permission]; + + assert!(has_catalog_access( + tenant_a, + catalog_id, + &grants, + &[Action::Read] + )); + assert!( + !has_catalog_access(tenant_b, catalog_id, &grants, &[Action::Read]), + "a tenant-A grant must not authorize a tenant-B catalog" + ); + assert!( + !has_namespace_access(tenant_b, catalog_id, "sales", &grants, &[Action::Read]), + "a tenant-A grant must not authorize a tenant-B namespace" + ); + assert!( + !has_asset_access( + tenant_b, + catalog_id, + "sales", + asset_id, + &grants, + &[Action::Read] + ), + "a tenant-A grant must not authorize a tenant-B asset" + ); + } + #[test] fn test_has_catalog_access_without_permission() { + let tenant_id = Uuid::new_v4(); let catalog_id = Uuid::new_v4(); let other_catalog_id = Uuid::new_v4(); - let mut actions = HashSet::new(); - actions.insert(Action::Read); - - let permission = Permission { - id: Uuid::new_v4(), - user_id: Uuid::new_v4(), - tenant_id: Uuid::new_v4(), - scope: PermissionScope::Catalog { + let permission = read_permission( + tenant_id, + PermissionScope::Catalog { catalog_id: other_catalog_id, }, - actions, - granted_by: Uuid::new_v4(), - granted_at: Utc::now(), - }; + ); assert!(!has_catalog_access( + tenant_id, catalog_id, &[permission], &[Action::Read] @@ -299,7 +362,7 @@ mod tests { properties: std::collections::HashMap::new(), }]; - let filtered = filter_catalogs(catalogs.clone(), &[], UserRole::Root); + let filtered = filter_catalogs(Uuid::new_v4(), catalogs.clone(), &[], UserRole::Root); assert_eq!(filtered.len(), catalogs.len()); } @@ -316,20 +379,15 @@ mod tests { properties: std::collections::HashMap::new(), }]; - let mut actions = HashSet::new(); - actions.insert(Action::Read); - - let permission = Permission { - id: Uuid::new_v4(), - user_id: Uuid::new_v4(), - tenant_id: Uuid::new_v4(), - scope: PermissionScope::Catalog { catalog_id }, - actions, - granted_by: Uuid::new_v4(), - granted_at: Utc::now(), - }; + let tenant_id = Uuid::new_v4(); + let permission = read_permission(tenant_id, PermissionScope::Catalog { catalog_id }); - let filtered = filter_catalogs(catalogs.clone(), &[permission], UserRole::TenantUser); + let filtered = filter_catalogs( + tenant_id, + catalogs.clone(), + &[permission], + UserRole::TenantUser, + ); assert_eq!(filtered.len(), 1); } @@ -346,7 +404,7 @@ mod tests { properties: std::collections::HashMap::new(), }]; - let filtered = filter_catalogs(catalogs, &[], UserRole::TenantUser); + let filtered = filter_catalogs(Uuid::new_v4(), catalogs, &[], UserRole::TenantUser); assert_eq!(filtered.len(), 0); } } diff --git a/pangolin/pangolin_api/src/business_metadata_handlers.rs b/pangolin/pangolin_api/src/business_metadata_handlers.rs index 1b41c7e..f77272f 100644 --- a/pangolin/pangolin_api/src/business_metadata_handlers.rs +++ b/pangolin/pangolin_api/src/business_metadata_handlers.rs @@ -14,6 +14,7 @@ use utoipa::ToSchema; use uuid::Uuid; #[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct AddMetadataRequest { pub description: Option, pub tags: Vec, @@ -26,6 +27,33 @@ pub struct MetadataResponse { pub metadata: BusinessMetadata, } +/// Resolve an asset's permission scope, for the handlers keyed only by asset id. +/// +/// Several business-metadata handlers took an asset id straight off the path +/// and acted on it with no authorization at all - the same class as B0e. An +/// asset id alone is not a scope, so the catalog and namespace have to be +/// resolved first; `get_asset_by_id` does that in one lookup. +async fn asset_scope( + store: &AppState, + tenant_id: uuid::Uuid, + asset_id: uuid::Uuid, +) -> Result, anyhow::Error> { + let Some((asset, catalog_name, namespace)) = store.get_asset_by_id(tenant_id, asset_id).await? + else { + return Ok(None); + }; + + let Some(catalog) = store.get_catalog(tenant_id, catalog_name).await? else { + return Ok(None); + }; + + Ok(Some(PermissionScope::Asset { + catalog_id: catalog.id, + namespace: namespace.join("."), + asset_id: asset.id, + })) +} + #[utoipa::path( post, path = "/api/v1/assets/{asset_id}/metadata", @@ -171,10 +199,31 @@ pub async fn get_business_metadata( )] pub async fn delete_business_metadata( State(store): State, - Extension(_session): Extension, + Extension(session): Extension, Path(asset_id): Path, ) -> impl IntoResponse { - // Check permission logic + // The `// Check permission logic` that stood here was the whole of the + // authorization: any authenticated tenant member could delete any asset's + // business metadata - descriptions, tags, and the `discoverable` flag that + // governs who can see the asset at all. + let tenant_id = session.tenant_id.unwrap_or_default(); + let scope = match asset_scope(&store, tenant_id, asset_id).await { + Ok(Some(scope)) => scope, + Ok(None) => return (StatusCode::NOT_FOUND, "Asset not found").into_response(), + Err(e) => { + tracing::error!(error = %e, "delete_business_metadata: failed to resolve asset"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(); + } + }; + + match crate::authz::check_permission(&store, &session, &Action::Delete, &scope).await { + Ok(true) => (), + Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Err(e) => { + tracing::error!(error = %e, "delete_business_metadata: permission check failed"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Permission check failed").into_response(); + } + } match store.delete_business_metadata(asset_id).await { Ok(_) => StatusCode::NO_CONTENT.into_response(), @@ -186,12 +235,42 @@ pub async fn delete_business_metadata( } } +/// Query parameters for asset search. +/// +/// B35: `tags` was typed `Option>` and extracted with axum's +/// `Query`, which uses `serde_urlencoded` - and `serde_urlencoded` cannot +/// deserialize repeated keys (`?tags=a&tags=b`) into a `Vec`. The UI sent +/// exactly that, so every tag-filtered search returned `400 Bad Request`: +/// tag-filtered search was broken end to end and had presumably never worked. +/// +/// A comma-separated string is the fix that keeps working with a plain `Query` +/// extractor, and it is also what an operator would type by hand. Repeated +/// keys are still accepted by the deserializer below, so a client that sends +/// the old shape gets the last value rather than a 400. #[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct SearchRequest { pub query: String, + /// Comma-separated tag names, e.g. `?tags=pii,finance`. + #[serde(default, deserialize_with = "deserialize_comma_separated")] pub tags: Option>, } +/// Deserialize `a,b,c` into `["a", "b", "c"]`, trimming and dropping blanks. +fn deserialize_comma_separated<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, +{ + let raw = Option::::deserialize(deserializer)?; + Ok(raw.map(|value| { + value + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect() + })) +} + #[utoipa::path( get, path = "/api/v1/assets/search", @@ -243,6 +322,7 @@ pub async fn search_assets( // Apply permission-based filtering let filtered_results = crate::authz_utils::filter_assets( + tenant_id, results, &permissions, session.role.clone(), @@ -262,6 +342,7 @@ pub async fn search_assets( let ns_str = namespace.join("."); let required_actions = vec![pangolin_core::permission::Action::Read]; crate::authz_utils::has_asset_access( + tenant_id, catalog_id, &ns_str, asset.id, diff --git a/pangolin/pangolin_api/src/cached_store.rs b/pangolin/pangolin_api/src/cached_store.rs index ff99fed..52a36e0 100644 --- a/pangolin/pangolin_api/src/cached_store.rs +++ b/pangolin/pangolin_api/src/cached_store.rs @@ -101,16 +101,25 @@ impl CatalogStore for CachedCatalogStore { } async fn delete_warehouse(&self, tenant_id: Uuid, name: String) -> Result<()> { - // Invalidate cache on delete + // B16m: delete first, *then* invalidate. Invalidating first opened a + // window in which a concurrent `get_warehouse` missed the cache, read the + // still-present row, and re-inserted it with a full TTL - so the deleted + // warehouse's cloud credentials kept being vended for up to the TTL after + // delete returned success. Deleting first means any racing read either + // sees the row (and the invalidate below clears it) or does not find it. + let key = (tenant_id, name.clone()); + let result = self.inner.delete_warehouse(tenant_id, name.clone()).await; + tracing::info!( "Cache INVALIDATE (Delete) for warehouse: {}/{}", tenant_id, name ); - self.warehouse_cache - .invalidate(&(tenant_id, name.clone())) - .await; - self.inner.delete_warehouse(tenant_id, name).await + // Invalidate on the error path too: the delete may have partially + // applied, and a stale credential entry is the worse failure. + self.warehouse_cache.invalidate(&key).await; + + result } // --- Passthrough Operations (Uncached) --- @@ -224,6 +233,18 @@ impl CatalogStore for CachedCatalogStore { .await } + async fn replace_namespace_properties( + &self, + tenant_id: Uuid, + catalog_name: &str, + namespace: Vec, + properties: std::collections::HashMap, + ) -> Result<()> { + self.inner + .replace_namespace_properties(tenant_id, catalog_name, namespace, properties) + .await + } + // Asset async fn create_asset( &self, @@ -476,6 +497,9 @@ impl CatalogStore for CachedCatalogStore { async fn write_file(&self, location: &str, content: Vec) -> Result<()> { self.inner.write_file(location, content).await } + async fn delete_file(&self, location: &str) -> Result<()> { + self.inner.delete_file(location).await + } // Maintenance Operations async fn expire_snapshots( diff --git a/pangolin/pangolin_api/src/config.rs b/pangolin/pangolin_api/src/config.rs index 919bcbb..b08649d 100644 --- a/pangolin/pangolin_api/src/config.rs +++ b/pangolin/pangolin_api/src/config.rs @@ -102,6 +102,13 @@ pub struct AppConfig { pub oauth_redirect_allowlist: Vec, /// Whether API keys minted before the key-id format are still accepted. pub allow_legacy_api_keys: bool, + /// Whether JWTs carrying no `jti` are still accepted (off by default). + /// + /// `Claims.jti` is `Option` "for compatibility", and the middleware + /// used to skip the revocation check for such tokens - making them + /// unrevocable for their full lifetime (B0o). They are now rejected unless + /// an operator opts in for a migration window. + pub allow_tokens_without_jti: bool, /// Bind address and port. pub bind_address: String, pub port: u16, @@ -213,7 +220,14 @@ impl AppConfig { let bind_address = env_opt("PANGOLIN_BIND_ADDRESS").unwrap_or_else(|| "0.0.0.0".to_string()); - if no_auth && !dev_mode && !is_loopback(&bind_address) { + // B0h: the guard used to read `no_auth && !dev_mode && !is_loopback(..)`. + // That `!dev_mode` term meant `PANGOLIN_NO_AUTH=true PANGOLIN_DEV_MODE=true` + // started happily on the default `0.0.0.0` bind and treated every + // anonymous request as `TenantAdmin` - and those two flags are routinely + // set together in compose and dev setups, so the escape hatch was the + // common case. Dev mode relaxes secret strength, never network exposure: + // if auth is off, the listener must be loopback, unconditionally. + if no_auth && !is_loopback(&bind_address) { return Err(ConfigError::NoAuthOnPublicBind(bind_address)); } @@ -250,6 +264,7 @@ impl AppConfig { frontend_url, oauth_redirect_allowlist, allow_legacy_api_keys: env_bool("PANGOLIN_ALLOW_LEGACY_API_KEYS"), + allow_tokens_without_jti: env_bool("PANGOLIN_ALLOW_TOKENS_WITHOUT_JTI"), bind_address, port: env_parsed("PORT", 8080u16)?, log_format, @@ -342,6 +357,14 @@ pub fn allow_legacy_api_keys() -> bool { env_bool("PANGOLIN_ALLOW_LEGACY_API_KEYS") } +/// Whether JWTs with no `jti` are still honoured (off by default, see B0o). +pub fn allow_tokens_without_jti() -> bool { + if let Some(cfg) = CONFIG.get() { + return cfg.allow_tokens_without_jti; + } + env_bool("PANGOLIN_ALLOW_TOKENS_WITHOUT_JTI") +} + /// Constant-time string comparison, for credential checks. /// /// `==` on `&str` short-circuits on the first differing byte and therefore diff --git a/pangolin/pangolin_api/src/credential_signers/azure_signer.rs b/pangolin/pangolin_api/src/credential_signers/azure_signer.rs index cdb7343..db712a8 100644 --- a/pangolin/pangolin_api/src/credential_signers/azure_signer.rs +++ b/pangolin/pangolin_api/src/credential_signers/azure_signer.rs @@ -49,13 +49,18 @@ impl AzureSasSigner { #[async_trait] impl CredentialSigner for AzureSasSigner { + // These are read only under the feature; without the allow, a default + // build warns on every one of them. Prefixing them with `_` instead - + // which is what was here - meant the cfg block referenced names that did + // not exist, so the feature could not compile at all. + #[allow(unused_variables)] async fn generate_credentials( &self, - _resource_path: &str, - _permissions: &[String], + resource_path: &str, + permissions: &[String], duration: Duration, ) -> Result { - let _expires_at = Utc::now() + duration; + let expires_at = Utc::now() + duration; // Check for account key first (works regardless of feature flags) if let Some(account_key) = &self.account_key { diff --git a/pangolin/pangolin_api/src/credential_signers/gcp_signer.rs b/pangolin/pangolin_api/src/credential_signers/gcp_signer.rs index a4fc8c2..4d84b6d 100644 --- a/pangolin/pangolin_api/src/credential_signers/gcp_signer.rs +++ b/pangolin/pangolin_api/src/credential_signers/gcp_signer.rs @@ -1,5 +1,9 @@ use super::{CredentialSigner, VendedCredentials}; use anyhow::Result; +// Only the feature path raises errors with the macro; importing it +// unconditionally warns in a default build. +#[cfg(feature = "gcp-oauth")] +use anyhow::anyhow; use async_trait::async_trait; use chrono::{Duration, Utc}; use std::collections::HashMap; @@ -30,10 +34,15 @@ impl GcpTokenSigner { #[async_trait] impl CredentialSigner for GcpTokenSigner { + // These are read only under the feature; without the allow, a default + // build warns on every one of them. Prefixing them with `_` instead - + // which is what was here - meant the cfg block referenced names that did + // not exist, so the feature could not compile at all. + #[allow(unused_variables)] async fn generate_credentials( &self, - _resource_path: &str, - _permissions: &[String], + resource_path: &str, + permissions: &[String], duration: Duration, ) -> Result { #[cfg(feature = "gcp-oauth")] diff --git a/pangolin/pangolin_api/src/credential_signers/s3_signer.rs b/pangolin/pangolin_api/src/credential_signers/s3_signer.rs index aaafe28..76fdd19 100644 --- a/pangolin/pangolin_api/src/credential_signers/s3_signer.rs +++ b/pangolin/pangolin_api/src/credential_signers/s3_signer.rs @@ -44,11 +44,14 @@ impl S3Signer { #[async_trait] impl CredentialSigner for S3Signer { + // `duration` is read only under `aws-sts`; the other two are unused either + // way. Without the allow, a default build warns on all three. + #[allow(unused_variables)] async fn generate_credentials( &self, _resource_path: &str, _permissions: &[String], - _duration: Duration, + duration: Duration, ) -> Result { #[cfg(feature = "aws-sts")] { @@ -102,9 +105,12 @@ impl CredentialSigner for S3Signer { config.insert("s3.endpoint".to_string(), endpoint.clone()); } - let expires_at = chrono::DateTime::parse_from_rfc3339(creds.expiration()) - .ok() - .map(|dt| dt.with_timezone(&Utc)); + // `expiration()` hands back an `aws_smithy_types::DateTime`, not + // an RFC3339 string, so this went through `parse_from_rfc3339` + // on a value that was never text. + let expiry = creds.expiration(); + let expires_at = + chrono::DateTime::from_timestamp(expiry.secs(), expiry.subsec_nanos()); tracing::info!("✅ Successfully assumed AWS role"); diff --git a/pangolin/pangolin_api/src/dashboard_handlers.rs b/pangolin/pangolin_api/src/dashboard_handlers.rs index 02f81b6..f43b4e1 100644 --- a/pangolin/pangolin_api/src/dashboard_handlers.rs +++ b/pangolin/pangolin_api/src/dashboard_handlers.rs @@ -132,6 +132,7 @@ pub async fn get_dashboard_stats( .await .map_err(ApiError::from)?; let accessible_catalogs = crate::authz_utils::filter_catalogs( + tenant_id, all_catalogs, &permissions, session.role.clone(), @@ -172,6 +173,7 @@ pub async fn get_dashboard_stats( .collect(); let filtered = crate::authz_utils::filter_namespaces( + tenant_id, namespace_tuples, &permissions, session.role.clone(), @@ -186,6 +188,7 @@ pub async fn get_dashboard_stats( let mut accessible_tables_count = 0; if let Ok(all_assets) = store.search_assets(tenant_id, "", None).await { let filtered_assets = crate::authz_utils::filter_assets( + tenant_id, all_assets, &permissions, session.role.clone(), diff --git a/pangolin/pangolin_api/src/federated_catalog_handlers.rs b/pangolin/pangolin_api/src/federated_catalog_handlers.rs index 6003fb1..1042cf8 100644 --- a/pangolin/pangolin_api/src/federated_catalog_handlers.rs +++ b/pangolin/pangolin_api/src/federated_catalog_handlers.rs @@ -18,6 +18,7 @@ type AppState = Arc; // Request/Response types #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CreateFederatedCatalogRequest { pub name: String, pub config: FederatedCatalogConfig, diff --git a/pangolin/pangolin_api/src/iceberg/commit.rs b/pangolin/pangolin_api/src/iceberg/commit.rs index 1ff21b3..9357ac2 100644 --- a/pangolin/pangolin_api/src/iceberg/commit.rs +++ b/pangolin/pangolin_api/src/iceberg/commit.rs @@ -207,14 +207,22 @@ pub fn check_requirements( } } - // Pangolin does not track partition-field id assignment separately - // from the specs themselves, so this cannot be verified. Refusing - // is the honest answer: the alternative is pretending a - // precondition held when it was never examined. - CommitRequirement::AssertLastAssignedPartitionId { .. } => { - return Err(CommitError::Unsupported { - operation: "assert-last-assigned-partition-id".into(), - }) + // Now checkable: `last-partition-id` is a real field on + // `TableMetadata` (B12). Before it existed this requirement had to + // be refused, because the alternative was pretending a precondition + // held when it had never been examined. + CommitRequirement::AssertLastAssignedPartitionId { + last_assigned_partition_id, + } => { + if metadata.last_partition_id != *last_assigned_partition_id { + return Err(CommitError::RequirementFailed { + requirement: "assert-last-assigned-partition-id".into(), + detail: format!( + "last-partition-id is {}, expected {last_assigned_partition_id}", + metadata.last_partition_id + ), + }); + } } CommitRequirement::Unknown => { @@ -236,6 +244,18 @@ pub fn apply_updates( updates: &[CommitUpdate], branch: &str, ) -> Result<(), CommitError> { + // B16c: `-1` in `set-current-schema` / `set-default-spec` / + // `set-default-sort-order` means "the one added *by this commit*". The old + // code resolved it against `metadata.schemas.last()` etc., which for an + // existing table is never empty - so a `-1` sent *without* a preceding + // `add-schema` silently repointed the table at whatever happened to be last + // in the persisted vector (arbitrary if the vector is not in creation + // order), instead of hitting the error arm the message claims. Tracking + // what this commit actually added makes the sentinel mean what it says. + let mut last_added_schema_id: Option = None; + let mut last_added_spec_id: Option = None; + let mut last_added_sort_order_id: Option = None; + for update in updates { match update { CommitUpdate::AssignUuid { uuid } => { @@ -266,17 +286,33 @@ pub fn apply_updates( serde_json::from_value(schema.clone()).map_err(|e| CommitError::Invalid { detail: format!("add-schema: {e}"), })?; - metadata.last_column_id = metadata - .last_column_id - .max(new_schema.fields.iter().map(|f| f.id).max().unwrap_or(0)); + // Reject a duplicate id rather than pushing a second schema the + // rest of the metadata cannot tell apart (B16c). + if metadata + .schemas + .iter() + .any(|s| s.schema_id == new_schema.schema_id) + { + return Err(CommitError::Invalid { + detail: format!( + "add-schema: schema id {} already exists", + new_schema.schema_id + ), + }); + } + // `max_field_id` walks nested fields; taking the max over the + // top-level `fields` alone understates `last-column-id` for any + // schema containing a struct, list or map. + metadata.last_column_id = metadata.last_column_id.max(new_schema.max_field_id()); + last_added_schema_id = Some(new_schema.schema_id); metadata.schemas.push(new_schema); } CommitUpdate::SetCurrentSchema { schema_id } => { // -1 means "the schema added by this same commit". if *schema_id == -1 { - match metadata.schemas.last() { - Some(last) => metadata.current_schema_id = last.schema_id, + match last_added_schema_id { + Some(id) => metadata.current_schema_id = id, None => { return Err(CommitError::Invalid { detail: "set-current-schema: -1 with no schema in this commit" @@ -299,7 +335,7 @@ pub fn apply_updates( serde_json::from_value(snapshot.clone()).map_err(|e| CommitError::Invalid { detail: format!("add-snapshot: {e}"), })?; - add_snapshot(metadata, snapshot_obj, branch); + add_snapshot(metadata, snapshot_obj, branch)?; } CommitUpdate::SetSnapshotRef { @@ -371,18 +407,26 @@ pub fn apply_updates( serde_json::from_value(spec.clone()).map_err(|e| CommitError::Invalid { detail: format!("add-spec: {e}"), })?; + if metadata + .partition_specs + .iter() + .any(|s| s.spec_id == new_spec.spec_id) + { + return Err(CommitError::Invalid { + detail: format!("add-spec: spec id {} already exists", new_spec.spec_id), + }); + } + last_added_spec_id = Some(new_spec.spec_id); metadata.partition_specs.push(new_spec); + // Keep the required `last-partition-id` true (B12). + metadata.recompute_last_partition_id(); } CommitUpdate::SetDefaultSpec { spec_id } => { let target = if *spec_id == -1 { - metadata - .partition_specs - .last() - .map(|s| s.spec_id) - .ok_or_else(|| CommitError::Invalid { - detail: "set-default-spec: -1 with no spec in this commit".into(), - })? + last_added_spec_id.ok_or_else(|| CommitError::Invalid { + detail: "set-default-spec: -1 with no spec in this commit".into(), + })? } else { *spec_id }; @@ -401,19 +445,28 @@ pub fn apply_updates( detail: format!("add-sort-order: {e}"), } })?; + if metadata + .sort_orders + .iter() + .any(|o| o.order_id == new_order.order_id) + { + return Err(CommitError::Invalid { + detail: format!( + "add-sort-order: sort order id {} already exists", + new_order.order_id + ), + }); + } + last_added_sort_order_id = Some(new_order.order_id); metadata.sort_orders.push(new_order); } CommitUpdate::SetDefaultSortOrder { sort_order_id } => { let target = if *sort_order_id == -1 { - metadata - .sort_orders - .last() - .map(|o| o.order_id) - .ok_or_else(|| CommitError::Invalid { - detail: "set-default-sort-order: -1 with no sort order in this commit" - .into(), - })? + last_added_sort_order_id.ok_or_else(|| CommitError::Invalid { + detail: "set-default-sort-order: -1 with no sort order in this commit" + .into(), + })? } else { *sort_order_id }; @@ -466,7 +519,11 @@ pub fn apply_updates( } /// Append a snapshot and move the branch to it. -fn add_snapshot(metadata: &mut TableMetadata, snapshot: Snapshot, branch: &str) { +fn add_snapshot( + metadata: &mut TableMetadata, + snapshot: Snapshot, + branch: &str, +) -> Result<(), CommitError> { let snapshot_id = snapshot.snapshot_id; let timestamp_ms = if snapshot.timestamp_ms > 0 { snapshot.timestamp_ms @@ -475,10 +532,21 @@ fn add_snapshot(metadata: &mut TableMetadata, snapshot: Snapshot, branch: &str) }; // The sequence number is a monotonic counter, *not* a snapshot ID (A-3). - // Honour the client's value when it advances the counter; otherwise assign - // the next one. - let next_sequence = metadata.last_sequence_number + 1; - let sequence_number = if snapshot.sequence_number > metadata.last_sequence_number { + // + // B15: the old rule was "honour any client value greater than the current + // counter". A client could therefore submit `i64::MAX`, after which the next + // commit computed `last_sequence_number + 1` and overflowed - a panic in + // debug builds, a wrap in release, and corrupt commit ordering either way. + // The counter is the server's to advance: a client value is accepted only if + // it is exactly the next one, and anything else is assigned rather than + // honoured. + let next_sequence = metadata + .last_sequence_number + .checked_add(1) + .ok_or_else(|| CommitError::Invalid { + detail: "add-snapshot: sequence number counter is exhausted".into(), + })?; + let sequence_number = if snapshot.sequence_number == next_sequence { snapshot.sequence_number } else { next_sequence @@ -494,15 +562,22 @@ fn add_snapshot(metadata: &mut TableMetadata, snapshot: Snapshot, branch: &str) .push(snapshot); metadata.last_sequence_number = sequence_number; metadata.last_updated_ms = timestamp_ms; - metadata.current_snapshot_id = Some(snapshot_id); - metadata - .snapshot_log - .get_or_insert_with(Vec::new) - .push(SnapshotLogEntry { - timestamp_ms, - snapshot_id, - }); + // B16: `current_snapshot_id` and the `snapshot_log` describe *main*. Setting + // them for any branch meant a `dev`-branch commit changed what `main` + // readers resolve if the metadata document was ever shared across branches + // or exported. Same for fabricating a `main` ref pointing at the branch's + // snapshot: that silently published unreviewed work to main. + if branch == MAIN_REF { + metadata.current_snapshot_id = Some(snapshot_id); + metadata + .snapshot_log + .get_or_insert_with(Vec::new) + .push(SnapshotLogEntry { + timestamp_ms, + snapshot_id, + }); + } let refs = metadata.refs.get_or_insert_with(HashMap::new); refs.insert( @@ -515,18 +590,8 @@ fn add_snapshot(metadata: &mut TableMetadata, snapshot: Snapshot, branch: &str) max_ref_age_ms: None, }, ); - if branch != MAIN_REF { - // Keep `main` consistent with `current_snapshot_id` for readers that - // predate ref tracking. - refs.entry(MAIN_REF.to_string()) - .or_insert(SnapshotReference { - snapshot_id, - ref_type: "branch".to_string(), - min_snapshots_to_keep: None, - max_snapshot_age_ms: None, - max_ref_age_ms: None, - }); - } + + Ok(()) } #[cfg(test)] @@ -545,6 +610,7 @@ mod tests { last_column_id: 1, current_schema_id: 0, schemas: vec![Schema { + type_: "struct".to_string(), schema_id: 0, identifier_field_ids: None, fields: vec![NestedField { @@ -557,6 +623,7 @@ mod tests { }], current_partition_spec_id: 0, partition_specs: vec![], + last_partition_id: 999, default_sort_order_id: 0, sort_orders: vec![], properties: None, @@ -921,8 +988,12 @@ mod tests { assert_eq!(snapshots[1].sequence_number, 2); } + /// B15: the counter is the server's. A client value is honoured only when it + /// is exactly the next one; anything else is replaced by the next value + /// rather than letting the client jump the counter to, say, `i64::MAX` and + /// overflow the *following* commit. #[test] - fn a_client_supplied_sequence_number_may_advance_the_counter() { + fn a_client_supplied_sequence_number_cannot_jump_the_counter() { let mut metadata = base_metadata(); let mut snap = snapshot_json(1, None); snap["sequence-number"] = serde_json::json!(5); @@ -932,25 +1003,100 @@ mod tests { MAIN_REF, ) .unwrap(); - assert_eq!(metadata.last_sequence_number, 5); + assert_eq!(metadata.last_sequence_number, 1); + + // The exact next value is accepted. + let mut snap = snapshot_json(2, None); + snap["sequence-number"] = serde_json::json!(2); + apply_updates( + &mut metadata, + &[CommitUpdate::AddSnapshot { snapshot: snap }], + MAIN_REF, + ) + .unwrap(); + assert_eq!(metadata.last_sequence_number, 2); + } + + /// B15 regression: `i64::MAX` used to be honoured verbatim, so the next + /// commit's `last_sequence_number + 1` overflowed. + #[test] + fn an_absurd_client_sequence_number_does_not_overflow_the_next_commit() { + let mut metadata = base_metadata(); + let mut snap = snapshot_json(1, None); + snap["sequence-number"] = serde_json::json!(i64::MAX); + apply_updates( + &mut metadata, + &[CommitUpdate::AddSnapshot { snapshot: snap }], + MAIN_REF, + ) + .unwrap(); + assert_eq!(metadata.last_sequence_number, 1); + + // The follow-up commit is ordinary arithmetic, not an overflow. + apply_updates( + &mut metadata, + &[CommitUpdate::AddSnapshot { + snapshot: snapshot_json(2, None), + }], + MAIN_REF, + ) + .unwrap(); + assert_eq!(metadata.last_sequence_number, 2); } #[test] - fn adding_a_snapshot_records_the_snapshot_log_and_moves_the_branch() { + fn adding_a_snapshot_on_main_records_the_snapshot_log_and_moves_the_branch() { let mut metadata = base_metadata(); apply_updates( &mut metadata, &[CommitUpdate::AddSnapshot { snapshot: snapshot_json(99, None), }], - "feature", + MAIN_REF, ) .unwrap(); assert_eq!(metadata.current_snapshot_id, Some(99)); - assert_eq!(ref_snapshot_id(&metadata, "feature"), Some(99)); + assert_eq!(ref_snapshot_id(&metadata, MAIN_REF), Some(99)); assert_eq!(metadata.snapshot_log.as_ref().unwrap().len(), 1); } + /// B16: a commit to a feature branch must move *only* that branch. It used + /// to set `current_snapshot_id` for any branch and fabricate a `main` ref + /// pointing at the branch's snapshot, so a `dev` commit changed what `main` + /// readers resolve. + #[test] + fn committing_to_a_feature_branch_leaves_main_alone() { + let mut metadata = base_metadata(); + let main_before = ref_snapshot_id(&metadata, MAIN_REF); + + apply_updates( + &mut metadata, + &[CommitUpdate::AddSnapshot { + snapshot: snapshot_json(99, None), + }], + "feature", + ) + .unwrap(); + + assert_eq!(ref_snapshot_id(&metadata, "feature"), Some(99)); + assert_eq!( + ref_snapshot_id(&metadata, MAIN_REF), + main_before, + "a feature-branch commit must not move main" + ); + assert_eq!( + metadata.current_snapshot_id, main_before, + "current-snapshot-id describes main, not the committed branch" + ); + assert!( + metadata + .snapshot_log + .as_ref() + .is_none_or(|log| log.is_empty()), + "the snapshot log describes main's history" + ); + } + #[test] fn metadata_round_trips_through_json_with_refs() { let mut metadata = base_metadata(); diff --git a/pangolin/pangolin_api/src/iceberg/error.rs b/pangolin/pangolin_api/src/iceberg/error.rs index d05168d..b7f3b18 100644 --- a/pangolin/pangolin_api/src/iceberg/error.rs +++ b/pangolin/pangolin_api/src/iceberg/error.rs @@ -8,7 +8,10 @@ //! //! Pangolin emitted a flat `{"error": ""}`, and most Iceberg handlers //! bypassed the error type entirely and returned bare `(StatusCode, &str)` -//! tuples with a plain-text body (A-6). Engines parse the envelope to tell a +//! tuples with a plain-text body (A-6, and still open as B16j at the August +//! audit despite this module reading as though it were resolved). Every return +//! in `iceberg/` now routes through the helpers below; there are zero bare +//! tuples left in that module. Engines parse the envelope to tell a //! `NoSuchTableException` from a `CommitFailedException`, which is what drives //! their retry logic, so a non-conforming body breaks retries rather than //! merely looking untidy. @@ -80,6 +83,47 @@ pub fn forbidden(detail: &str) -> Response { iceberg_error(StatusCode::FORBIDDEN, "ForbiddenException", detail) } +/// `404` for a view that does not exist. +pub fn no_such_view(identifier: &str) -> Response { + iceberg_error( + StatusCode::NOT_FOUND, + "NoSuchViewException", + &format!("View does not exist: {identifier}"), + ) +} + +/// `400` for a malformed or unusable request. +pub fn bad_request(detail: &str) -> Response { + iceberg_error(StatusCode::BAD_REQUEST, "BadRequestException", detail) +} + +/// `409` for a table that already exists. +pub fn table_already_exists(identifier: &str) -> Response { + iceberg_error( + StatusCode::CONFLICT, + "AlreadyExistsException", + &format!("Table already exists: {identifier}"), + ) +} + +/// `409` for a namespace that already exists. +pub fn namespace_already_exists(namespace: &str) -> Response { + iceberg_error( + StatusCode::CONFLICT, + "AlreadyExistsException", + &format!("Namespace already exists: {namespace}"), + ) +} + +/// `409` for a namespace that still has children. +pub fn namespace_not_empty(namespace: &str) -> Response { + iceberg_error( + StatusCode::CONFLICT, + "NamespaceNotEmptyException", + &format!("Namespace is not empty: {namespace}"), + ) +} + /// `500`, with the underlying cause logged rather than returned. pub fn internal(context: &str) -> Response { iceberg_error( diff --git a/pangolin/pangolin_api/src/iceberg/mod.rs b/pangolin/pangolin_api/src/iceberg/mod.rs index 1b93310..db57d9a 100644 --- a/pangolin/pangolin_api/src/iceberg/mod.rs +++ b/pangolin/pangolin_api/src/iceberg/mod.rs @@ -18,7 +18,10 @@ pub mod tables; pub mod types; // Re-export types for convenience -pub use error::iceberg_error; +pub use error::{ + bad_request, forbidden, iceberg_error, internal, namespace_already_exists, namespace_not_empty, + no_such_namespace, no_such_table, no_such_view, table_already_exists, +}; pub use types::*; pub type AppState = std::sync::Arc; diff --git a/pangolin/pangolin_api/src/iceberg/namespaces.rs b/pangolin/pangolin_api/src/iceberg/namespaces.rs index d2dc88f..770af8e 100644 --- a/pangolin/pangolin_api/src/iceberg/namespaces.rs +++ b/pangolin/pangolin_api/src/iceberg/namespaces.rs @@ -1,5 +1,8 @@ use super::types::*; -use super::{check_and_forward_if_federated, AppState}; +use super::{ + check_and_forward_if_federated, forbidden, internal, namespace_already_exists, + no_such_namespace, AppState, +}; use crate::auth::TenantId; use crate::authz::check_permission; use axum::{ @@ -37,7 +40,7 @@ pub async fn list_namespaces( Extension(session): Extension, Path(prefix): Path, Query(params): Query, - Query(pagination): Query, + Query(page): Query, ) -> impl IntoResponse { let tenant_id = tenant.0; let catalog_name = prefix.clone(); @@ -67,9 +70,10 @@ pub async fn list_namespaces( // Local catalog handling let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { Ok(Some(c)) => c, - Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "list_namespaces: failed to load catalog"); + return internal("Failed to load catalog"); } }; @@ -79,31 +83,193 @@ pub async fn list_namespaces( }; match check_permission(&store, &session, &Action::List, &scope).await { Ok(true) => (), - Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Ok(false) => return forbidden("Forbidden"), Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Permission check failed: {}", e), - ) - .into_response() + tracing::error!(error = %e, "list_namespaces: permission check failed"); + return internal("Permission check failed"); } } + let (offset, limit) = page.resolve(); + let pagination = PaginationParams { + limit: Some(limit as usize), + offset: Some(offset as usize), + }; + match store .list_namespaces(tenant_id, &catalog_name, params.parent, Some(pagination)) .await { Ok(namespaces) => { + let returned = namespaces.len(); let ns_list: Vec> = namespaces.into_iter().map(|n| n.name).collect(); ( StatusCode::OK, Json(ListNamespacesResponse { namespaces: ns_list, + next_page_token: next_page_token(returned, offset, limit), }), ) .into_response() } - Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(), + Err(e) => { + tracing::error!(error = %e, "list_namespaces: failed to list namespaces"); + internal("Failed to list namespaces") + } + } +} + +/// Load a namespace's metadata (`loadNamespaceMetadata`). +/// +/// Part of completing the Iceberg REST surface: this endpoint was on the +/// README's "not implemented" list, so clients could create a namespace and set +/// its properties but never read them back. +#[utoipa::path( + get, + path = "/v1/{prefix}/namespaces/{namespace}", + tag = "Iceberg REST", + params( + ("prefix" = String, Path, description = "Catalog name"), + ("namespace" = String, Path, description = "Namespace name") + ), + responses( + (status = 200, description = "Namespace metadata", body = CreateNamespaceResponse), + (status = 403, description = "Forbidden"), + (status = 404, description = "Namespace not found"), + (status = 500, description = "Internal server error") + ), + security(("bearer_auth" = [])) +)] +pub async fn load_namespace_metadata( + State(store): State, + Extension(tenant): Extension, + Extension(session): Extension, + Path((prefix, namespace)): Path<(String, String)>, +) -> impl IntoResponse { + let tenant_id = tenant.0; + let catalog_name = prefix; + + let path = format!("/namespaces/{}", namespace); + if let Some(response) = check_and_forward_if_federated( + &store, + tenant_id, + &catalog_name, + Method::GET, + &path, + None, + HeaderMap::new(), + ) + .await + { + return response; + } + + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "load_namespace_metadata: failed to load catalog"); + return internal("Failed to load catalog"); + } + }; + + let (namespace_parts, _branch) = parse_namespace(&namespace); + + let scope = PermissionScope::Namespace { + catalog_id: catalog.id, + namespace: namespace_parts.join("."), + }; + match check_permission(&store, &session, &Action::Read, &scope).await { + Ok(true) => (), + Ok(false) => return forbidden("Forbidden"), + Err(e) => { + tracing::error!(error = %e, "load_namespace_metadata: permission check failed"); + return internal("Permission check failed"); + } + } + + match store + .get_namespace(tenant_id, &catalog_name, namespace_parts.clone()) + .await + { + Ok(Some(ns)) => ( + StatusCode::OK, + Json(CreateNamespaceResponse { + namespace: ns.name, + properties: ns.properties, + }), + ) + .into_response(), + Ok(None) => no_such_namespace(&namespace_parts.join(".")), + Err(e) => { + tracing::error!(error = %e, "load_namespace_metadata: failed to load namespace"); + internal("Failed to load namespace") + } + } +} + +/// Check whether a namespace exists (`namespaceExists`). +/// +/// `HEAD` with an empty body, per the spec. Also on the README's +/// "not implemented" list. +#[utoipa::path( + head, + path = "/v1/{prefix}/namespaces/{namespace}", + tag = "Iceberg REST", + params( + ("prefix" = String, Path, description = "Catalog name"), + ("namespace" = String, Path, description = "Namespace name") + ), + responses( + (status = 204, description = "Namespace exists"), + (status = 403, description = "Forbidden"), + (status = 404, description = "Namespace not found") + ), + security(("bearer_auth" = [])) +)] +pub async fn namespace_exists( + State(store): State, + Extension(tenant): Extension, + Extension(session): Extension, + Path((prefix, namespace)): Path<(String, String)>, +) -> impl IntoResponse { + let tenant_id = tenant.0; + let catalog_name = prefix; + + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return StatusCode::NOT_FOUND.into_response(), + Err(e) => { + tracing::error!(error = %e, "namespace_exists: failed to load catalog"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + }; + + let (namespace_parts, _branch) = parse_namespace(&namespace); + + let scope = PermissionScope::Namespace { + catalog_id: catalog.id, + namespace: namespace_parts.join("."), + }; + match check_permission(&store, &session, &Action::Read, &scope).await { + Ok(true) => (), + Ok(false) => return StatusCode::FORBIDDEN.into_response(), + Err(e) => { + tracing::error!(error = %e, "namespace_exists: permission check failed"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + } + + match store + .get_namespace(tenant_id, &catalog_name, namespace_parts) + .await + { + Ok(Some(_)) => StatusCode::NO_CONTENT.into_response(), + Ok(None) => StatusCode::NOT_FOUND.into_response(), + Err(e) => { + tracing::error!(error = %e, "namespace_exists: failed to load namespace"); + StatusCode::INTERNAL_SERVER_ERROR.into_response() + } } } @@ -140,12 +306,34 @@ pub async fn create_namespace( catalog_name ); + // B16k: federated forwarding was missing here (and on delete and the + // namespace tree) although `list_namespaces` had it. On a `Federated` + // catalog, creating a namespace built a *local shadow* and returned 200 + // while `GET` listed the remote - the two views diverged permanently, and + // delete reported success for a namespace still present upstream. + let path = "/namespaces".to_string(); + let body_bytes = serde_json::to_vec(&payload).ok().map(Bytes::from); + if let Some(response) = check_and_forward_if_federated( + &store, + tenant_id, + &catalog_name, + Method::POST, + &path, + body_bytes, + HeaderMap::new(), + ) + .await + { + return response; + } + // Resolve catalog ID let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { Ok(Some(c)) => c, - Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "create_namespace: failed to load catalog"); + return internal("Failed to load catalog"); } }; @@ -155,13 +343,10 @@ pub async fn create_namespace( }; match check_permission(&store, &session, &Action::Create, &scope).await { Ok(true) => (), - Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Ok(false) => return forbidden("Forbidden"), Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Permission check failed: {}", e), - ) - .into_response() + tracing::error!(error = %e, "create_namespace: permission check failed"); + return internal("Permission check failed"); } } @@ -170,6 +355,21 @@ pub async fn create_namespace( properties: payload.properties.unwrap_or_default(), }; + // Report a conflict rather than silently overwriting an existing namespace's + // properties, which is what the create path did on backends whose insert is + // an upsert. + match store + .get_namespace(tenant_id, &catalog_name, ns.name.clone()) + .await + { + Ok(Some(_)) => return namespace_already_exists(&ns.name.join(".")), + Ok(None) => {} + Err(e) => { + tracing::error!(error = %e, "create_namespace: existence check failed"); + return internal("Failed to check namespace"); + } + } + match store .create_namespace(tenant_id, &catalog_name, ns.clone()) .await @@ -200,7 +400,10 @@ pub async fn create_namespace( ) .into_response() } - Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(), + Err(e) => { + tracing::error!(error = %e, "create_namespace: failed to create namespace"); + internal("Failed to create namespace") + } } } @@ -230,16 +433,33 @@ pub async fn delete_namespace( let tenant_id = tenant.0; let catalog_name = prefix; + // Federated forwarding (B16k) - see `create_namespace`. + let path = format!("/namespaces/{}", namespace); + if let Some(response) = check_and_forward_if_federated( + &store, + tenant_id, + &catalog_name, + Method::DELETE, + &path, + None, + HeaderMap::new(), + ) + .await + { + return response; + } + // Resolve catalog ID let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { Ok(Some(c)) => c, - Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "delete_namespace: failed to load catalog"); + return internal("Failed to load catalog"); } }; - let namespace_parts: Vec = namespace.split('\x1F').map(|s| s.to_string()).collect(); + let (namespace_parts, _branch) = parse_namespace(&namespace); // Check Permissions let scope = PermissionScope::Namespace { @@ -249,13 +469,10 @@ pub async fn delete_namespace( match check_permission(&store, &session, &Action::Delete, &scope).await { Ok(true) => (), - Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Ok(false) => return forbidden("Forbidden"), Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Permission check failed: {}", e), - ) - .into_response() + tracing::error!(error = %e, "delete_namespace: permission check failed"); + return internal("Permission check failed"); } } @@ -282,11 +499,25 @@ pub async fn delete_namespace( StatusCode::NO_CONTENT.into_response() } - Err(_) => (StatusCode::NOT_FOUND, "Namespace not found").into_response(), + Err(e) => { + tracing::debug!(error = %e, "delete_namespace: namespace not deleted"); + no_such_namespace(&namespace_parts.join(".")) + } } } -/// Update namespace properties +/// Update a namespace's properties. +/// +/// Two fixes here: +/// +/// * **B0d** - the handler bound `Extension(_session)` (deliberately discarding +/// it) and never called `check_permission`, and never resolved the catalog at +/// all. Any tenant member could rewrite any namespace's properties, including +/// `location`, which later table creation derives paths from. +/// * **B16h** - `removals` were silently ignored. A request carrying removals +/// got `200 OK` with `removed: []` / `missing: []` while nothing was removed: +/// exactly the "silent success" failure class 0.6.0 set out to eliminate. The +/// three response lists are now reported honestly. #[utoipa::path( post, path = "/v1/{prefix}/namespaces/{namespace}/properties", @@ -307,7 +538,7 @@ pub async fn delete_namespace( pub async fn update_namespace_properties( State(store): State, Extension(tenant): Extension, - Extension(_session): Extension, + Extension(session): Extension, Path((prefix, namespace)): Path<(String, String)>, Json(payload): Json, ) -> impl IntoResponse { @@ -331,26 +562,44 @@ pub async fn update_namespace_properties( { return response; } - let namespace_parts: Vec = namespace.split('\x1F').map(|s| s.to_string()).collect(); - - // For MVP, we only support updates. Removals are ignored or TODO. - if let Some(updates) = payload.updates { - match store - .update_namespace_properties(tenant_id, &catalog_name, namespace_parts, updates.clone()) - .await - { - Ok(_) => { - let response = UpdateNamespacePropertiesResponse { - updated: updates.keys().cloned().collect(), - removed: vec![], - missing: vec![], - }; - (StatusCode::OK, Json(response)).into_response() - } - Err(_) => (StatusCode::NOT_FOUND, "Namespace not found").into_response(), + + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "update_namespace_properties: failed to load catalog"); + return internal("Failed to load catalog"); + } + }; + + let (namespace_parts, _branch) = parse_namespace(&namespace); + + let scope = PermissionScope::Namespace { + catalog_id: catalog.id, + namespace: namespace_parts.join("."), + }; + match check_permission(&store, &session, &Action::Write, &scope).await { + Ok(true) => (), + Ok(false) => return forbidden("Forbidden"), + Err(e) => { + tracing::error!(error = %e, "update_namespace_properties: permission check failed"); + return internal("Permission check failed"); } - } else { - ( + } + + let updates = payload.updates.unwrap_or_default(); + let removals = payload.removals.unwrap_or_default(); + + // The spec rejects a key that appears in both lists rather than picking a + // winner. + if let Some(conflict) = removals.iter().find(|k| updates.contains_key(*k)) { + return super::bad_request(&format!( + "Property {conflict} appears in both updates and removals" + )); + } + + if updates.is_empty() && removals.is_empty() { + return ( StatusCode::OK, Json(UpdateNamespacePropertiesResponse { updated: vec![], @@ -358,7 +607,59 @@ pub async fn update_namespace_properties( missing: vec![], }), ) - .into_response() + .into_response(); + } + + // Read-modify-write: removals cannot be expressed by the merging store + // method, so the resulting map is computed here and written wholesale. + let existing = match store + .get_namespace(tenant_id, &catalog_name, namespace_parts.clone()) + .await + { + Ok(Some(ns)) => ns.properties, + Ok(None) => return no_such_namespace(&namespace_parts.join(".")), + Err(e) => { + tracing::error!(error = %e, "update_namespace_properties: failed to load namespace"); + return internal("Failed to load namespace"); + } + }; + + let mut properties = existing; + let mut removed = Vec::new(); + let mut missing = Vec::new(); + for key in &removals { + if properties.remove(key).is_some() { + removed.push(key.clone()); + } else { + missing.push(key.clone()); + } + } + + let updated: Vec = updates.keys().cloned().collect(); + properties.extend(updates); + + match store + .replace_namespace_properties( + tenant_id, + &catalog_name, + namespace_parts.clone(), + properties, + ) + .await + { + Ok(_) => ( + StatusCode::OK, + Json(UpdateNamespacePropertiesResponse { + updated, + removed, + missing, + }), + ) + .into_response(), + Err(e) => { + tracing::error!(error = %e, "update_namespace_properties: failed to write properties"); + no_such_namespace(&namespace_parts.join(".")) + } } } @@ -387,12 +688,29 @@ pub async fn list_namespaces_tree( let tenant_id = tenant.0; let catalog_name = prefix.clone(); + // Federated forwarding (B16k): without it the tree renders the local shadow + // while every other view shows the remote. + if let Some(response) = check_and_forward_if_federated( + &store, + tenant_id, + &catalog_name, + Method::GET, + "/namespaces", + None, + HeaderMap::new(), + ) + .await + { + return response; + } + // Resolve catalog ID let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { Ok(Some(c)) => c, - Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "list_namespaces_tree: failed to load catalog"); + return internal("Failed to load catalog"); } }; @@ -402,13 +720,10 @@ pub async fn list_namespaces_tree( }; match check_permission(&store, &session, &Action::List, &scope).await { Ok(true) => (), - Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Ok(false) => return forbidden("Forbidden"), Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Permission check failed: {}", e), - ) - .into_response() + tracing::error!(error = %e, "list_namespaces_tree: permission check failed"); + return internal("Permission check failed"); } } @@ -455,10 +770,9 @@ pub async fn list_namespaces_tree( ) .into_response() } - Err(e) => ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to list namespaces: {}", e), - ) - .into_response(), + Err(e) => { + tracing::error!(error = %e, "list_namespaces_tree: failed to list namespaces"); + internal("Failed to list namespaces") + } } } diff --git a/pangolin/pangolin_api/src/iceberg/oauth.rs b/pangolin/pangolin_api/src/iceberg/oauth.rs index 9ff80c2..1998520 100644 --- a/pangolin/pangolin_api/src/iceberg/oauth.rs +++ b/pangolin/pangolin_api/src/iceberg/oauth.rs @@ -18,6 +18,7 @@ use crate::iceberg::AppState; use utoipa::ToSchema; #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct OAuthTokenRequest { #[schema(example = "client_credentials")] grant_type: String, @@ -86,9 +87,15 @@ pub async fn handle_oauth_token( .map_err(ApiError::InternalError)? .ok_or_else(|| ApiError::unauthorized("Invalid client_id"))?; - // 4. Verify Active Status - if !service_user.active { - return Err(ApiError::unauthorized("Client is inactive")); + // 4. Verify Active Status *and* expiry. + // + // B0g: this checked only `active`, not `is_valid()` (= `active && + // !is_expired()`). The API-key path in `auth_middleware` uses `is_valid()` + // correctly, so an *expired* service user was refused there but could still + // exchange `client_credentials` here for a fresh 1-hour JWT - fully + // bypassing key expiry, and renewably. + if !service_user.is_valid() { + return Err(ApiError::unauthorized("Client is inactive or expired")); } // 5. Verify Secret (API Key) diff --git a/pangolin/pangolin_api/src/iceberg/tables.rs b/pangolin/pangolin_api/src/iceberg/tables.rs index 4c1bfdd..b327043 100644 --- a/pangolin/pangolin_api/src/iceberg/tables.rs +++ b/pangolin/pangolin_api/src/iceberg/tables.rs @@ -1,5 +1,8 @@ use super::types::*; -use super::{check_and_forward_if_federated, commit, iceberg_error, AppState}; +use super::{ + bad_request, check_and_forward_if_federated, commit, forbidden, iceberg_error, internal, + no_such_namespace, no_such_table, table_already_exists, AppState, +}; use crate::auth::TenantId; use crate::authz::check_permission; use axum::{ @@ -11,7 +14,7 @@ use axum::{ use bytes::Bytes; use chrono::Utc; use pangolin_core::iceberg_metadata::{ - NestedField, PartitionSpec, Schema, SortOrder, TableMetadata, Type, + MetadataLogEntry, PartitionSpec, Schema, SortOrder, TableMetadata, }; use pangolin_core::model::{Asset, AssetType}; use pangolin_core::permission::{Action, PermissionScope}; @@ -21,6 +24,11 @@ use std::collections::HashMap; use std::sync::Arc; use uuid::Uuid; +/// How many previous metadata files to keep in `metadata-log` when the table +/// does not set `write.metadata.previous-versions-max`. Matches the Iceberg +/// default. +const DEFAULT_PREVIOUS_VERSIONS_MAX: usize = 100; + /// List tables in a namespace #[utoipa::path( get, @@ -43,7 +51,7 @@ pub async fn list_tables( Extension(tenant): Extension, Extension(session): Extension, Path((prefix, namespace)): Path<(String, String)>, - Query(pagination): Query, + Query(page): Query, ) -> impl IntoResponse { let tenant_id = tenant.0; let catalog_name = prefix.clone(); @@ -66,32 +74,34 @@ pub async fn list_tables( let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { Ok(Some(c)) => c, - Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "list_tables: failed to load catalog"); + return internal("Failed to load catalog"); } }; - let (ns_name, branch) = parse_table_identifier(&namespace); + let (ns_vec, branch) = parse_namespace(&namespace); // Check Permissions let scope = PermissionScope::Namespace { catalog_id: catalog.id, - namespace: ns_name.clone(), + namespace: ns_vec.join("."), }; match check_permission(&store, &session, &Action::List, &scope).await { Ok(true) => (), - Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Ok(false) => return forbidden("Forbidden"), Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Permission check failed: {}", e), - ) - .into_response() + tracing::error!(error = %e, "list_tables: permission check failed"); + return internal("Permission check failed"); } } - let ns_vec = vec![ns_name]; + let (offset, limit) = page.resolve(); + let pagination = PaginationParams { + limit: Some(limit as usize), + offset: Some(offset as usize), + }; match store .list_assets( @@ -104,6 +114,11 @@ pub async fn list_tables( .await { Ok(assets) => { + // The page token has to be computed from the number of rows the + // store returned, before the asset-type filter narrows it - the + // store is what applied `limit`, so a full page of rows means there + // may be more even if none of them survive the filter. + let returned = assets.len(); let identifiers: Vec = assets .into_iter() .filter(|a| a.kind == AssetType::IcebergTable) @@ -112,13 +127,24 @@ pub async fn list_tables( name: a.name, }) .collect(); - (StatusCode::OK, Json(ListTablesResponse { identifiers })).into_response() + ( + StatusCode::OK, + Json(ListTablesResponse { + identifiers, + next_page_token: next_page_token(returned, offset, limit), + }), + ) + .into_response() + } + Err(e) => { + tracing::error!(error = %e, "list_tables: failed to list assets"); + internal("Failed to list tables") } - Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(), } } #[derive(Debug, serde::Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] pub struct MaintenanceRequest { pub job_type: String, // "expire_snapshots" or "remove_orphan_files" pub retention_ms: Option, @@ -126,6 +152,13 @@ pub struct MaintenanceRequest { } /// Perform maintenance on a table +/// +/// Two bugs fixed here (B0f): +/// 1. the catalog from the path was discarded and the literal `"default"` was +/// passed to `expire_snapshots`/`remove_orphan_files`, so destructive +/// maintenance ran against the wrong catalog entirely; +/// 2. there was no session and no permission check, so any tenant member could +/// trigger snapshot expiry and orphan-file deletion on any table. #[utoipa::path( post, path = "/api/v1/catalogs/{prefix}/namespaces/{namespace}/tables/{table}/maintenance", @@ -139,6 +172,8 @@ pub struct MaintenanceRequest { responses( (status = 200, description = "Maintenance accepted", body = serde_json::Value), (status = 400, description = "Bad request"), + (status = 403, description = "Forbidden"), + (status = 404, description = "Catalog or table not found"), (status = 500, description = "Internal server error") ), security(("bearer_auth" = [])) @@ -146,24 +181,67 @@ pub struct MaintenanceRequest { pub async fn perform_maintenance( State(store): State>, Extension(tenant_id): Extension, - Path((_prefix, namespace, table)): Path<(String, String, String)>, + Extension(session): Extension, + Path((prefix, namespace, table)): Path<(String, String, String)>, Json(payload): Json, ) -> Result, StatusCode> { - let namespace_parts: Vec = namespace.split('\x1F').map(|s| s.to_string()).collect(); - // Parse table@branch - let (table_name, branch_name) = if let Some((t, b)) = table.split_once('@') { - (t.to_string(), Some(b.to_string())) - } else { - (table.to_string(), None) + let tenant = tenant_id.0; + let catalog_name = prefix; + + let (namespace_parts, branch_from_ns) = parse_namespace(&namespace); + let (table_name, branch_from_table) = parse_table_identifier(&table); + let branch_name = branch_from_table.or(branch_from_ns); + + let catalog = match store.get_catalog(tenant, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return Err(StatusCode::NOT_FOUND), + Err(e) => { + tracing::error!(error = %e, "maintenance: failed to load catalog"); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } }; + let asset = match store + .get_asset( + tenant, + &catalog_name, + branch_name.clone(), + namespace_parts.clone(), + table_name.clone(), + ) + .await + { + Ok(Some(a)) => a, + Ok(None) => return Err(StatusCode::NOT_FOUND), + Err(e) => { + tracing::error!(error = %e, "maintenance: failed to load asset"); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } + }; + + // Snapshot expiry and orphan-file removal both destroy data, so they need + // Delete, not merely Write. + let scope = PermissionScope::Asset { + catalog_id: catalog.id, + namespace: namespace_parts.join("."), + asset_id: asset.id, + }; + match check_permission(&store, &session, &Action::Delete, &scope).await { + Ok(true) => (), + Ok(false) => return Err(StatusCode::FORBIDDEN), + Err(e) => { + tracing::error!(error = %e, "maintenance: permission check failed"); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } + } + match payload.job_type.as_str() { "expire_snapshots" => { let retention = payload.retention_ms.unwrap_or(86400000); // Default 1 day store .expire_snapshots( - tenant_id.0, - "default", + tenant, + &catalog_name, branch_name, namespace_parts, table_name, @@ -179,8 +257,8 @@ pub async fn perform_maintenance( let older_than = payload.older_than_ms.unwrap_or(86400000); // Default 1 day store .remove_orphan_files( - tenant_id.0, - "default", + tenant, + &catalog_name, branch_name, namespace_parts, table_name, @@ -256,16 +334,18 @@ pub async fn create_table( let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { Ok(Some(c)) => c, - Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "create_table: failed to load catalog"); + return internal("Failed to load catalog"); } }; let (tbl_name, branch_from_name) = parse_table_identifier(&payload.name); - let (ns_name, branch_from_ns) = parse_table_identifier(&namespace); + let (ns_vec, branch_from_ns) = parse_namespace(&namespace); let branch_from_query = params.get("branch").cloned(); let branch = branch_from_name.or(branch_from_ns).or(branch_from_query); + let ns_name = ns_vec.join("."); let scope = PermissionScope::Namespace { catalog_id: catalog.id, @@ -273,18 +353,13 @@ pub async fn create_table( }; match check_permission(&store, &session, &Action::Create, &scope).await { Ok(true) => (), - Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Ok(false) => return forbidden("Forbidden"), Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Permission check failed: {}", e), - ) - .into_response() + tracing::error!(error = %e, "create_table: permission check failed"); + return internal("Permission check failed"); } } - let ns_vec = vec![ns_name.clone()]; - let table_uuid = Uuid::new_v4(); let location = if let Some(loc) = payload.location { loc @@ -328,66 +403,57 @@ pub async fn create_table( ) }; - let schema_fields = if let Some(schema_value) = &payload.schema { - if let Some(fields) = schema_value.get("fields").and_then(|f| f.as_array()) { - fields - .iter() - .filter_map(|field| { - let id = field.get("id")?.as_i64()? as i32; - let name = field.get("name")?.as_str()?.to_string(); - let required = false; - let field_type_str = field.get("type")?.as_str()?; - - let field_type = match field_type_str { - "int" | "integer" => Type::Primitive("long".to_string()), - "long" => Type::Primitive("long".to_string()), - "string" => Type::Primitive("string".to_string()), - "boolean" => Type::Primitive("boolean".to_string()), - "float" => Type::Primitive("float".to_string()), - "double" => Type::Primitive("double".to_string()), - "date" => Type::Primitive("date".to_string()), - "time" => Type::Primitive("time".to_string()), - "timestamp" => Type::Primitive("timestamp".to_string()), - "timestamptz" => Type::Primitive("timestamptz".to_string()), - "binary" => Type::Primitive("binary".to_string()), - "uuid" => Type::Primitive("uuid".to_string()), - _ => Type::Primitive(field_type_str.to_string()), - }; - - Some(NestedField { - id, - name, - required, - field_type, - doc: None, - }) - }) - .collect() - } else { - vec![] - } - } else { - vec![] + // B16f: the schema used to be hand-parsed field by field, which silently + // lost data three ways - `required` was hardcoded to `false` so every column + // became optional, `int` was widened to `long`, and `field.get("type")? + // .as_str()?` inside a `filter_map` returned `None` for any struct/list/map/ + // decimal/fixed column, so those columns were *dropped* and `last_column_id` + // was computed from the survivors. The table was created `200 OK` with a + // schema missing columns. + // + // Deserializing straight into the core `Schema` (as the commit path already + // does) keeps nullability and complex types, and a malformed schema is now a + // 400 rather than a quietly mangled table. + let schema = match &payload.schema { + Some(schema_value) => match serde_json::from_value::(schema_value.clone()) { + Ok(mut s) => { + s.schema_id = 0; + if s.identifier_field_ids.is_none() { + s.identifier_field_ids = Some(vec![]); + } + s + } + Err(e) => { + return bad_request(&format!("Invalid schema: {}", e)); + } + }, + None => Schema { + type_: Schema::STRUCT.to_string(), + schema_id: 0, + identifier_field_ids: Some(vec![]), + fields: vec![], + }, }; + let last_column_id = schema.max_field_id(); + let metadata = TableMetadata { format_version: 2, table_uuid, location: location.clone(), last_sequence_number: 0, last_updated_ms: Utc::now().timestamp_millis(), - last_column_id: schema_fields.iter().map(|f| f.id).max().unwrap_or(0), - schemas: vec![Schema { - schema_id: 0, - identifier_field_ids: Some(vec![]), - fields: schema_fields, - }], + last_column_id, + schemas: vec![schema], current_schema_id: 0, current_partition_spec_id: 0, partition_specs: vec![PartitionSpec { spec_id: 0, fields: vec![], }], + // Required by the v2 spec; an empty unpartitioned spec assigns nothing, + // so the highest assigned partition id is the "unpartitioned" sentinel. + last_partition_id: pangolin_core::iceberg_metadata::PARTITION_FIELD_ID_START - 1, default_sort_order_id: 0, sort_orders: vec![SortOrder { order_id: 0, @@ -401,7 +467,13 @@ pub async fn create_table( refs: None, }; - let metadata_json = serde_json::to_string(&metadata).unwrap(); + let metadata_json = match serde_json::to_string(&metadata) { + Ok(json) => json, + Err(e) => { + tracing::error!(error = %e, "create_table: failed to serialize metadata"); + return internal("Failed to serialize table metadata"); + } + }; let metadata_location = format!( "{}/metadata/00000-{}.metadata.json", location, @@ -423,23 +495,30 @@ pub async fn create_table( }, }; + // B16g: write the metadata file *first*, then register the asset. The old + // order registered the asset and only then wrote the file, so a failed write + // left a permanently broken table - registered, pointing at a file that does + // not exist, with `load_table` 500ing and `update_table` 404ing and no repair + // path. This is also the order the commit path already uses. + if let Err(e) = store + .write_file(&metadata_location, metadata_json.into_bytes()) + .await + { + tracing::error!("Failed to write metadata file: {}", e); + return internal("Failed to write metadata"); + } + match store - .create_asset(tenant_id, &catalog_name, branch, ns_vec, asset.clone()) + .create_asset( + tenant_id, + &catalog_name, + branch, + ns_vec.clone(), + asset.clone(), + ) .await { Ok(_) => { - if let Err(e) = store - .write_file(&metadata_location, metadata_json.into_bytes()) - .await - { - tracing::error!("Failed to write metadata file: {}", e); - return ( - StatusCode::INTERNAL_SERVER_ERROR, - "Failed to write metadata", - ) - .into_response(); - } - let _ = store .log_audit_event( tenant_id, @@ -493,7 +572,12 @@ pub async fn create_table( ( StatusCode::OK, Json(TableResponse::with_credentials( - Some(location.clone()), + // B16e: this returned `location` - the table *directory* - + // where the spec (and `load_table`, correctly) return the + // metadata *file*. A client that keeps the returned `Table` + // (PyIceberg does) ended up with a `metadata_location` it + // could neither read nor refresh from. + Some(metadata_location.clone()), metadata, credentials, Some(table_uuid), @@ -501,7 +585,20 @@ pub async fn create_table( ) .into_response() } - Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(), + Err(e) => { + tracing::error!(error = %e, "create_table: failed to register asset"); + // The metadata file was written before registration (B16g); with the + // asset unregistered it is unreferenced, so clean it up rather than + // leaving an orphan behind. + if let Err(cleanup) = store.delete_file(&metadata_location).await { + tracing::warn!( + error = %cleanup, + location = %metadata_location, + "could not remove the orphaned metadata file after a failed create_asset" + ); + } + internal("Failed to create table") + } } } @@ -560,18 +657,19 @@ pub async fn load_table( let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { Ok(Some(c)) => c, - Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "load_table: failed to load catalog"); + return internal("Failed to load catalog"); } }; + // B16a: shared namespace parsing, so a nested namespace resolves the same + // way here as it does on the commit path. let (tbl_name, branch_from_name) = parse_table_identifier(&table); - let (ns_name, branch_from_ns) = parse_table_identifier(&namespace); + let (ns_vec, branch_from_ns) = parse_namespace(&namespace); let branch = branch_from_name.or(branch_from_ns); - let ns_vec = vec![ns_name]; - let asset = match store .get_asset( tenant_id, @@ -583,9 +681,10 @@ pub async fn load_table( .await { Ok(Some(a)) => a, - Ok(None) => return (StatusCode::NOT_FOUND, "Table not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_table(&format!("{}.{}", ns_vec.join("."), tbl_name)), + Err(e) => { + tracing::error!(error = %e, "load_table: failed to load asset"); + return internal("Failed to load table"); } }; @@ -597,13 +696,10 @@ pub async fn load_table( match check_permission(&store, &session, &Action::Read, &scope).await { Ok(true) => (), - Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Ok(false) => return forbidden("Forbidden"), Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Permission check failed: {}", e), - ) - .into_response() + tracing::error!(error = %e, "load_table: permission check failed"); + return internal("Permission check failed"); } } @@ -612,12 +708,9 @@ pub async fn load_table( if let Some(location) = current_metadata_location { let metadata_bytes = match store.read_file(&location).await { Ok(bytes) => bytes, - Err(_) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - "Failed to read metadata file", - ) - .into_response() + Err(e) => { + tracing::error!(error = %e, "load_table: failed to read metadata file"); + return internal("Failed to read metadata file"); } }; @@ -629,15 +722,13 @@ pub async fn load_table( .await { Ok(Ok(m)) => m, - Ok(Err(_)) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - "Failed to parse metadata", - ) - .into_response() + Ok(Err(e)) => { + tracing::error!(error = %e, "update_table: failed to parse metadata"); + return internal("Failed to parse metadata"); } - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Task join error").into_response() + Err(e) => { + tracing::error!(error = %e, "update_table: metadata parse task panicked"); + return internal("Failed to parse metadata"); } }; @@ -683,7 +774,7 @@ pub async fn load_table( ) .into_response() } else { - (StatusCode::NOT_FOUND, "Metadata location not found").into_response() + no_such_table(&format!("{}.{}", ns_vec.join("."), tbl_name)) } } @@ -736,15 +827,23 @@ pub async fn update_table( let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { Ok(Some(c)) => c, - Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "update_table: failed to load catalog"); + return internal("Failed to load catalog"); } }; + // B16a: this used to split the namespace on 0x1F while `create_table` and + // `load_table` went through `parse_table_identifier` (a *single*-element + // namespace). A table created in namespace `a\x1Fb` was registered under + // `["a\x1Fb"]` and looked up here under `["a", "b"]`, so every commit to a + // nested namespace 404'd and the CAS loop below never ran at all. let (table_name, branch_from_name) = parse_table_identifier(&table); - let branch = branch_from_name.unwrap_or("main".to_string()); - let namespace_parts: Vec = namespace.split('\x1F').map(|s| s.to_string()).collect(); + let (namespace_parts, branch_from_ns) = parse_namespace(&namespace); + let branch = branch_from_name + .or(branch_from_ns) + .unwrap_or_else(|| "main".to_string()); let asset = match store .get_asset( @@ -757,9 +856,10 @@ pub async fn update_table( .await { Ok(Some(a)) => a, - Ok(None) => return (StatusCode::NOT_FOUND, "Table not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_table(&format!("{}.{}", namespace_parts.join("."), table_name)), + Err(e) => { + tracing::error!(error = %e, "update_table: failed to load asset"); + return internal("Failed to load table"); } }; @@ -771,13 +871,10 @@ pub async fn update_table( match check_permission(&store, &session, &Action::Write, &scope).await { Ok(true) => (), - Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Ok(false) => return forbidden("Forbidden"), Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Permission check failed: {}", e), - ) - .into_response() + tracing::error!(error = %e, "update_table: permission check failed"); + return internal("Permission check failed"); } } @@ -798,9 +895,12 @@ pub async fn update_table( .await { Ok(Some(a)) => a, - Ok(None) => return (StatusCode::NOT_FOUND, "Table not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => { + return no_such_table(&format!("{}.{}", namespace_parts.join("."), table_name)) + } + Err(e) => { + tracing::error!(error = %e, "update_table: failed to re-read asset"); + return internal("Failed to load table"); } }; @@ -809,16 +909,13 @@ pub async fn update_table( let metadata_bytes = if let Some(loc) = ¤t_metadata_location { match store.read_file(loc).await { Ok(bytes) => bytes, - Err(_) => { - return (StatusCode::NOT_FOUND, "Failed to read metadata file").into_response() + Err(e) => { + tracing::error!(error = %e, "update_table: failed to read metadata file"); + return internal("Failed to read metadata file"); } } } else { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - "Table corrupted (no metadata)", - ) - .into_response(); + return internal("Table corrupted (no metadata location)"); }; // Parse metadata in a blocking task to avoid stalling the executor @@ -829,15 +926,13 @@ pub async fn update_table( .await { Ok(Ok(m)) => m, - Ok(Err(_)) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - "Failed to parse metadata", - ) - .into_response() + Ok(Err(e)) => { + tracing::error!(error = %e, "load_table: failed to parse metadata"); + return internal("Failed to parse metadata"); } - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Task join error").into_response() + Err(e) => { + tracing::error!(error = %e, "load_table: metadata parse task panicked"); + return internal("Failed to parse metadata"); } }; @@ -854,6 +949,36 @@ pub async fn update_table( return commit_error_response(e); } + // B16b: `last-updated-ms` was only ever assigned inside `add_snapshot`, + // so a commit of only `set-properties` / `add-schema` / `set-location` / + // `add-spec` / `set-snapshot-ref` / `remove-snapshots` published a new + // metadata file carrying an *unchanged* timestamp - and any consumer + // that orders or dedupes metadata by that field treated the two versions + // as identical. Every successful set of updates bumps it. + metadata.last_updated_ms = Utc::now().timestamp_millis(); + + // B13: record the metadata file this one supersedes. `metadata-log` was + // initialised to an empty vec at table creation and never appended to, + // so metadata time-travel and previous-version cleanup + // (`write.metadata.previous-versions-max`) had nothing to work with. + if let Some(previous) = ¤t_metadata_location { + let log = metadata.metadata_log.get_or_insert_with(Vec::new); + log.push(MetadataLogEntry { + timestamp_ms: metadata.last_updated_ms, + metadata_file: previous.clone(), + }); + let max_entries = metadata + .properties + .as_ref() + .and_then(|p| p.get("write.metadata.previous-versions-max")) + .and_then(|v| v.parse::().ok()) + .unwrap_or(DEFAULT_PREVIOUS_VERSIONS_MAX); + if log.len() > max_entries { + let excess = log.len() - max_entries; + log.drain(0..excess); + } + } + let new_metadata_location = format!( "{}/metadata/00000-{}.metadata.json", metadata.location, @@ -863,23 +988,16 @@ pub async fn update_table( Ok(json) => json, Err(e) => { tracing::error!(error = %e, "could not serialise table metadata"); - return ( - StatusCode::INTERNAL_SERVER_ERROR, - "Failed to serialise table metadata", - ) - .into_response(); + return internal("Failed to serialise table metadata"); } }; - if let Err(_) = store + if store .write_file(&new_metadata_location, metadata_json.into_bytes()) .await + .is_err() { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - "Failed to write new metadata", - ) - .into_response(); + return internal("Failed to write new metadata"); } match store @@ -938,6 +1056,19 @@ pub async fn update_table( // Re-read and re-check requirements on the next pass. tracing::debug!(error = %e, attempt = retries, "metadata CAS lost, retrying"); crate::metrics::inc(&crate::metrics::COMMIT_CAS_RETRIES); + // B16d: the metadata file was written *before* the CAS, so on a + // lost CAS it is unreferenced - and the old code just + // `continue`d, orphaning it. Under contention that leaked up to + // one file per retry, with up to five left behind on a final + // give-up, and an orphan is indistinguishable from live metadata + // from the outside so nothing could reap it later. + if let Err(cleanup) = store.delete_file(&new_metadata_location).await { + tracing::warn!( + error = %cleanup, + location = %new_metadata_location, + "could not remove the metadata file orphaned by a lost CAS" + ); + } retries += 1; continue; } @@ -974,6 +1105,13 @@ fn commit_error_response(error: commit::CommitError) -> axum::response::Response } /// Rename a table +/// +/// B0c: this handler bound `Extension(session)` but never called +/// `check_permission` - the only table handler that didn't. Any tenant member +/// could move any table into any namespace: an effective delete (the table +/// vanishes from where its readers look for it) and a way to smuggle a table +/// into a namespace where the caller *does* have read rights. It now needs +/// `Write` on the source table and `Create` on the destination namespace. #[utoipa::path( post, path = "/v1/{prefix}/tables/rename", @@ -984,6 +1122,7 @@ fn commit_error_response(error: commit::CommitError) -> axum::response::Response request_body = RenameTableRequest, responses( (status = 204, description = "Table renamed"), + (status = 403, description = "Forbidden"), (status = 404, description = "Source table not found"), (status = 500, description = "Internal server error") ), @@ -1022,6 +1161,85 @@ pub async fn rename_table( let dest_name = payload.destination.name; let branch = Some("main".to_string()); + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "rename_table: failed to load catalog"); + return internal("Failed to load catalog"); + } + }; + + let source_asset = match store + .get_asset( + tenant_id, + &catalog_name, + branch.clone(), + source_ns.clone(), + source_name.clone(), + ) + .await + { + Ok(Some(a)) => a, + Ok(None) => return no_such_table(&format!("{}.{}", source_ns.join("."), source_name)), + Err(e) => { + tracing::error!(error = %e, "rename_table: failed to load source asset"); + return internal("Failed to load source table"); + } + }; + + // Write on the source: renaming is a mutation of the table's identity, and + // from every reader's point of view it is a delete at the old path. + let source_scope = PermissionScope::Asset { + catalog_id: catalog.id, + namespace: source_ns.join("."), + asset_id: source_asset.id, + }; + match check_permission(&store, &session, &Action::Write, &source_scope).await { + Ok(true) => (), + Ok(false) => return forbidden("Forbidden: no write access to the source table"), + Err(e) => { + tracing::error!(error = %e, "rename_table: source permission check failed"); + return internal("Permission check failed"); + } + } + + // Create on the destination namespace: otherwise a caller with write on one + // table could plant it anywhere they can read. + let dest_scope = PermissionScope::Namespace { + catalog_id: catalog.id, + namespace: dest_ns.join("."), + }; + match check_permission(&store, &session, &Action::Create, &dest_scope).await { + Ok(true) => (), + Ok(false) => return forbidden("Forbidden: no create access in the destination namespace"), + Err(e) => { + tracing::error!(error = %e, "rename_table: destination permission check failed"); + return internal("Permission check failed"); + } + } + + // The spec returns 409 rather than clobbering an existing destination. + match store + .get_asset( + tenant_id, + &catalog_name, + branch.clone(), + dest_ns.clone(), + dest_name.clone(), + ) + .await + { + Ok(Some(_)) => { + return table_already_exists(&format!("{}.{}", dest_ns.join("."), dest_name)) + } + Ok(None) => {} + Err(e) => { + tracing::error!(error = %e, "rename_table: destination existence check failed"); + return internal("Failed to check the destination table"); + } + } + match store .rename_asset( tenant_id, @@ -1044,7 +1262,7 @@ pub async fn rename_table( session.username.clone(), pangolin_core::audit::AuditAction::RenameTable, pangolin_core::audit::ResourceType::Table, - None, // Cannot determine asset ID easily without lookup + Some(source_asset.id), format!( "{}/{}.{} -> {}.{}", catalog_name, @@ -1059,7 +1277,10 @@ pub async fn rename_table( StatusCode::NO_CONTENT.into_response() } - Err(_) => (StatusCode::NOT_FOUND, "Table not found").into_response(), + Err(e) => { + tracing::error!(error = %e, "rename_table: rename failed"); + no_such_table(&format!("{}.{}", source_ns.join("."), source_name)) + } } } @@ -1107,15 +1328,18 @@ pub async fn delete_table( let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { Ok(Some(c)) => c, - Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal server error").into_response() + Ok(None) => return no_such_namespace(&catalog_name), + Err(e) => { + tracing::error!(error = %e, "delete_table: failed to load catalog"); + return internal("Failed to load catalog"); } }; let (table_name, branch_from_name) = parse_table_identifier(&table); - let branch = branch_from_name.or(Some("main".to_string())); - let namespace_parts: Vec = namespace.split('\x1F').map(|s| s.to_string()).collect(); + let (namespace_parts, branch_from_ns) = parse_namespace(&namespace); + let branch = branch_from_name + .or(branch_from_ns) + .or(Some("main".to_string())); let asset = match store .get_asset( @@ -1128,9 +1352,10 @@ pub async fn delete_table( .await { Ok(Some(a)) => a, - Ok(None) => return (StatusCode::NOT_FOUND, "Table not found").into_response(), - Err(_) => { - return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response() + Ok(None) => return no_such_table(&format!("{}.{}", namespace_parts.join("."), table_name)), + Err(e) => { + tracing::error!(error = %e, "delete_table: failed to load asset"); + return internal("Failed to load table"); } }; @@ -1142,13 +1367,10 @@ pub async fn delete_table( match check_permission(&store, &session, &Action::Delete, &scope).await { Ok(true) => (), - Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Ok(false) => return forbidden("Forbidden"), Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Permission check failed: {}", e), - ) - .into_response() + tracing::error!(error = %e, "delete_table: permission check failed"); + return internal("Permission check failed"); } } @@ -1172,7 +1394,7 @@ pub async fn delete_table( session.username.clone(), pangolin_core::audit::AuditAction::DropTable, pangolin_core::audit::ResourceType::Table, - None, + Some(asset.id), format!("{}/{}/{}", catalog_name, namespace, table), ), ) @@ -1180,7 +1402,10 @@ pub async fn delete_table( StatusCode::NO_CONTENT.into_response() } - Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(), + Err(e) => { + tracing::error!(error = %e, "delete_table: delete failed"); + internal("Failed to delete table") + } } } @@ -1203,6 +1428,7 @@ pub async fn delete_table( pub async fn table_exists( State(store): State, Extension(tenant): Extension, + Extension(session): Extension, Path((prefix, namespace, table)): Path<(String, String, String)>, ) -> impl IntoResponse { let tenant_id = tenant.0; @@ -1223,26 +1449,52 @@ pub async fn table_exists( return response; } - let namespace_parts: Vec = namespace.split('\x1F').map(|s| s.to_string()).collect(); - let (table_name, branch_name) = if let Some((t, b)) = table.split_once('@') { - (t.to_string(), Some(b.to_string())) - } else { - (table.to_string(), None) + let (namespace_parts, branch_from_ns) = parse_namespace(&namespace); + let (table_name, branch_from_table) = parse_table_identifier(&table); + let branch_name = branch_from_table.or(branch_from_ns); + + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return StatusCode::NOT_FOUND.into_response(), + Err(e) => { + tracing::error!(error = %e, "table_exists: failed to load catalog"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } }; - match store + let asset = match store .get_asset( tenant_id, &catalog_name, branch_name, - namespace_parts, + namespace_parts.clone(), table_name, ) .await { - Ok(Some(_)) => StatusCode::OK.into_response(), - Ok(None) => StatusCode::NOT_FOUND.into_response(), - Err(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), + Ok(Some(a)) => a, + Ok(None) => return StatusCode::NOT_FOUND.into_response(), + Err(e) => { + tracing::error!(error = %e, "table_exists: failed to load asset"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + }; + + // Existence is information. Without a check this endpoint is an oracle that + // reports whether a table the caller cannot read exists - every sibling + // handler gates on Read, so this one does too. + let scope = PermissionScope::Asset { + catalog_id: catalog.id, + namespace: namespace_parts.join("."), + asset_id: asset.id, + }; + match check_permission(&store, &session, &Action::Read, &scope).await { + Ok(true) => StatusCode::OK.into_response(), + Ok(false) => StatusCode::NOT_FOUND.into_response(), + Err(e) => { + tracing::error!(error = %e, "table_exists: permission check failed"); + StatusCode::INTERNAL_SERVER_ERROR.into_response() + } } } diff --git a/pangolin/pangolin_api/src/iceberg/types.rs b/pangolin/pangolin_api/src/iceberg/types.rs index d792784..fba94c8 100644 --- a/pangolin/pangolin_api/src/iceberg/types.rs +++ b/pangolin/pangolin_api/src/iceberg/types.rs @@ -12,6 +12,9 @@ pub struct CatalogConfig { #[derive(Serialize, ToSchema)] pub struct ListNamespacesResponse { pub namespaces: Vec>, + /// Continuation token; absent on the final page (B16i). + #[serde(rename = "next-page-token", skip_serializing_if = "Option::is_none")] + pub next_page_token: Option, } #[derive(Deserialize, IntoParams)] @@ -32,6 +35,7 @@ pub struct ListNamespacesTreeResponse { } #[derive(Serialize, Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CreateNamespaceRequest { pub namespace: Vec, pub properties: Option>, @@ -44,6 +48,7 @@ pub struct CreateNamespaceResponse { } #[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CreateTableRequest { pub name: String, pub location: Option, @@ -107,6 +112,13 @@ impl TableResponse { #[derive(Serialize, Deserialize, ToSchema)] pub struct ListTablesResponse { pub identifiers: Vec, + /// Continuation token; absent on the final page (B16i). + #[serde( + rename = "next-page-token", + skip_serializing_if = "Option::is_none", + default + )] + pub next_page_token: Option, } #[derive(Serialize, Deserialize, ToSchema)] @@ -124,6 +136,7 @@ pub struct PartitionField { } #[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CommitTableRequest { pub identifier: Option, pub requirements: Vec, @@ -258,13 +271,117 @@ pub fn parse_table_identifier(identifier: &str) -> (String, Option) { } } +/// The unit separator the Iceberg REST spec uses to encode a multi-level +/// namespace inside a single path segment. +pub const NAMESPACE_SEPARATOR: char = '\u{1F}'; + +/// Parse a namespace path segment into its levels plus an optional branch. +/// +/// This is the single parser for namespace path segments (B16a). Handlers used +/// to disagree: `list_tables`/`create_table`/`load_table` went through +/// [`parse_table_identifier`], which yields a *single-element* namespace, while +/// `update_table`/`delete_table`/`table_exists` split on `0x1F` and yielded the +/// real multi-element path. So a table created in namespace `a\x1Fb` was +/// registered under `["a\x1Fb"]` but looked up under `["a", "b"]` on commit - +/// a guaranteed `404 Table not found`, with the CAS loop never running. +/// +/// The `@branch` suffix is stripped first so a `ns@branch` form works +/// everywhere, not just on the handlers that happened to call +/// `parse_table_identifier`. +pub fn parse_namespace(namespace: &str) -> (Vec, Option) { + let (path, branch) = match namespace.split_once('@') { + Some((path, branch)) if !branch.is_empty() => (path, Some(branch.to_string())), + _ => (namespace, None), + }; + + let levels: Vec = path + .split(NAMESPACE_SEPARATOR) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()) + .collect(); + + (levels, branch) +} + +/// Spec-shaped pagination query parameters. +/// +/// The Iceberg REST spec paginates with `pageToken`/`pageSize`; Pangolin only +/// understood `limit`/`offset`, so a spec client's paging parameters were +/// silently ignored and it had no way to detect a truncated listing (B16i). +/// Both spellings are accepted, with the spec ones taking precedence. +#[derive(Deserialize, IntoParams, Default)] +pub struct IcebergPageParams { + #[serde(rename = "pageToken")] + pub page_token: Option, + #[serde(rename = "pageSize")] + pub page_size: Option, + pub limit: Option, + pub offset: Option, +} + +/// Default page size when a client asks for pagination without naming one. +pub const DEFAULT_PAGE_SIZE: u32 = 100; + +impl IcebergPageParams { + /// Resolve to `(offset, limit)`. + /// + /// The page token is an opaque encoding of the offset, per the spec's + /// "clients must treat the token as opaque" rule; the encoding here is just + /// a prefixed decimal so it stays debuggable, and an unparseable token + /// degrades to offset 0 rather than erroring the listing. + pub fn resolve(&self) -> (u32, u32) { + let limit = self + .page_size + .or(self.limit) + .filter(|l| *l > 0) + .unwrap_or(DEFAULT_PAGE_SIZE); + + let offset = self + .page_token + .as_deref() + .and_then(decode_page_token) + .or(self.offset) + .unwrap_or(0); + + (offset, limit) + } +} + +/// Encode an offset as an opaque continuation token. +pub fn encode_page_token(offset: u32) -> String { + use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; + URL_SAFE_NO_PAD.encode(format!("o:{}", offset)) +} + +/// Decode a continuation token produced by [`encode_page_token`]. +pub fn decode_page_token(token: &str) -> Option { + use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; + let decoded = URL_SAFE_NO_PAD.decode(token).ok()?; + let decoded = String::from_utf8(decoded).ok()?; + decoded.strip_prefix("o:")?.parse().ok() +} + +/// Compute the `next-page-token` for a listing. +/// +/// Returns `None` when the page came back short, which is how a client knows it +/// has reached the end. +pub fn next_page_token(returned: usize, offset: u32, limit: u32) -> Option { + if returned as u32 == limit { + Some(encode_page_token(offset + limit)) + } else { + None + } +} + #[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct RenameTableRequest { pub source: TableIdentifier, pub destination: TableIdentifier, } #[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct UpdateNamespacePropertiesRequest { pub removals: Option>, pub updates: Option>, diff --git a/pangolin/pangolin_api/src/lib.rs b/pangolin/pangolin_api/src/lib.rs index e44e056..7f8b4b9 100644 --- a/pangolin/pangolin_api/src/lib.rs +++ b/pangolin/pangolin_api/src/lib.rs @@ -24,6 +24,9 @@ pub mod audit_handlers; pub mod audit_tests; pub mod auth_middleware; pub mod authz; +/// Permission matrix: who is allowed to call what (roadmap improvement #0). +#[cfg(test)] +pub mod authz_matrix_tests; pub mod authz_utils; // Permission filtering utilities pub mod business_metadata_handlers; pub mod config; @@ -151,8 +154,14 @@ pub fn app_with_options( get(iceberg::namespaces::list_namespaces).post(iceberg::namespaces::create_namespace), ) .route( + // `loadNamespaceMetadata` and `namespaceExists` were on the README's + // "not implemented" list: a client could create a namespace and set + // properties but never read them back, and had no cheap existence + // probe. "/v1/:prefix/namespaces/:namespace", - delete(iceberg::namespaces::delete_namespace), + get(iceberg::namespaces::load_namespace_metadata) + .head(iceberg::namespaces::namespace_exists) + .delete(iceberg::namespaces::delete_namespace), ) .route( "/v1/:prefix/namespaces/:namespace/properties", @@ -192,7 +201,9 @@ pub fn app_with_options( ) .route( "/v1/:prefix/v1/namespaces/:namespace", - delete(iceberg::namespaces::delete_namespace), + get(iceberg::namespaces::load_namespace_metadata) + .head(iceberg::namespaces::namespace_exists) + .delete(iceberg::namespaces::delete_namespace), ) .route( "/v1/:prefix/v1/namespaces/:namespace/properties", @@ -252,7 +263,12 @@ pub fn app_with_options( "/api/v1/branches/:name/rebase", post(pangolin_handlers::rebase_branch), ) // Rebase endpoint - .route("/api/v1/branches/:name", get(pangolin_handlers::get_branch)) + .route( + // B33: the UI's branch-delete control has been 404ing because only + // GET was registered here. + "/api/v1/branches/:name", + get(pangolin_handlers::get_branch).delete(pangolin_handlers::delete_branch), + ) .route( "/api/v1/branches/:name/commits", get(pangolin_handlers::list_commits), @@ -517,6 +533,13 @@ pub fn app_with_options( "/api/v1/oauth/exchange", post(oauth_handlers::oauth_exchange), ) + .route( + // B33: the login page needs to know which providers are configured + // *before* anyone is authenticated; without this it rendered all + // four buttons unconditionally, several of them dead. + "/api/v1/oauth/providers", + get(oauth_handlers::list_oauth_providers), + ) // Business Metadata (by asset id) .route( "/api/v1/business-metadata/:asset_id", @@ -558,15 +581,33 @@ pub fn app_with_options( // Resource safety. There were previously no limits of any kind (A-20): // a single large POST could be buffered without bound and a slow // backend request had no deadline. + // + // Layer order matters, and `.layer()` applies *outermost last*, so this + // list reads inside-out: body limit is innermost, then the concurrency + // limiter, then the timeout, then load shedding. + // + // B16l: the timeout used to sit *inside* the concurrency limiter, so a + // request queued for one of the permits had no deadline at all - the + // 30s clock only started once it was admitted. Under sustained overload + // the queue grew without bound and clients saw latencies far past + // `PANGOLIN_REQUEST_TIMEOUT_SECS`. With the timeout outside, the + // deadline covers queueing time, which is what a client's timeout budget + // actually cares about. .layer(DefaultBodyLimit::max(options.body_limit_bytes)) - .layer(tower_http::timeout::TimeoutLayer::new( - options.request_timeout, - )) // GlobalConcurrencyLimitLayer rather than ConcurrencyLimitLayer: the // latter's service is not `Clone`, which axum requires. .layer(tower::limit::GlobalConcurrencyLimitLayer::new( options.concurrency_limit, )) + .layer(tower_http::timeout::TimeoutLayer::new( + options.request_timeout, + )) + // B0m: token issuance could be driven to panic by a request-controlled + // `expires_in_hours`, and with no catch-panic layer the panic tore down + // the whole connection task rather than failing one request. The + // arithmetic is now total, but a panic anywhere else should still be a + // 500 for one caller rather than a dropped connection for several. + .layer(tower_http::catch_panic::CatchPanicLayer::new()) // Request IDs, access logging and metrics. `tower-http` was already // built with the `trace` feature but `TraceLayer` was never applied. .layer(axum::middleware::from_fn(observability::track_request)) diff --git a/pangolin/pangolin_api/src/main.rs b/pangolin/pangolin_api/src/main.rs index 18d2d1a..92e2778 100644 --- a/pangolin/pangolin_api/src/main.rs +++ b/pangolin/pangolin_api/src/main.rs @@ -114,11 +114,26 @@ async fn main() { // the compare-and-swap that publishes it leaks an orphaned metadata file. let serve = axum::serve(listener, app).with_graceful_shutdown(shutdown_signal(shutdown_grace)); - if let Err(e) = serve.await { - tracing::error!("server error: {e}"); - std::process::exit(1); + // B16n: `shutdown_grace` used to be logged and nothing else. There was no + // bound on the drain at all, so `with_graceful_shutdown` waited for + // in-flight connections *indefinitely* - one hung upstream call blocked + // SIGTERM past the k8s termination grace period and into a SIGKILL, which + // is exactly the mid-commit kill this whole path exists to avoid. + // + // The first attempt at that bound was `tokio::time::timeout(shutdown_grace, + // serve)`, which is wrong in the worst possible way: `serve` is the whole + // server, not the drain, so it bounded the *lifetime of the process*. The + // server exited cleanly `shutdown_grace` seconds after startup - 25 by + // default - having received no signal at all. Every container would have + // crash-looped. The deadline has to start when the signal arrives, so it + // lives in `shutdown_signal` now, armed only once a signal is seen. + match serve.await { + Ok(()) => tracing::info!("shutdown complete"), + Err(e) => { + tracing::error!("server error: {e}"); + std::process::exit(1); + } } - tracing::info!("shutdown complete"); } /// Probe this instance's readiness endpoint. Returns a process exit code. @@ -367,4 +382,30 @@ async fn shutdown_signal(grace: std::time::Duration) { // while in-flight requests finish. health::mark_draining(); tracing::info!(grace_secs = grace.as_secs(), "draining in-flight requests"); + + // Give the load balancer a moment to observe the failing readiness probe + // before the listener stops accepting. Without this pause the LB can still + // be routing new connections at the instant we stop accepting them, which + // shows up to clients as connection resets during every rolling update. + // Capped so it can never consume the whole grace budget. + let deregistration_pause = std::cmp::min(grace / 4, std::time::Duration::from_secs(5)); + tokio::time::sleep(deregistration_pause).await; + + // Arm the hard deadline *here*, once a signal has actually been seen, + // rather than around the whole server. `with_graceful_shutdown` waits for + // in-flight connections with no bound of its own, so a single hung request + // would otherwise hold the process past Kubernetes' termination grace + // period and earn a SIGKILL mid-commit - the exact failure this path + // exists to prevent. + // + // Detached so returning from here still lets axum begin its drain; the + // task only wins if the drain overruns. + tokio::spawn(async move { + tokio::time::sleep(grace).await; + tracing::warn!( + grace_secs = grace.as_secs(), + "drain did not finish within the shutdown grace period; exiting anyway" + ); + std::process::exit(0); + }); } diff --git a/pangolin/pangolin_api/src/merge_handlers.rs b/pangolin/pangolin_api/src/merge_handlers.rs index 5ea9ddf..e563b47 100644 --- a/pangolin/pangolin_api/src/merge_handlers.rs +++ b/pangolin/pangolin_api/src/merge_handlers.rs @@ -17,6 +17,7 @@ type AppState = Arc; // Request/Response types #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct ResolveConflictRequest { pub strategy: ResolutionStrategy, pub resolved_value: Option, diff --git a/pangolin/pangolin_api/src/oauth_handlers.rs b/pangolin/pangolin_api/src/oauth_handlers.rs index a3bf06e..fed5091 100644 --- a/pangolin/pangolin_api/src/oauth_handlers.rs +++ b/pangolin/pangolin_api/src/oauth_handlers.rs @@ -25,6 +25,50 @@ pub struct OAuthUserInfo { pub sub: String, pub email: String, pub name: Option, + /// Whether the provider asserts the address has been verified. + /// + /// Absent on providers that do not report it - notably GitHub's + /// `/user` endpoint - which is why an absent value is treated as + /// *unverified* (B0l). + #[serde(default)] + pub email_verified: Option, +} + +impl OAuthUserInfo { + fn email_is_verified(&self) -> bool { + self.email_verified.unwrap_or(false) + } +} + +/// Domains whose *verified* addresses may link to a pre-existing local account. +/// +/// Empty by default: with no allowlist configured, email never links an account, +/// and only a `(provider, subject)` match does. +fn email_link_domain_allowlist() -> Vec { + std::env::var("PANGOLIN_OAUTH_EMAIL_LINK_DOMAINS") + .ok() + .map(|v| { + v.split(',') + .map(|s| s.trim().to_ascii_lowercase()) + .filter(|s| !s.is_empty()) + .collect() + }) + .unwrap_or_default() +} + +/// May `email` be used to adopt an existing local account? +fn email_may_link(user_info: &OAuthUserInfo, allowlist: &[String]) -> bool { + if !user_info.email_is_verified() { + return false; + } + let Some(domain) = user_info + .email + .rsplit_once('@') + .map(|(_, d)| d.to_ascii_lowercase()) + else { + return false; + }; + allowlist.contains(&domain) } #[derive(Deserialize, ToSchema)] @@ -180,23 +224,32 @@ pub async fn oauth_callback( }; // 4. Find or Create User - // We assume email is unique and can be used to link accounts or create new ones - // For MVP, we'll create a new user if not found by email, or maybe by oauth_subject - // Ideally look up by (provider, subject) - - // Since CatalogStore doesn't expose `get_user_by_oauth` yet, we'll use `get_user_by_username` as a fallback or iterate - // But `CatalogStore` trait needs a method for this efficiently. - // For now, let's list users and filter (inefficient but works for MemoryStore) - // Or better, let's just stick to email for now if unique. - - // Let's implement a rudimentary lookup + // + // B0l: the match used to include `|| u.email == user_info.email`, with no + // `email_verified` check and no provider binding. Anyone who could set a + // matching address on *any* configured provider - GitHub happily reports + // unverified addresses - logged in as that Pangolin user, including the + // seeded `TenantAdmin`. Identity is `(provider, subject)`; an address is + // only allowed to adopt a pre-existing account when the provider says it is + // verified *and* its domain is one the operator listed. let all_users = store.list_users(None, None).await.unwrap_or_default(); + let allowlist = email_link_domain_allowlist(); + let may_link_by_email = email_may_link(&user_info, &allowlist); + let existing_user = all_users.into_iter().find(|u| { - (u.oauth_provider == Some(provider_enum.clone()) - && u.oauth_subject == Some(user_info.sub.clone())) - || u.email == user_info.email + let subject_match = u.oauth_provider == Some(provider_enum.clone()) + && u.oauth_subject == Some(user_info.sub.clone()); + let email_match = may_link_by_email && u.email == user_info.email; + subject_match || email_match }); + if !may_link_by_email && !allowlist.is_empty() && !user_info.email_is_verified() { + tracing::warn!( + provider = %provider, + "OAuth provider reported an unverified email; not linking by address" + ); + } + let user = match existing_user { Some(u) => { // Update last login or details if needed @@ -311,6 +364,7 @@ pub async fn oauth_callback( /// Request body for exchanging a one-time OAuth code for a session token. #[derive(Debug, Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct OAuthExchangeRequest { pub code: String, } @@ -354,6 +408,37 @@ pub async fn oauth_exchange(Json(req): Json) -> Response { } } +/// The providers this deployment has actually configured. +/// +/// B33: the UI called `GET /api/v1/oauth/providers`, which did not exist, so it +/// 404'd - which is why the login page fell back to rendering all four provider +/// buttons unconditionally, including ones the operator had never configured. +/// Clicking those went nowhere. +/// +/// This endpoint is public (see `public_paths`): the login page has to render +/// before anyone is authenticated. It reveals only which provider *names* are +/// enabled - never a client id, and never a secret. +#[derive(serde::Serialize, utoipa::ToSchema)] +pub struct OAuthProvidersResponse { + pub providers: Vec, +} + +#[utoipa::path( + get, + path = "/api/v1/oauth/providers", + tag = "Authentication", + responses((status = 200, description = "Configured OAuth providers", body = OAuthProvidersResponse)) +)] +pub async fn list_oauth_providers() -> impl IntoResponse { + let providers = ["google", "microsoft", "github", "okta"] + .into_iter() + .filter(|p| get_oauth_config(p).is_some()) + .map(|p| p.to_string()) + .collect(); + + (StatusCode::OK, Json(OAuthProvidersResponse { providers })) +} + /// Get OAuth configuration for provider fn get_oauth_config(provider: &str) -> Option { // TODO: Load from environment variables or config file diff --git a/pangolin/pangolin_api/src/optimization_handlers.rs b/pangolin/pangolin_api/src/optimization_handlers.rs index 63358fd..c46fd9b 100644 --- a/pangolin/pangolin_api/src/optimization_handlers.rs +++ b/pangolin/pangolin_api/src/optimization_handlers.rs @@ -106,8 +106,13 @@ pub async fn search_assets_by_name( catalogs.iter().map(|c| (c.name.clone(), c.id)).collect(); // Apply permission-based filtering - let filtered_assets = - crate::authz_utils::filter_assets(assets, &permissions, session.role.clone(), &catalog_map); + let filtered_assets = crate::authz_utils::filter_assets( + tenant_id, + assets, + &permissions, + session.role.clone(), + &catalog_map, + ); // Filter by catalog if specified let mut all_results = Vec::new(); @@ -128,6 +133,7 @@ pub async fn search_assets_by_name( let namespace_str = namespace.join("."); let required_actions = vec![pangolin_core::permission::Action::Read]; crate::authz_utils::has_asset_access( + tenant_id, catalog_id, &namespace_str, asset.id, @@ -171,6 +177,7 @@ pub async fn search_assets_by_name( } #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct BulkDeleteAssetsRequest { /// List of asset UUIDs to delete (maximum 100) pub asset_ids: Vec, @@ -261,6 +268,7 @@ pub async fn bulk_delete_assets( } #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct ValidateNamesRequest { /// Resource type: "catalog" or "warehouse" pub resource_type: String, @@ -401,8 +409,12 @@ pub async fn unified_search( .search_catalogs(tenant_id, &query.q) .await .map_err(ApiError::from)?; - let filtered_catalogs = - crate::authz_utils::filter_catalogs(catalogs, &permissions, session.role.clone()); + let filtered_catalogs = crate::authz_utils::filter_catalogs( + tenant_id, + catalogs, + &permissions, + session.role.clone(), + ); for c in filtered_catalogs { results.push(UnifiedSearchResult { id: Some(c.id.to_string()), @@ -430,6 +442,7 @@ pub async fn unified_search( .collect(); let filtered_namespaces = crate::authz_utils::filter_namespaces( + tenant_id, namespaces, &permissions, session.role.clone(), @@ -452,6 +465,7 @@ pub async fn unified_search( .await .map_err(ApiError::from)?; let filtered_assets = crate::authz_utils::filter_assets( + tenant_id, assets, &permissions, session.role.clone(), @@ -481,6 +495,7 @@ pub async fn unified_search( session.role, pangolin_core::user::UserRole::Root | pangolin_core::user::UserRole::TenantAdmin ) || crate::authz_utils::has_catalog_access( + tenant_id, catalog_id, &permissions, &[pangolin_core::permission::Action::Read], diff --git a/pangolin/pangolin_api/src/pangolin_handlers.rs b/pangolin/pangolin_api/src/pangolin_handlers.rs index 1cd126c..bd0a9bb 100644 --- a/pangolin/pangolin_api/src/pangolin_handlers.rs +++ b/pangolin/pangolin_api/src/pangolin_handlers.rs @@ -19,6 +19,7 @@ use uuid::Uuid; pub type AppState = Arc; #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CreateBranchRequest { name: String, branch_type: Option, // "ingest" or "experimental", defaults to experimental @@ -36,6 +37,7 @@ pub struct ListBranchParams { } #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct MergeBranchRequest { pub source_branch: String, pub target_branch: String, @@ -304,6 +306,99 @@ pub async fn get_branch( } } +/// Delete a branch. +/// +/// B33: the UI has had a "delete branch" control calling +/// `DELETE /api/v1/branches/{catalog}/{name}` since before this audit, but the +/// router only ever registered `GET` on `/api/v1/branches/:name` - so branch +/// deletion from the UI always 404'd. The store has supported it all along; the +/// route simply did not exist. Added here with the catalog as a query parameter, +/// matching `get_branch` rather than inventing a third path shape. +#[utoipa::path( + delete, + path = "/api/v1/branches/{name}", + tag = "Branches", + params( + ("name" = String, Path, description = "Branch name"), + ("catalog" = String, Query, description = "Catalog the branch belongs to") + ), + responses( + (status = 204, description = "Branch deleted"), + (status = 400, description = "The main branch cannot be deleted"), + (status = 403, description = "Forbidden"), + (status = 404, description = "Branch or catalog not found"), + (status = 500, description = "Internal server error") + ), + security(("bearer_auth" = [])) +)] +pub async fn delete_branch( + State(store): State, + Extension(tenant): Extension, + Extension(session): Extension, + Path(name): Path, + Query(params): Query, +) -> impl IntoResponse { + let tenant_id = tenant.0; + let catalog_name = ¶ms.catalog; + + // Deleting `main` would strand every asset on it with no branch to reach + // them through. + if name == "main" { + return (StatusCode::BAD_REQUEST, "The main branch cannot be deleted").into_response(); + } + + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), + Err(e) => { + tracing::error!(error = %e, "delete_branch: failed to load catalog"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(); + } + }; + + let scope = PermissionScope::Catalog { + catalog_id: catalog.id, + }; + match crate::authz::check_permission(&store, &session, &Action::Delete, &scope).await { + Ok(true) => (), + Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Err(e) => { + tracing::error!(error = %e, "delete_branch: permission check failed"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Permission check failed").into_response(); + } + } + + match store + .delete_branch(tenant_id, catalog_name, name.clone()) + .await + { + Ok(_) => { + if let Err(e) = store + .log_audit_event( + tenant_id, + pangolin_core::audit::AuditLogEntry::success( + tenant_id, + Some(session.user_id), + session.username.clone(), + pangolin_core::audit::AuditAction::DeleteBranch, + pangolin_core::audit::ResourceType::Branch, + None, + format!("{}/{}", catalog_name, name), + ), + ) + .await + { + tracing::error!(error = %e, "failed to write the branch-delete audit record"); + } + StatusCode::NO_CONTENT.into_response() + } + Err(e) => { + tracing::debug!(error = %e, "delete_branch: branch not deleted"); + (StatusCode::NOT_FOUND, "Branch not found").into_response() + } + } +} + #[utoipa::path( post, path = "/api/v1/branches/merge", @@ -524,6 +619,7 @@ pub async fn list_commits( } #[derive(Deserialize)] +#[serde(deny_unknown_fields)] pub struct CreateTagRequest { name: String, catalog: Option, @@ -659,7 +755,7 @@ pub async fn delete_tag( pub async fn rebase_branch( State(store): State, Extension(tenant): Extension, - Extension(_session): Extension, + Extension(session): Extension, Path(branch_name): Path, Json(payload): Json, // We need catalog_name from body ) -> impl IntoResponse { @@ -669,12 +765,33 @@ pub async fn rebase_branch( // Source: main // Target: branch_name - // Check permissions - // TODO: Granular permissions? For now assume Write on Catalog - // Catalog is now required let catalog_name = &payload.catalog; + // The `// TODO: Granular permissions? For now assume Write on Catalog` that + // stood here *was* the authorization: any tenant member could rebase any + // branch, which rewrites that branch's contents. Same class as B0c-B0f. + let catalog = match store.get_catalog(tenant_id, catalog_name.clone()).await { + Ok(Some(c)) => c, + Ok(None) => return (StatusCode::NOT_FOUND, "Catalog not found").into_response(), + Err(e) => { + tracing::error!(error = %e, "rebase_branch: failed to load catalog"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(); + } + }; + + let scope = PermissionScope::Catalog { + catalog_id: catalog.id, + }; + match crate::authz::check_permission(&store, &session, &Action::Write, &scope).await { + Ok(true) => (), + Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Err(e) => { + tracing::error!(error = %e, "rebase_branch: permission check failed"); + return (StatusCode::INTERNAL_SERVER_ERROR, "Permission check failed").into_response(); + } + } + match store .merge_branch(tenant_id, catalog_name, "main".to_string(), branch_name) .await @@ -690,6 +807,7 @@ pub async fn rebase_branch( // Catalog Management #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CreateCatalogRequest { name: String, catalog_type: Option, @@ -700,6 +818,7 @@ pub struct CreateCatalogRequest { } #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct UpdateCatalogRequest { warehouse_name: Option, storage_location: Option, @@ -751,7 +870,16 @@ pub async fn list_catalogs( let tenant_id = tenant.0; tracing::info!("list_catalogs called with tenant_id: {}", tenant_id); - match store.list_catalogs(tenant_id, Some(pagination)).await { + // B42: the store used to paginate first and `filter_catalogs` removed the + // unauthorized rows afterwards, so a `TenantUser` got variable-size pages - + // including *empty pages while more authorized data existed*. Any client + // that stops on an empty page (the normal idiom) silently missed data. + // + // Filtering has to happen before slicing. The permitted set is not + // expressible as a store predicate today, so the rows are fetched unpaged + // and the page is cut after filtering; the page window is applied below. + let requested = pagination; + match store.list_catalogs(tenant_id, None).await { Ok(catalogs) => { // Use authz_utils for consistent permission filtering let permissions = if matches!(session.role, UserRole::TenantUser) { @@ -769,8 +897,12 @@ pub async fn list_catalogs( Vec::new() // Root/TenantAdmin bypass filtering }; - let filtered_catalogs = - crate::authz_utils::filter_catalogs(catalogs, &permissions, session.role.clone()); + let filtered_catalogs = crate::authz_utils::filter_catalogs( + tenant_id, + catalogs, + &permissions, + session.role.clone(), + ); tracing::info!( "list_catalogs returning {} catalogs for user {}", @@ -778,8 +910,16 @@ pub async fn list_catalogs( session.user_id ); - let resp: Vec = - filtered_catalogs.into_iter().map(|c| c.into()).collect(); + // Now slice the *authorized* set, so a page is empty only when the + // caller has actually reached the end. + let offset = requested.offset.unwrap_or(0); + let limit = requested.limit.unwrap_or(usize::MAX); + let resp: Vec = filtered_catalogs + .into_iter() + .skip(offset) + .take(limit) + .map(|c| c.into()) + .collect(); (StatusCode::OK, Json(resp)).into_response() } Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error").into_response(), diff --git a/pangolin/pangolin_api/src/permission_handlers.rs b/pangolin/pangolin_api/src/permission_handlers.rs index c570d88..3bd9401 100644 --- a/pangolin/pangolin_api/src/permission_handlers.rs +++ b/pangolin/pangolin_api/src/permission_handlers.rs @@ -16,6 +16,7 @@ use uuid::Uuid; /// Request to create a new role #[derive(Debug, Deserialize, ToSchema)] #[serde(rename_all = "kebab-case")] +#[serde(deny_unknown_fields)] pub struct CreateRoleRequest { pub name: String, pub description: Option, @@ -25,6 +26,7 @@ pub struct CreateRoleRequest { /// Request to assign a role to a user #[derive(Debug, Deserialize, ToSchema)] #[serde(rename_all = "kebab-case")] +#[serde(deny_unknown_fields)] pub struct AssignRoleRequest { pub role_id: Uuid, } @@ -32,6 +34,7 @@ pub struct AssignRoleRequest { /// Request to grant a permission #[derive(Debug, Deserialize, ToSchema)] #[serde(rename_all = "kebab-case")] +#[serde(deny_unknown_fields)] pub struct GrantPermissionRequest { pub user_id: Uuid, pub scope: PermissionScope, diff --git a/pangolin/pangolin_api/src/public_paths.rs b/pangolin/pangolin_api/src/public_paths.rs index ef128e3..c987141 100644 --- a/pangolin/pangolin_api/src/public_paths.rs +++ b/pangolin/pangolin_api/src/public_paths.rs @@ -53,6 +53,21 @@ pub fn is_public_path(path: &str) -> bool { ["oauth", "authorize", _provider] => true, ["oauth", "callback", _provider] => true, + // Redeeming the one-time code the OAuth callback hands back. + // + // B0k: this was missing, so the middleware demanded a bearer token on + // the very endpoint whose job is to obtain the first one. The 0.6.0 + // callback -> one-time code -> POST exchange flow was therefore + // unreachable in production: the browser landed with a `?code=...` it + // could never redeem. The code itself is single-use and short-lived, + // which is what makes this endpoint safe to expose unauthenticated. + ["api", "v1", "oauth", "exchange"] => true, + + // Which OAuth providers are configured. The login page needs this + // before anyone is authenticated (see B33); it reveals only provider + // names, never secrets. + ["api", "v1", "oauth", "providers"] => true, + _ => false, } } @@ -87,6 +102,18 @@ mod tests { } } + /// Regression test for B0k: without this the OAuth login flow cannot + /// complete, because the code-exchange endpoint demanded the token it + /// exists to issue. + #[test] + fn oauth_exchange_and_providers_are_public() { + assert!(is_public_path("/api/v1/oauth/exchange")); + assert!(is_public_path("/api/v1/oauth/providers")); + // ...but nothing deeper under the same prefix. + assert!(!is_public_path("/api/v1/oauth/exchange/steal")); + assert!(!is_public_path("/api/v1/oauth/tokens")); + } + /// Regression test for A-11: a resource named `config` must not bypass auth. #[test] fn resources_named_config_are_not_public() { diff --git a/pangolin/pangolin_api/src/service_user_handlers.rs b/pangolin/pangolin_api/src/service_user_handlers.rs index 22beb26..90868d4 100644 --- a/pangolin/pangolin_api/src/service_user_handlers.rs +++ b/pangolin/pangolin_api/src/service_user_handlers.rs @@ -18,6 +18,7 @@ type AppState = Arc; // Request/Response types #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CreateServiceUserRequest { pub name: String, pub description: Option, @@ -26,6 +27,7 @@ pub struct CreateServiceUserRequest { } #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct UpdateServiceUserRequest { pub name: Option, pub description: Option, diff --git a/pangolin/pangolin_api/src/signing_handlers.rs b/pangolin/pangolin_api/src/signing_handlers.rs index f0c2f7b..70010f8 100644 --- a/pangolin/pangolin_api/src/signing_handlers.rs +++ b/pangolin/pangolin_api/src/signing_handlers.rs @@ -1,5 +1,6 @@ use crate::auth::TenantId; -use crate::iceberg::AppState; +use crate::authz::check_permission; +use crate::iceberg::{parse_namespace, parse_table_identifier, AppState}; use axum::Extension; use axum::{ extract::{Path, Query, State}, @@ -7,6 +8,8 @@ use axum::{ response::IntoResponse, Json, }; +use pangolin_core::permission::{Action, PermissionScope}; +use pangolin_core::user::UserSession; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use utoipa::{IntoParams, ToSchema}; @@ -180,6 +183,17 @@ pub async fn get_gcp_token(_service_account_key_json: &str) -> Result Result Result, Extension(tenant): Extension, + Extension(session): Extension, Path((catalog_name, namespace, table)): Path<(String, String, String)>, ) -> impl IntoResponse { let tenant_id = tenant.0; @@ -218,7 +233,55 @@ pub async fn get_table_credentials( Err(e) => return (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response(), }; - // 2. Check if catalog has a warehouse + // 2. Resolve the asset the caller is asking for. Vending credentials for a + // table that does not exist is not a meaningful operation, and resolving it + // is what makes an asset-scoped permission check possible at all. + let (ns_levels, branch) = parse_namespace(&namespace); + let (table_name, branch_from_table) = parse_table_identifier(&table); + let branch = branch_from_table.or(branch); + + let asset = match store + .get_asset( + tenant_id, + &catalog_name, + branch, + ns_levels.clone(), + table_name.clone(), + ) + .await + { + Ok(Some(a)) => a, + Ok(None) => return (StatusCode::NOT_FOUND, "Table not found").into_response(), + Err(e) => return (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response(), + }; + + let scope = PermissionScope::Asset { + catalog_id: catalog.id, + namespace: ns_levels.join("."), + asset_id: asset.id, + }; + + // 3. Read is the floor: without it, no credentials at all. + match check_permission(&store, &session, &Action::Read, &scope).await { + Ok(true) => (), + Ok(false) => return (StatusCode::FORBIDDEN, "Forbidden").into_response(), + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Permission check failed: {}", e), + ) + .into_response() + } + } + + // Write is vended only when actually held, so a read-only principal gets + // read-only cloud credentials. + let can_write = matches!( + check_permission(&store, &session, &Action::Write, &scope).await, + Ok(true) + ); + + // 4. Check if catalog has a warehouse let warehouse_name = match catalog.warehouse_name { Some(name) => name, None => { @@ -231,16 +294,23 @@ pub async fn get_table_credentials( } }; - // 3. Get warehouse configuration + // 5. Get warehouse configuration let warehouse = match store.get_warehouse(tenant_id, warehouse_name).await { Ok(Some(wh)) => wh, Ok(None) => return (StatusCode::NOT_FOUND, "Warehouse not found").into_response(), Err(e) => return (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response(), }; - // 4. Vend credentials using the credential signer infrastructure - let resource_path = format!("{}/{}", namespace, table); - let permissions = vec!["read".to_string(), "write".to_string()]; + // 6. Vend credentials using the credential signer infrastructure + let resource_path = if ns_levels.is_empty() { + table_name.clone() + } else { + format!("{}/{}", ns_levels.join("/"), table_name) + }; + let mut permissions = vec!["read".to_string()]; + if can_write { + permissions.push("write".to_string()); + } match crate::credential_vending::vend_credentials_for_warehouse( &warehouse, diff --git a/pangolin/pangolin_api/src/tenant_handlers.rs b/pangolin/pangolin_api/src/tenant_handlers.rs index d16738b..63304c1 100644 --- a/pangolin/pangolin_api/src/tenant_handlers.rs +++ b/pangolin/pangolin_api/src/tenant_handlers.rs @@ -15,12 +15,14 @@ use utoipa::ToSchema; use uuid::Uuid; #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CreateTenantRequest { name: String, properties: Option>, } #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct UpdateTenantRequest { name: Option, properties: Option>, diff --git a/pangolin/pangolin_api/src/token_handlers.rs b/pangolin/pangolin_api/src/token_handlers.rs index 83db728..1110159 100644 --- a/pangolin/pangolin_api/src/token_handlers.rs +++ b/pangolin/pangolin_api/src/token_handlers.rs @@ -15,6 +15,7 @@ use utoipa::ToSchema; use uuid::Uuid; #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct GenerateTokenRequest { pub tenant_id: String, pub username: Option, @@ -22,6 +23,38 @@ pub struct GenerateTokenRequest { pub expires_in_hours: Option, } +/// Upper bound on a caller-requested token lifetime. +/// +/// `expires_in_hours` is attacker-controlled and used to be fed straight into +/// `chrono::Duration::hours`, which *panics* on a large enough value - before +/// the `checked_add_signed().unwrap()` below it could even run (B0m). There is +/// no `CatchPanicLayer` under the router, so that panic aborted the connection +/// task. Clamping first makes the arithmetic total. +const MAX_TOKEN_LIFETIME_HOURS: u64 = 24 * 365; + +/// Rank roles so a caller cannot mint a token more privileged than itself. +fn role_rank(role: &UserRole) -> u8 { + match role { + UserRole::Root => 3, + UserRole::TenantAdmin => 2, + UserRole::TenantUser => 1, + } +} + +/// Parse a role name from a token request. +/// +/// Accepts both the Debug-ish spellings the old code matched and the kebab-case +/// serde names, but unknown values are now an error rather than a silent +/// downgrade to `TenantUser`. +fn parse_role(name: &str) -> Option { + match name { + "Root" | "root" => Some(UserRole::Root), + "Admin" | "admin" | "TenantAdmin" | "tenant-admin" => Some(UserRole::TenantAdmin), + "TenantUser" | "tenant-user" | "user" => Some(UserRole::TenantUser), + _ => None, + } +} + #[derive(Serialize, ToSchema)] pub struct GenerateTokenResponse { pub token: String, @@ -29,8 +62,21 @@ pub struct GenerateTokenResponse { pub tenant_id: String, } -/// Generate a JWT token for a tenant -/// This endpoint allows generating tokens for testing and development +/// Generate a JWT token for a tenant. +/// +/// Authorization (B0a): this handler used to take no session at all, so any +/// authenticated principal - including the lowest-privilege `TenantUser` or any +/// service-user API key - could POST `{"tenant_id": "", "roles":["Root"]}` +/// and receive a signed `Root` token for an arbitrary tenant. That is a total +/// privilege escalation, since `check_permission` short-circuits for `Root`. +/// +/// The rules now are: +/// * `Root` may mint anything. +/// * `TenantAdmin` may mint only for its own tenant, and only a role at or +/// below its own. +/// * everyone else is refused. +/// A role supplied in the body is never trusted for a non-`Root` caller beyond +/// those bounds. #[utoipa::path( post, path = "/api/v1/tokens", @@ -39,60 +85,107 @@ pub struct GenerateTokenResponse { responses( (status = 200, description = "Token generated", body = GenerateTokenResponse), (status = 400, description = "Bad request"), + (status = 403, description = "Forbidden"), (status = 500, description = "Internal server error") - ) + ), + security(("bearer_auth" = [])) )] pub async fn generate_token( State(store): State, + Extension(session): Extension, Json(payload): Json, ) -> impl IntoResponse { // Validate tenant_id is a valid UUID - let _tenant_uuid = match Uuid::parse_str(&payload.tenant_id) { + let tenant_uuid = match Uuid::parse_str(&payload.tenant_id) { Ok(uuid) => uuid, Err(_) => return (StatusCode::BAD_REQUEST, "Invalid tenant_id format").into_response(), }; + let caller_is_root = session.role == UserRole::Root; + if !caller_is_root { + if session.role != UserRole::TenantAdmin { + return ( + StatusCode::FORBIDDEN, + "Root or tenant-admin access required to mint tokens", + ) + .into_response(); + } + if session.tenant_id != Some(tenant_uuid) { + return ( + StatusCode::FORBIDDEN, + "Cannot mint a token for another tenant", + ) + .into_response(); + } + } + let secret = crate::config::jwt_secret(); - let expires_in = payload.expires_in_hours.unwrap_or(24); + let expires_in = payload + .expires_in_hours + .unwrap_or(24) + .min(MAX_TOKEN_LIFETIME_HOURS); let now = chrono::Utc::now(); - let exp = now + let Some(exp) = now .checked_add_signed(chrono::Duration::hours(expires_in as i64)) - .unwrap() - .timestamp(); + .map(|t| t.timestamp()) + else { + return (StatusCode::BAD_REQUEST, "expires_in_hours out of range").into_response(); + }; let username = payload.username.unwrap_or_else(|| "api-user".to_string()); - // Map role strings to UserRole - // Default to lookup user role or TenantUser if not specified - let role = if let Some(roles) = &payload.roles { - if let Some(first_role) = roles.first() { - match first_role.as_str() { - "Root" | "root" => UserRole::Root, - "Admin" | "admin" | "TenantAdmin" | "tenant-admin" => UserRole::TenantAdmin, - _ => UserRole::TenantUser, - } - } else { - UserRole::TenantUser + // Map role strings to UserRole. An unknown name is now a 400 rather than a + // silent downgrade, so a typo cannot quietly hand out the wrong role. + let requested_role = if let Some(roles) = &payload.roles { + match roles.first() { + Some(first_role) => match parse_role(first_role) { + Some(r) => Some(r), + None => { + return ( + StatusCode::BAD_REQUEST, + format!("Unknown role: {}", first_role), + ) + .into_response() + } + }, + None => None, } } else { - // Try to lookup user - if let Ok(Some(user)) = store.get_user_by_username(&username).await { - tracing::info!( - "generate_token: Found user '{}' with role {:?} ({})", - username, - user.role, - user.id - ); - user.role - } else { - tracing::warn!( - "generate_token: User '{}' not found, defaulting to TenantUser", - username - ); - UserRole::TenantUser + None + }; + + let role = match requested_role { + Some(r) => r, + None => { + // Try to look up the user's own role. + if let Ok(Some(user)) = store.get_user_by_username(&username).await { + tracing::info!( + "generate_token: Found user '{}' with role {:?} ({})", + username, + user.role, + user.id + ); + user.role + } else { + tracing::warn!( + "generate_token: User '{}' not found, defaulting to TenantUser", + username + ); + UserRole::TenantUser + } } }; + // A non-root caller can never mint above its own rank, whatever the body or + // the looked-up user says. + if !caller_is_root && role_rank(&role) > role_rank(&session.role) { + return ( + StatusCode::FORBIDDEN, + "Cannot mint a token more privileged than the caller", + ) + .into_response(); + } + // sub MUST be a UUID for to_session() to work // If user exists, use their ID. Else generate one. let user_id = if let Ok(Some(user)) = store.get_user_by_username(&username).await { @@ -121,10 +214,7 @@ pub async fn generate_token( // Store token info for listing let token_info = TokenInfo { id: token_id, - tenant_id: match Uuid::parse_str(&payload.tenant_id) { - Ok(u) => u, - Err(_) => Uuid::default(), // Should be validated above - }, + tenant_id: tenant_uuid, user_id, username: username.clone(), expires_at: chrono::DateTime::from_timestamp(exp, 0).unwrap_or_default(), @@ -141,7 +231,7 @@ pub async fn generate_token( let response = GenerateTokenResponse { token, expires_at: chrono::DateTime::from_timestamp(exp, 0) - .unwrap() + .unwrap_or_default() .to_rfc3339(), tenant_id: payload.tenant_id, }; @@ -164,6 +254,7 @@ use pangolin_store::CatalogStore; use std::sync::Arc; #[derive(Debug, Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct RevokeTokenRequest { pub reason: Option, } @@ -190,11 +281,22 @@ pub async fn revoke_current_token( Extension(session): Extension, Json(payload): Json, ) -> impl IntoResponse { - // Generate an expiration time (tokens typically expire in 24 hours) - let expires_at = Utc::now() + Duration::hours(24); - - // Use the user_id as the token_id for revocation - let token_id = session.user_id; + // Revoke until the token would have expired anyway; the blacklist entry + // only has to outlive the token. + let expires_at = session.expires_at.max(Utc::now()); + + // B0j: revoke the token's own `jti`. This used to revoke `session.user_id`, + // which no token ever carries as its `jti`, so the middleware's revocation + // check (keyed by `jti`) never matched: logout returned 200 and the token + // stayed valid for its full lifetime. + let Some(token_id) = session.token_id else { + // API-key and root-basic-auth sessions have no revocable JWT. + return ( + StatusCode::BAD_REQUEST, + "This session is not backed by a revocable token", + ) + .into_response(); + }; match store .revoke_token(token_id, expires_at, payload.reason) @@ -474,10 +576,16 @@ pub async fn rotate_token( let secret = crate::config::jwt_secret(); let now = chrono::Utc::now(); let expires_in = 24; // Default rotation to 24h - let exp = now + let Some(exp) = now .checked_add_signed(chrono::Duration::hours(expires_in)) - .unwrap() - .timestamp(); + .map(|t| t.timestamp()) + else { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to compute token expiry", + ) + .into_response(); + }; let token_id = Uuid::new_v4(); let tenant_id_str = session.tenant_id.map(|t| t.to_string()).unwrap_or_default(); @@ -514,21 +622,31 @@ pub async fn rotate_token( tracing::warn!("Failed to store rotated token info: {}", e); } - // 3. Revoke old token (if we knew its ID - session doesn't carry jti currently in UserSession struct?) - // Wait, UserSession struct usually just has user info. - // Current `auth_middleware` decodes claims but might not pass `jti` to `UserSession`. - // Let's check `UserSession`. If it doesn't have token ID, we can't revoke the *specific* old token easily. - // We can revoke ALL other tokens for this user? No, that's too aggressive. - // If we can't revoke the old one, "Rotation" is just "Get New Token". - // Ideally UserSession should have `token_id`. - // I'll skip revocation of old token for now if I lack the ID, but assume the client will discard it. - // Implementation: Just return new token. - // Update: We can update UserSession to include token_id (jti) later. + // 3. Revoke the old token. `UserSession` now carries the presenting + // token's `jti` (B0j), so rotation is a real rotation rather than + // "issue a second valid token and hope the client forgets the first". + if let Some(old_token_id) = session.token_id { + if let Err(e) = store + .revoke_token( + old_token_id, + session.expires_at.max(Utc::now()), + Some("Rotated".to_string()), + ) + .await + { + tracing::error!(error = %e, "failed to revoke the rotated-out token"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + "Issued a new token but could not revoke the old one", + ) + .into_response(); + } + } let response = GenerateTokenResponse { token, expires_at: chrono::DateTime::from_timestamp(exp, 0) - .unwrap() + .unwrap_or_default() .to_rfc3339(), tenant_id: tenant_id_str, }; diff --git a/pangolin/pangolin_api/src/user_handlers.rs b/pangolin/pangolin_api/src/user_handlers.rs index 72b17f4..1cdf2e2 100644 --- a/pangolin/pangolin_api/src/user_handlers.rs +++ b/pangolin/pangolin_api/src/user_handlers.rs @@ -13,6 +13,7 @@ use uuid::Uuid; /// Request to create a new user #[derive(Debug, Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CreateUserRequest { pub username: String, pub email: String, @@ -24,6 +25,7 @@ pub struct CreateUserRequest { /// Request to update a user #[derive(Debug, Deserialize, ToSchema)] #[serde(rename_all = "kebab-case")] +#[serde(deny_unknown_fields)] pub struct UpdateUserRequest { pub email: Option, pub password: Option, @@ -33,6 +35,7 @@ pub struct UpdateUserRequest { /// User login request #[derive(Debug, Deserialize, Serialize, ToSchema)] #[serde(rename_all = "kebab-case")] +#[serde(deny_unknown_fields)] pub struct LoginRequest { pub username: String, pub password: String, diff --git a/pangolin/pangolin_api/src/verification_tests.rs b/pangolin/pangolin_api/src/verification_tests.rs index 737d5cd..8ea5656 100644 --- a/pangolin/pangolin_api/src/verification_tests.rs +++ b/pangolin/pangolin_api/src/verification_tests.rs @@ -172,7 +172,10 @@ async fn test_verified_flow_regression() { .body(Body::from( json!({ "name": "test-tenant", - "organization": "TestOrg" + // `CreateTenantRequest` has no `organization` field. + // With `deny_unknown_fields` this is now a 422 rather + // than a silently dropped value (improvement #0). + "properties": { "organization": "TestOrg" } }) .to_string(), )) @@ -254,7 +257,10 @@ async fn test_verified_flow_regression() { json!({ "name": "test-catalog", "warehouse_name": "success-warehouse", - "type": "pangolin" + // `CreateCatalogRequest` takes `catalog_type`, not + // `type` - the old spelling was silently dropped and + // the catalog defaulted to Local anyway. + "catalog_type": "Local" }) .to_string(), )) @@ -304,7 +310,7 @@ async fn test_iceberg_namespace_creation() { .header("Authorization", format!("Bearer {}", root_token)) .header("Content-Type", "application/json") .body(Body::from( - json!({"name":"t1", "organization":"o1"}).to_string(), + json!({"name":"t1", "properties": {"organization":"o1"}}).to_string(), )) .unwrap(), ) @@ -339,7 +345,7 @@ async fn test_iceberg_namespace_creation() { .header("Authorization", format!("Bearer {}", ta_token)) .header("Content-Type", "application/json") .body(Body::from( - json!({"name":"cat","warehouse_name":"wh","type":"pangolin"}).to_string(), + json!({"name":"cat","warehouse_name":"wh","catalog_type":"Local"}).to_string(), )) .unwrap(), ) diff --git a/pangolin/pangolin_api/src/warehouse_handlers.rs b/pangolin/pangolin_api/src/warehouse_handlers.rs index 05e6e5c..597f498 100644 --- a/pangolin/pangolin_api/src/warehouse_handlers.rs +++ b/pangolin/pangolin_api/src/warehouse_handlers.rs @@ -18,6 +18,7 @@ pub struct GetCredentialsParams { } #[derive(Serialize, Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct CreateWarehouseRequest { pub name: String, pub use_sts: Option, // If true, use STS credential vending; if false, pass through static creds @@ -26,6 +27,7 @@ pub struct CreateWarehouseRequest { } #[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] pub struct UpdateWarehouseRequest { name: Option, use_sts: Option, diff --git a/pangolin/pangolin_api/tests/business_metadata_test.rs b/pangolin/pangolin_api/tests/business_metadata_test.rs index c65fed4..8e785fd 100644 --- a/pangolin/pangolin_api/tests/business_metadata_test.rs +++ b/pangolin/pangolin_api/tests/business_metadata_test.rs @@ -124,6 +124,7 @@ async fn test_business_metadata_flow() { role: UserRole::Root, issued_at: chrono::Utc::now(), expires_at: chrono::Utc::now() + chrono::Duration::hours(1), + token_id: None, }; let token = pangolin_api::auth_middleware::generate_token(session, secret).unwrap(); diff --git a/pangolin/pangolin_api/tests/credential_vending_integration.rs b/pangolin/pangolin_api/tests/credential_vending_integration.rs index 2b06b02..4129f69 100644 --- a/pangolin/pangolin_api/tests/credential_vending_integration.rs +++ b/pangolin/pangolin_api/tests/credential_vending_integration.rs @@ -294,6 +294,6 @@ async fn test_permission_scoping() { assert!(read_write.config.contains_key("gcp-oauth-token")); // Both should succeed but in real implementation would have different scopes - assert!(read_only.config.get("gcp-oauth-token").is_some()); - assert!(read_write.config.get("gcp-oauth-token").is_some()); + assert!(read_only.config.contains_key("gcp-oauth-token")); + assert!(read_write.config.contains_key("gcp-oauth-token")); } diff --git a/pangolin/pangolin_api/tests/credential_vending_tests.rs b/pangolin/pangolin_api/tests/credential_vending_tests.rs index fc8240c..7a80bad 100644 --- a/pangolin/pangolin_api/tests/credential_vending_tests.rs +++ b/pangolin/pangolin_api/tests/credential_vending_tests.rs @@ -233,6 +233,7 @@ fn create_test_metadata() -> TableMetadata { last_updated_ms: 1234567890, last_column_id: 1, schemas: vec![Schema { + type_: "struct".to_string(), schema_id: 0, fields: vec![], identifier_field_ids: None, @@ -242,6 +243,7 @@ fn create_test_metadata() -> TableMetadata { spec_id: 0, fields: vec![], }], + last_partition_id: 999, properties: Some(HashMap::new()), current_snapshot_id: None, snapshots: None, diff --git a/pangolin/pangolin_api/tests/full_system_test.rs b/pangolin/pangolin_api/tests/full_system_test.rs index d3cc45d..7f9e69b 100644 --- a/pangolin/pangolin_api/tests/full_system_test.rs +++ b/pangolin/pangolin_api/tests/full_system_test.rs @@ -24,7 +24,9 @@ async fn test_full_system_flow() { let app = app(store.clone()); // 2. Create Tenant with Initial Admin - let tenant_id = Uuid::new_v4(); + // + // The client-chosen id is gone: `CreateTenantRequest` never accepted one, + // so this only ever looked like it was being used. let admin_username = "system_admin"; let admin_password = "secure_password"; @@ -35,11 +37,11 @@ async fn test_full_system_flow() { .header("Authorization", "Basic YWRtaW46cGFzc3dvcmQ=") // root auth .body(Body::from( json!({ + // Only `name` and `properties` exist on + // `CreateTenantRequest`; the rest were silently discarded, so + // the tenant id below never matched what the server assigned. "name": "E2ETenant", - "id": tenant_id.to_string(), // In memory store usually ignores ID in payload but useful if supported - "properties": {}, - "admin_username": admin_username, - "admin_password": admin_password + "properties": {} }) .to_string(), )) @@ -103,7 +105,33 @@ async fn test_full_system_flow() { println!("Logged in as Tenant Admin. Token length: {}", token.len()); - // 4. Create Catalog + // 4. Create the warehouse the catalog references. + // + // This step did not exist. The catalog payload below named a warehouse + // called "default" under a *kebab-case* key the server does not read, so it + // was dropped, `warehouse_name` arrived as `None`, and the existence check + // was skipped entirely - the test passed while asserting nothing about + // warehouses. With the key corrected the check runs, and it needs a + // warehouse to find. + let create_warehouse_req = Request::builder() + .method("POST") + .uri("/api/v1/warehouses") + .header("Authorization", &auth_header) + .header("Content-Type", "application/json") + .body(Body::from( + json!({ + "name": "default", + "use_sts": false, + "storage_config": { "type": "filesystem", "root": "/tmp/pangolin-e2e" } + }) + .to_string(), + )) + .unwrap(); + + let resp = app.clone().oneshot(create_warehouse_req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::CREATED); + + // 5. Create Catalog // Note: We need to use the Tenant ID header for tenant-scoped operations let create_catalog_req = Request::builder() .method("POST") @@ -112,10 +140,17 @@ async fn test_full_system_flow() { .header("Content-Type", "application/json") .body(Body::from( json!({ + // Three phantom fields lived here: `type` (the field is + // `catalog_type`) and kebab-case `warehouse-name` / + // `storage-location` (the struct has no rename_all, so they are + // snake_case). All three were silently dropped, so this test + // asserted 201 for a catalog created with no warehouse and no + // storage location - the exact class of drift + // `deny_unknown_fields` now catches. "name": "data_catalog", - "type": "nessie", - "warehouse-name": "default", - "storage-location": "s3://bucket/data", + "catalog_type": "Local", + "warehouse_name": "default", + "storage_location": "s3://bucket/data", "properties": {} }) .to_string(), diff --git a/pangolin/pangolin_api/tests/iceberg_handlers_test.rs b/pangolin/pangolin_api/tests/iceberg_handlers_test.rs index ed59401..09623b2 100644 --- a/pangolin/pangolin_api/tests/iceberg_handlers_test.rs +++ b/pangolin/pangolin_api/tests/iceberg_handlers_test.rs @@ -28,6 +28,7 @@ fn test_add_snapshot_deserializes_full_object() { current_schema_id: 0, current_partition_spec_id: 0, partition_specs: vec![], + last_partition_id: 999, default_sort_order_id: 0, sort_orders: vec![], properties: None, @@ -77,6 +78,7 @@ fn test_add_snapshot_handles_multiple_snapshots() { current_schema_id: 0, current_partition_spec_id: 0, partition_specs: vec![], + last_partition_id: 999, default_sort_order_id: 0, sort_orders: vec![], properties: None, @@ -140,6 +142,7 @@ fn test_table_response_includes_config() { current_schema_id: 0, current_partition_spec_id: 0, partition_specs: vec![], + last_partition_id: 999, default_sort_order_id: 0, sort_orders: vec![], properties: None, diff --git a/pangolin/pangolin_api/tests/merge_tests.rs b/pangolin/pangolin_api/tests/merge_tests.rs index 374ed1e..1d76c56 100644 --- a/pangolin/pangolin_api/tests/merge_tests.rs +++ b/pangolin/pangolin_api/tests/merge_tests.rs @@ -33,10 +33,10 @@ async fn test_merge_branch_flow() { .header("Authorization", "Basic YWRtaW46cGFzc3dvcmQ=") // admin:password .body(Body::from( json!({ - "name": "MergeTestTenant", - "id": tenant_id.to_string(), - "admin_username": "merge_admin", - "admin_password": "password" + // `CreateTenantRequest` takes only `name` and `properties`. + // `id`, `admin_username` and `admin_password` were all silently + // discarded; `deny_unknown_fields` now names them. + "name": "MergeTestTenant" }) .to_string(), )) diff --git a/pangolin/pangolin_api/tests/signing_handlers_test.rs b/pangolin/pangolin_api/tests/signing_handlers_test.rs index 246507a..a838f0d 100644 --- a/pangolin/pangolin_api/tests/signing_handlers_test.rs +++ b/pangolin/pangolin_api/tests/signing_handlers_test.rs @@ -338,6 +338,7 @@ fn test_table_response_includes_credentials() { schemas: vec![], current_schema_id: 0, partition_specs: vec![], + last_partition_id: 999, current_partition_spec_id: 0, properties: None, current_snapshot_id: None, @@ -394,6 +395,7 @@ fn test_table_response_without_credentials() { schemas: vec![], current_schema_id: 0, partition_specs: vec![], + last_partition_id: 999, current_partition_spec_id: 0, properties: None, current_snapshot_id: None, diff --git a/pangolin/pangolin_cli_admin/Cargo.toml b/pangolin/pangolin_cli_admin/Cargo.toml index 396535b..ad6507b 100644 --- a/pangolin/pangolin_cli_admin/Cargo.toml +++ b/pangolin/pangolin_cli_admin/Cargo.toml @@ -15,7 +15,7 @@ categories = ["database"] workspace = true [dependencies] -pangolin_cli_common = { path = "../pangolin_cli_common", version = "0.6.0" } +pangolin_cli_common = { path = "../pangolin_cli_common", version = "0.7.0" } clap = { version = "4.4", features = ["derive", "env"] } tokio = { version = "1.0", features = ["full"] } rustyline = "14.0" diff --git a/pangolin/pangolin_cli_admin/src/handlers/catalogs.rs b/pangolin/pangolin_cli_admin/src/handlers/catalogs.rs index 7245886..12a2bdc 100644 --- a/pangolin/pangolin_cli_admin/src/handlers/catalogs.rs +++ b/pangolin/pangolin_cli_admin/src/handlers/catalogs.rs @@ -47,10 +47,16 @@ pub async fn handle_create_catalog( )); } + // B_cli1: this sent `warehouse` and `type`. `CreateCatalogRequest` has + // neither - it takes `warehouse_name` and `catalog_type` - and serde + // ignored both unknown fields, so `warehouse_name` arrived as `None` (which + // *skips the warehouse existence check*) and `catalog_type` fell back to + // `Local`. The required `--warehouse` flag was thrown away and the CLI + // printed success for a catalog with no warehouse attached. let body = serde_json::json!({ "name": name, - "warehouse": warehouse, - "type": "pangea" // defaulting to internal type + "warehouse_name": warehouse, + "catalog_type": "Local" }); let res = client.post("/api/v1/catalogs", &body).await?; diff --git a/pangolin/pangolin_cli_admin/src/handlers/federated.rs b/pangolin/pangolin_cli_admin/src/handlers/federated.rs index 960bd8a..f9ccf8f 100644 --- a/pangolin/pangolin_cli_admin/src/handlers/federated.rs +++ b/pangolin/pangolin_cli_admin/src/handlers/federated.rs @@ -9,7 +9,7 @@ pub async fn handle_sync_federated_catalog( ) -> Result<(), CliError> { let res = client .post( - &format!("/api/v1/catalogs/{}/sync", name), + &format!("/api/v1/federated-catalogs/{}/sync", name), &serde_json::json!({}), ) .await?; @@ -25,7 +25,7 @@ pub async fn handle_get_federated_catalog_stats( name: String, ) -> Result<(), CliError> { let res = client - .get(&format!("/api/v1/catalogs/{}/stats", name)) + .get(&format!("/api/v1/federated-catalogs/{}/stats", name)) .await?; if !res.status().is_success() { return Err(CliError::ApiError(format!( @@ -60,14 +60,20 @@ pub async fn handle_create_federated_catalog( } } + // B_cli3: this POSTed to `/api/v1/catalogs` with a `type` field that + // `CreateCatalogRequest` does not have, so serde dropped it and the command + // created an ordinary *Local* catalog while reporting a federated one. The + // dedicated endpoint takes the config the federated proxy actually reads. + props.insert( + "storage_location".to_string(), + serde_json::Value::String(storage_location.clone()), + ); let body = serde_json::json!({ "name": name, - "type": "federated", - "storage_location": storage_location, - "properties": props + "config": { "properties": props } }); - let res = client.post("/api/v1/catalogs", &body).await?; + let res = client.post("/api/v1/federated-catalogs", &body).await?; if !res.status().is_success() { let s = res.status(); let t = res.text().await.unwrap_or_default(); @@ -100,10 +106,13 @@ pub async fn handle_list_federated_catalogs( .await .map_err(|e| CliError::ApiError(e.to_string()))?; - // Filter client side just in case API doesn't support query param perfectly yet or to be safe + // B_cli3: this filtered on `i["type"] == "federated"`. The response key is + // `catalog_type` and its value is `"Federated"` - so the predicate never + // matched and `list-federated-catalogs` always printed an empty table, even + // with federated catalogs present. let fed_cats: Vec> = items .iter() - .filter(|i| i["type"].as_str() == Some("federated")) + .filter(|i| i["catalog_type"].as_str() == Some("Federated")) .map(|i| { vec![ i["name"].as_str().unwrap_or("").to_string(), @@ -139,7 +148,7 @@ pub async fn handle_test_federated_catalog( ) -> Result<(), CliError> { let res = client .post( - &format!("/api/v1/catalogs/{}/test", name), + &format!("/api/v1/federated-catalogs/{}/test", name), &serde_json::json!({}), ) .await?; diff --git a/pangolin/pangolin_cli_admin/src/handlers/governance.rs b/pangolin/pangolin_cli_admin/src/handlers/governance.rs index 240a05e..0efd70b 100644 --- a/pangolin/pangolin_cli_admin/src/handlers/governance.rs +++ b/pangolin/pangolin_cli_admin/src/handlers/governance.rs @@ -132,7 +132,12 @@ async fn resolve_scope( .await .map_err(|e| CliError::ApiError(e.to_string()))?; if let Some(c) = catalogs.iter().find(|c| c["name"].as_str() == Some(path)) { - let id = c["id"].as_str().unwrap().to_string(); + // B_cli5: `.unwrap()` here panicked the whole CLI whenever the + // server omitted `id` - a crash rather than an error message. + let id = c["id"] + .as_str() + .ok_or_else(|| CliError::ApiError(format!("Catalog '{}' has no id", path)))? + .to_string(); Ok(serde_json::json!({ "type": "catalog", "catalog_id": id @@ -160,7 +165,12 @@ async fn resolve_scope( .iter() .find(|c| c["name"].as_str() == Some(cat_name)) { - let id = c["id"].as_str().unwrap().to_string(); + // B_cli5: `.unwrap()` here panicked the whole CLI whenever the + // server omitted `id` - a crash rather than an error message. + let id = c["id"] + .as_str() + .ok_or_else(|| CliError::ApiError(format!("Catalog '{}' has no id", path)))? + .to_string(); Ok(serde_json::json!({ "type": "namespace", "catalog_id": id, @@ -185,7 +195,10 @@ async fn resolve_scope( .iter() .find(|c| c["name"].as_str() == Some(cat_name)) { - c["id"].as_str().unwrap().to_string() + c["id"] + .as_str() + .ok_or_else(|| CliError::ApiError("Catalog has no id".to_string()))? + .to_string() } else { return Err(CliError::ApiError(format!( "Catalog '{}' not found", @@ -283,11 +296,46 @@ pub async fn handle_revoke_permission( action: String, resource: String, ) -> Result<(), CliError> { - let url = format!( - "/api/v1/permissions?role={}&action={}&resource={}", - role, action, resource - ); - let res = client.delete(&url).await?; + // B_cli5: this issued `DELETE /api/v1/permissions?role=..&action=..&resource=..`. + // The only delete route is `DELETE /api/v1/permissions/{id}` - there is no + // filter form - so the request 405'd and no permission was ever revoked. + // The grant has to be located first. + let res = client.get("/api/v1/permissions").await?; + if !res.status().is_success() { + return Err(CliError::ApiError(format!("Error: {}", res.status()))); + } + let permissions: Vec = res + .json() + .await + .map_err(|e| CliError::ApiError(e.to_string()))?; + + let target = permissions.iter().find(|p| { + let actions_match = p["actions"] + .as_array() + .map(|a| a.iter().any(|v| v.as_str() == Some(action.as_str()))) + .unwrap_or(false); + let scope_matches = p["scope"]["type"].as_str() == Some(resource.as_str()) + || p["scope"]["catalog_id"].as_str() == Some(resource.as_str()) + || p["scope"]["namespace"].as_str() == Some(resource.as_str()) + || p["scope"]["asset_id"].as_str() == Some(resource.as_str()) + || p["scope"]["tag_name"].as_str() == Some(resource.as_str()); + actions_match && scope_matches + }); + + let Some(permission) = target else { + return Err(CliError::ApiError(format!( + "No permission found granting '{}' on '{}' (role filter: {})", + action, resource, role + ))); + }; + + let id = permission["id"] + .as_str() + .ok_or_else(|| CliError::ApiError("Permission has no id".to_string()))?; + + let res = client + .delete(&format!("/api/v1/permissions/{}", id)) + .await?; if !res.status().is_success() { return Err(CliError::ApiError(format!("Error: {}", res.status()))); } @@ -464,11 +512,17 @@ pub async fn handle_request_access( // So permission arg is NOT in main.rs invocation for RequestAccess command. // I should remove `permission` arg from here. ) -> Result<(), CliError> { - let payload = serde_json::json!({ - "asset_id": asset_id, - "reason": reason - }); - let res = client.post("/api/v1/access-requests", &payload).await?; + // B_cli5: this POSTed to `/api/v1/access-requests`, which is registered for + // GET only - creating a request lives under the asset + // (`POST /api/v1/assets/{id}/access-requests`), and the asset id belongs in + // the path rather than the body. + let payload = serde_json::json!({ "reason": reason }); + let res = client + .post( + &format!("/api/v1/assets/{}/access-requests", asset_id), + &payload, + ) + .await?; if !res.status().is_success() { return Err(CliError::ApiError(format!("Failed: {}", res.status()))); } diff --git a/pangolin/pangolin_cli_admin/src/handlers/merge.rs b/pangolin/pangolin_cli_admin/src/handlers/merge.rs index 4ee93d5..5b3de51 100644 --- a/pangolin/pangolin_cli_admin/src/handlers/merge.rs +++ b/pangolin/pangolin_cli_admin/src/handlers/merge.rs @@ -18,7 +18,15 @@ pub async fn handle_list_merge_operations( query.push_str(&pag); } - let res = client.get(&format!("/api/v1/merges?{}", query)).await?; + // B_cli2: every merge command targeted `/api/v1/merges/...`, a prefix + // the router has never registered - all six 404'd. Listing is scoped by + // catalog in the path, not by a query parameter. + let res = client + .get(&format!( + "/api/v1/catalogs/{}/merge-operations?{}", + catalog, query + )) + .await?; if !res.status().is_success() { return Err(CliError::ApiError(format!("Error: {}", res.status()))); } @@ -58,7 +66,9 @@ pub async fn handle_get_merge_operation( client: &PangolinClient, id: String, ) -> Result<(), CliError> { - let res = client.get(&format!("/api/v1/merges/{}", id)).await?; + let res = client + .get(&format!("/api/v1/merge-operations/{}", id)) + .await?; if !res.status().is_success() { return Err(CliError::ApiError(format!( "Failed to get merge operation: {}", @@ -97,9 +107,9 @@ pub async fn handle_list_merge_conflicts( ) -> Result<(), CliError> { let q = pangolin_cli_common::utils::pagination_query(limit, offset); let path = if q.is_empty() { - format!("/api/v1/merges/{}/conflicts", id) + format!("/api/v1/merge-operations/{}/conflicts", id) } else { - format!("/api/v1/merges/{}/conflicts?{}", id, q) + format!("/api/v1/merge-operations/{}/conflicts?{}", id, q) }; let res = client.get(&path).await?; if !res.status().is_success() { @@ -176,7 +186,7 @@ pub async fn handle_resolve_merge_conflict( let res = client .post( - &format!("/api/v1/merges/conflicts/{}/resolve", conflict_id), + &format!("/api/v1/conflicts/{}/resolve", conflict_id), &payload, ) .await?; @@ -198,7 +208,7 @@ pub async fn handle_complete_merge_operation( ) -> Result<(), CliError> { let res = client .post( - &format!("/api/v1/merges/{}/complete", id), + &format!("/api/v1/merge-operations/{}/complete", id), &serde_json::json!({}), ) .await?; @@ -220,7 +230,7 @@ pub async fn handle_abort_merge_operation( ) -> Result<(), CliError> { let res = client .post( - &format!("/api/v1/merges/{}/abort", id), + &format!("/api/v1/merge-operations/{}/abort", id), &serde_json::json!({}), ) .await?; diff --git a/pangolin/pangolin_cli_admin/src/handlers/tokens.rs b/pangolin/pangolin_cli_admin/src/handlers/tokens.rs index e524e7e..83a42ac 100644 --- a/pangolin/pangolin_cli_admin/src/handlers/tokens.rs +++ b/pangolin/pangolin_cli_admin/src/handlers/tokens.rs @@ -5,7 +5,10 @@ use serde_json::Value; pub async fn handle_revoke_token(client: &PangolinClient) -> Result<(), CliError> { let res = client - .post("/api/v1/tokens/revoke", &serde_json::json!({})) + // B_cli4: `/api/v1/tokens/revoke` does not exist; the routes are under + // `/api/v1/auth/`. Both revocation commands 404'd, so the documented + // logout path never worked from the admin CLI. + .post("/api/v1/auth/revoke", &serde_json::json!({})) .await?; if !res.status().is_success() { @@ -32,7 +35,7 @@ pub async fn handle_revoke_token_by_id( ) -> Result<(), CliError> { let res = client .post( - &format!("/api/v1/tokens/revoke/{}", id), + &format!("/api/v1/auth/revoke/{}", id), &serde_json::json!({}), ) .await?; diff --git a/pangolin/pangolin_cli_admin/src/handlers/users.rs b/pangolin/pangolin_cli_admin/src/handlers/users.rs index 67ec2d3..e0e0036 100644 --- a/pangolin/pangolin_cli_admin/src/handlers/users.rs +++ b/pangolin/pangolin_cli_admin/src/handlers/users.rs @@ -5,9 +5,29 @@ use pangolin_cli_common::utils::print_table; use serde_json::Value; pub async fn handle_delete_user(client: &PangolinClient, username: String) -> Result<(), CliError> { - let res = client - .delete(&format!("/api/v1/users/{}", username)) - .await?; + // B_cli5: this interpolated the *username* into a route whose handler takes + // `Path`, so every `delete-user` returned 400 before reaching any + // logic. The CLI takes a username because that is what an operator knows, + // so resolve it to an id first. + let listing = client.get("/api/v1/users").await?; + if !listing.status().is_success() { + return Err(CliError::ApiError(format!( + "Failed to look up user: {}", + listing.status() + ))); + } + let users: Vec = listing + .json() + .await + .map_err(|e| CliError::ApiError(e.to_string()))?; + + let user_id = users + .iter() + .find(|u| u["username"].as_str() == Some(username.as_str())) + .and_then(|u| u["id"].as_str()) + .ok_or_else(|| CliError::ApiError(format!("User '{}' not found", username)))?; + + let res = client.delete(&format!("/api/v1/users/{}", user_id)).await?; if !res.status().is_success() { let status = res.status(); let t = res.text().await.unwrap_or_default(); @@ -96,8 +116,15 @@ pub async fn handle_update_user( ) -> Result<(), CliError> { let mut payload = serde_json::json!({}); - if let Some(u) = username { - payload["username"] = serde_json::Value::String(u); + // B_cli6: `--username` was accepted and written into the payload, but + // `UpdateUserRequest` has no `username` field - the server cannot rename a + // user - so serde dropped it and the CLI reported success for a rename that + // never happened. Saying so is better than pretending. + if username.is_some() { + return Err(CliError::ApiError( + "Renaming a user is not supported by the server; --username is not applied." + .to_string(), + )); } if let Some(e) = email { diff --git a/pangolin/pangolin_cli_admin/src/main.rs b/pangolin/pangolin_cli_admin/src/main.rs index 610da61..a1b1a90 100644 --- a/pangolin/pangolin_cli_admin/src/main.rs +++ b/pangolin/pangolin_cli_admin/src/main.rs @@ -33,7 +33,20 @@ async fn main() -> anyhow::Result<()> { let args = Args::parse(); // Load saved config - let config_manager = ConfigManager::new(args.profile.as_deref()).unwrap(); + // B_cli7: this was `.unwrap()`, so with neither $HOME nor $XDG_CONFIG_HOME + // set - a container, a systemd unit, a CI runner - the admin CLI panicked + // with a backtrace instead of saying what was wrong. The user CLI already + // used `?` here. + let config_manager = match ConfigManager::new(args.profile.as_deref()) { + Ok(m) => m, + Err(e) => { + eprintln!( + "Could not locate a config directory: {e}\n\ + Set $HOME or $XDG_CONFIG_HOME, or pass --url and --token." + ); + std::process::exit(1); + } + }; let mut config = config_manager.load().unwrap_or_default(); // Override URL if provided via Args or Env @@ -307,6 +320,10 @@ async fn main() -> anyhow::Result<()> { handlers::merge::handle_list_merge_conflicts(&client, merge_id, limit, offset) .await? } + // B_cli6: `--merge-id` is accepted and discarded. The resolve + // route is keyed on the conflict alone (`/api/v1/conflicts/{id}/resolve`), + // so the flag is genuinely redundant rather than lost - noted here + // so the discard is deliberate rather than a silent drop. AdminCommand::ResolveConflict { merge_id: _, conflict_id, @@ -429,7 +446,17 @@ async fn main() -> anyhow::Result<()> { AdminCommand::ListNamespaceTree { catalog } => { handlers::explorer::handle_namespace_tree(&client, catalog).await? } - _ => println!("Command not available in non-interactive mode."), + // B_cli6/B_cli7: this printed a note and fell through to + // `return Ok(())`, so a command the non-interactive path cannot run + // exited 0 - indistinguishable from success to any script or CI job. + // (`assign-role` and `revoke-user-role` land here.) + other => { + eprintln!( + "Command {:?} is only available in interactive mode.", + std::mem::discriminant(&other) + ); + std::process::exit(2); + } } return Ok(()); } diff --git a/pangolin/pangolin_cli_common/src/client.rs b/pangolin/pangolin_cli_common/src/client.rs index c85629d..c369f44 100644 --- a/pangolin/pangolin_cli_common/src/client.rs +++ b/pangolin/pangolin_cli_common/src/client.rs @@ -9,12 +9,25 @@ pub struct PangolinClient { pub config: CliConfig, } +/// Default per-request timeout. +/// +/// B_cli7: `Client::new()` has no timeout, so a hung or unreachable server left +/// the CLI blocked indefinitely with no output and no way out but Ctrl-C. +const REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); +const CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + impl PangolinClient { pub fn new(config: CliConfig) -> Self { - Self { - client: Client::new(), - config, - } + let client = Client::builder() + .timeout(REQUEST_TIMEOUT) + .connect_timeout(CONNECT_TIMEOUT) + .build() + // The builder only fails on a TLS backend problem, which would make + // every request fail anyway; falling back keeps `new` infallible for + // its callers. + .unwrap_or_else(|_| Client::new()); + + Self { client, config } } pub fn update_config(&mut self, config: CliConfig) { diff --git a/pangolin/pangolin_cli_common/src/config.rs b/pangolin/pangolin_cli_common/src/config.rs index 7fddd86..5646876 100644 --- a/pangolin/pangolin_cli_common/src/config.rs +++ b/pangolin/pangolin_cli_common/src/config.rs @@ -60,8 +60,22 @@ impl ConfigManager { .map_err(|e| CliError::ConfigError(e.to_string()))?; if let Some(parent) = self.config_path.parent() { fs::create_dir_all(parent)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = fs::set_permissions(parent, fs::Permissions::from_mode(0o700)); + } } fs::write(&self.config_path, content)?; + + // B_cli7: this file holds the auth token and was written at the process + // umask - typically 0644 - so any local account could read it. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&self.config_path, fs::Permissions::from_mode(0o600))?; + } + Ok(()) } } diff --git a/pangolin/pangolin_cli_user/Cargo.toml b/pangolin/pangolin_cli_user/Cargo.toml index d6c38e2..fa97ed4 100644 --- a/pangolin/pangolin_cli_user/Cargo.toml +++ b/pangolin/pangolin_cli_user/Cargo.toml @@ -15,7 +15,7 @@ categories = ["database"] workspace = true [dependencies] -pangolin_cli_common = { path = "../pangolin_cli_common", version = "0.6.0" } +pangolin_cli_common = { path = "../pangolin_cli_common", version = "0.7.0" } clap = { version = "4.4", features = ["derive", "env"] } tokio = { version = "1.0", features = ["full"] } rustyline = "14.0" diff --git a/pangolin/pangolin_cli_user/src/handlers.rs b/pangolin/pangolin_cli_user/src/handlers.rs index 5ec6403..66cef4f 100644 --- a/pangolin/pangolin_cli_user/src/handlers.rs +++ b/pangolin/pangolin_cli_user/src/handlers.rs @@ -77,11 +77,60 @@ pub async fn handle_list_catalogs( Ok(()) } -pub async fn handle_search(_client: &PangolinClient, query: String) -> Result<(), CliError> { - // Placeholder - Search API not fully implemented yet in handlers.rs of common? - // We will just mock it for now or hit the real endpoint if available - println!("Searching for '{}'...", query); - println!("(Search functionality pending backend index implementation)"); +/// Search assets. +/// +/// B_cli8: this printed "(Search functionality pending backend index +/// implementation)" and returned. Two search endpoints have existed the whole +/// time; the placeholder simply told users the feature was missing. +pub async fn handle_search(client: &PangolinClient, query: String) -> Result<(), CliError> { + let res = client + .get(&format!( + "/api/v1/assets/search?query={}", + // Percent-encode the term so a query containing & or = does not + // split into extra parameters. + query + .chars() + .map(|c| match c { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '.' | '~' => c.to_string(), + other => other + .to_string() + .bytes() + .map(|b| format!("%{:02X}", b)) + .collect::(), + }) + .collect::() + )) + .await?; + + if !res.status().is_success() { + let status = res.status(); + let text = res.text().await.unwrap_or_default(); + return Err(CliError::ApiError(format!("{} - {}", status, text))); + } + + let results: Vec = res + .json() + .await + .map_err(|e| CliError::ApiError(e.to_string()))?; + + if results.is_empty() { + println!("No assets matched '{}'.", query); + return Ok(()); + } + + let rows: Vec> = results + .iter() + .map(|r| { + vec![ + r["name"].as_str().unwrap_or("-").to_string(), + r["kind"].as_str().unwrap_or("-").to_string(), + r["catalog"].as_str().unwrap_or("-").to_string(), + r["namespace"].as_str().unwrap_or("-").to_string(), + ] + }) + .collect(); + + print_table(vec!["Name", "Type", "Catalog", "Namespace"], rows); Ok(()) } @@ -91,11 +140,11 @@ pub async fn handle_generate_code( table: String, ) -> Result<(), CliError> { let url = &client.config.base_url; - let token = client - .config - .auth_token - .as_deref() - .unwrap_or(""); + // B_cli8: this interpolated the *live* JWT into the generated snippet, which + // is explicitly copy-paste output - it went into scrollback, into whatever + // the user pasted it into, and often into a committed file. A placeholder is + // what a code sample should carry; the reader substitutes their own. + let token = ""; let parts: Vec<&str> = table.split('.').collect(); let (catalog, namespace, table_name) = if parts.len() == 3 { @@ -277,10 +326,14 @@ pub async fn handle_merge_branch( source: String, target: String, ) -> Result<(), CliError> { + // B_cli8: this sent `source`/`target`. `MergeBranchRequest` takes + // `source_branch`/`target_branch`, so the request 422'd - and with + // `deny_unknown_fields` on the server it now says which field is wrong + // rather than reporting a missing one. let body = serde_json::json!({ "catalog": catalog, - "source": source, - "target": target + "source_branch": source, + "target_branch": target }); let res = client.post("/api/v1/branches/merge", &body).await?; if !res.status().is_success() { @@ -381,12 +434,33 @@ pub async fn handle_list_requests( Ok(()) } +/// Request access to an asset. +/// +/// B_cli8: the whole body was `Ok(())`. The command accepted its arguments, +/// contacted nothing, and returned success - so a user could "request access", +/// see no error, and wait indefinitely for a review of a request that was never +/// created. pub async fn handle_request_access( - _client: &PangolinClient, - _resource: String, + client: &PangolinClient, + resource: String, _role: String, - _reason: String, + reason: String, ) -> Result<(), CliError> { + let body = serde_json::json!({ "reason": reason }); + let res = client + .post( + &format!("/api/v1/assets/{}/access-requests", resource), + &body, + ) + .await?; + + if !res.status().is_success() { + let status = res.status(); + let text = res.text().await.unwrap_or_default(); + return Err(CliError::ApiError(format!("{} - {}", status, text))); + } + + println!("✅ Access request submitted for asset {}.", resource); Ok(()) } @@ -396,13 +470,23 @@ pub async fn handle_get_token( description: String, expires_in: u32, ) -> Result<(), CliError> { + // B_cli8: this sent `description`, which `GenerateTokenRequest` does not + // have (it was silently dropped), and passed `client.config.tenant_id` + // straight through - which is `None` when no tenant is configured, so the + // server received `"tenant_id": null` and rejected it with an unhelpful + // parse error. `tenant_id` is required, so say so up front. + let _ = description; + + let Some(tenant_id) = client.config.tenant_id.clone() else { + return Err(CliError::ApiError( + "No tenant selected. Set one with `pangolin config set-tenant `.".to_string(), + )); + }; + let body = serde_json::json!({ - "description": description, - "expires_in_hours": expires_in * 24, // Endpoint expects hours? Wait, previous code said expires_in_days but endpoint is hours? - // token_handlers.rs uses `payload.expires_in_hours.unwrap_or(24)`. - // The CLI arg is `expires_in` (u32). The payload key in `handle_get_token` was `expires_in_days`. - // Let's check GenerateTokenRequest struct in `token_handlers.rs`. - "tenant_id": client.config.tenant_id, + // The server's field is `expires_in_hours`; the CLI flag is a day count. + "expires_in_hours": u64::from(expires_in) * 24, + "tenant_id": tenant_id, "username": client.config.username }); diff --git a/pangolin/pangolin_core/src/audit.rs b/pangolin/pangolin_core/src/audit.rs index 2091388..1f3ce09 100644 --- a/pangolin/pangolin_core/src/audit.rs +++ b/pangolin/pangolin_core/src/audit.rs @@ -206,6 +206,55 @@ pub enum AuditResult { Failure, } +/// Serialize an audit enum to the spelling that goes into storage. +/// +/// B22: the SQLite backend persisted `format!("{:?}", action)` - the Debug +/// spelling, `"CreateBranch"` - and then read it back by lowercasing to +/// `"createbranch"` and deserializing against serde's snake_case +/// (`"create_branch"`). Nothing matched, and the result was +/// `.unwrap_or(AuditAction::CreateCatalog)`, so nearly every multi-word action +/// in the SQLite audit trail was recorded as `CreateCatalog`. The audit log is +/// the one artefact that has to be right after an incident. +/// +/// Both directions now go through serde, so the write and read spellings cannot +/// drift apart again. +pub fn audit_enum_to_stored(value: &T) -> String { + serde_json::to_value(value) + .ok() + .and_then(|v| v.as_str().map(str::to_owned)) + .unwrap_or_else(|| format!("{:?}", value)) +} + +/// Parse an audit enum from its stored spelling. +/// +/// Accepts the canonical serde name and, for rows written before B22 was fixed, +/// the legacy Debug name. An unrecognised value is an error rather than a silent +/// substitution. +pub fn audit_enum_from_stored(stored: &str) -> Result +where + T: serde::de::DeserializeOwned, +{ + // Canonical snake_case. + if let Ok(v) = serde_json::from_value::(serde_json::Value::String(stored.to_string())) { + return Ok(v); + } + + // Legacy Debug spelling: "CreateBranch" -> "create_branch". + let mut snake = String::with_capacity(stored.len() + 4); + for (i, ch) in stored.chars().enumerate() { + if ch.is_ascii_uppercase() { + if i != 0 { + snake.push('_'); + } + snake.push(ch.to_ascii_lowercase()); + } else { + snake.push(ch); + } + } + serde_json::from_value::(serde_json::Value::String(snake)) + .map_err(|_| format!("unknown audit enum value {stored:?}")) +} + /// Filter for querying audit logs #[derive(Debug, Clone, Deserialize)] #[cfg_attr(feature = "utoipa", derive(ToSchema))] diff --git a/pangolin/pangolin_core/src/iceberg_metadata.rs b/pangolin/pangolin_core/src/iceberg_metadata.rs index f925e58..51003fb 100644 --- a/pangolin/pangolin_core/src/iceberg_metadata.rs +++ b/pangolin/pangolin_core/src/iceberg_metadata.rs @@ -3,6 +3,10 @@ use std::collections::HashMap; use utoipa::ToSchema; use uuid::Uuid; +/// Partition field ids are assigned from 1000 upward by the Iceberg spec, so an +/// unpartitioned table's highest *assigned* partition id is 999. +pub const PARTITION_FIELD_ID_START: i32 = 1000; + #[derive(Debug, Serialize, Deserialize, Clone, ToSchema)] #[serde(rename_all = "kebab-case")] pub struct TableMetadata { @@ -14,8 +18,25 @@ pub struct TableMetadata { pub last_column_id: i32, pub current_schema_id: i32, pub schemas: Vec, + /// The default partition spec id. + /// + /// The spec field is `default-spec-id`. Under the struct's kebab-case rule + /// this serialized as `current-partition-spec-id` (B11), which no + /// spec-conformant reader looks for - metadata Pangolin wrote could not be + /// read as v2 metadata by an external engine reading the file directly, and + /// a conformant engine's metadata could not round-trip in. The alias keeps + /// already-written Pangolin files parseable. + #[serde(rename = "default-spec-id", alias = "current-partition-spec-id")] pub current_partition_spec_id: i32, pub partition_specs: Vec, + /// Highest assigned partition field id. + /// + /// Required by the v2 spec and missing entirely before (B12); Java-based + /// readers reject metadata without it. Defaulted on read so files Pangolin + /// wrote earlier still parse, and recomputed from `partition_specs` by + /// [`TableMetadata::recompute_last_partition_id`]. + #[serde(default = "default_last_partition_id")] + pub last_partition_id: i32, pub default_sort_order_id: i32, pub sort_orders: Vec, pub properties: Option>, @@ -32,6 +53,28 @@ pub struct TableMetadata { pub refs: Option>, } +fn default_last_partition_id() -> i32 { + PARTITION_FIELD_ID_START - 1 +} + +impl TableMetadata { + /// Recompute `last_partition_id` from the partition specs. + /// + /// Called after any change to `partition_specs` so the field stays true; + /// the spec defines it as the highest partition field id ever assigned, so + /// it only ever moves up. + pub fn recompute_last_partition_id(&mut self) { + let highest = self + .partition_specs + .iter() + .flat_map(|spec| spec.fields.iter()) + .map(|f| f.field_id) + .max() + .unwrap_or(PARTITION_FIELD_ID_START - 1); + self.last_partition_id = self.last_partition_id.max(highest); + } +} + /// A named branch or tag pointing at a snapshot. #[derive(Debug, Serialize, Deserialize, Clone, ToSchema, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] @@ -51,11 +94,79 @@ pub struct SnapshotReference { #[derive(Debug, Serialize, Deserialize, Clone, ToSchema)] #[serde(rename_all = "kebab-case")] pub struct Schema { + /// Always `"struct"`. + /// + /// Spec schemas *are* struct types and conformant writers emit this; it was + /// missing entirely (B14), so strict readers rejected the schema object. + #[serde(rename = "type", default = "struct_type_name")] + pub type_: String, + /// Defaulted because a `createTable` request body may legitimately omit it - + /// the server assigns the id. Without the default, deserializing an incoming + /// schema (which `create_table` now does instead of hand-parsing it, B16f) + /// would reject a spec-legal request. + #[serde(default)] pub schema_id: i32, + /// Omitted when absent rather than written as an explicit `null` (B14) - + /// some strict parsers reject `"identifier-field-ids": null`. + #[serde(default, skip_serializing_if = "Option::is_none")] pub identifier_field_ids: Option>, pub fields: Vec, } +fn struct_type_name() -> String { + Schema::STRUCT.to_string() +} + +impl Schema { + /// The only legal value of a schema's `type` field. + pub const STRUCT: &'static str = "struct"; + + /// Highest field id in this schema, including nested fields. + /// + /// `last-column-id` must cover nested ids too; computing it from the + /// top-level fields alone (as `create_table` used to) understates it for any + /// schema containing a struct, list or map. + pub fn max_field_id(&self) -> i32 { + fn walk(t: &Type, acc: &mut i32) { + match t { + Type::Primitive(_) => {} + Type::Struct { fields, .. } => { + for f in fields { + *acc = (*acc).max(f.id); + walk(&f.field_type, acc); + } + } + Type::List { + element_id, + element, + .. + } => { + *acc = (*acc).max(*element_id); + walk(element, acc); + } + Type::Map { + key_id, + key, + value_id, + value, + .. + } => { + *acc = (*acc).max(*key_id).max(*value_id); + walk(key, acc); + walk(value, acc); + } + } + } + + let mut max = 0; + for field in &self.fields { + max = max.max(field.id); + walk(&field.field_type, &mut max); + } + max + } +} + #[derive(Debug, Serialize, Deserialize, Clone, ToSchema)] #[serde(rename_all = "kebab-case")] pub struct NestedField { @@ -64,6 +175,8 @@ pub struct NestedField { pub required: bool, #[serde(rename = "type")] pub field_type: Type, + /// Omitted when absent rather than serialized as `"doc": null` (B14). + #[serde(default, skip_serializing_if = "Option::is_none")] pub doc: Option, } diff --git a/pangolin/pangolin_core/src/model.rs b/pangolin/pangolin_core/src/model.rs index 0b94f89..ca18774 100644 --- a/pangolin/pangolin_core/src/model.rs +++ b/pangolin/pangolin_core/src/model.rs @@ -111,6 +111,73 @@ pub enum AssetType { Other, } +impl AssetType { + /// The canonical persisted spelling, e.g. `"DELTA_TABLE"`. + /// + /// B7: all three persistent backends wrote `format!("{:?}", asset.kind)` - + /// the *Debug* spelling - and read it back through a match that recognised + /// only `IcebergTable` and `View`, defaulting everything else to + /// `IcebergTable`. A `DeltaTable`, `MlModel`, `Lance` or any of the other + /// 13 variants round-tripped as an Iceberg table, silently defeating the + /// headline "tracks any lakehouse asset type" feature. Going through serde + /// means the enum's own rename policy is the single source of truth, and + /// adding a variant cannot reintroduce the drift. + pub fn as_stored_str(&self) -> String { + serde_json::to_value(self) + .ok() + .and_then(|v| v.as_str().map(str::to_owned)) + // The enum is a plain unit-variant enum, so serialization cannot + // fail; the Debug spelling is a belt-and-braces fallback only. + .unwrap_or_else(|| format!("{:?}", self)) + } + + /// Parse a persisted asset type. + /// + /// Accepts the canonical serde spelling *and* the legacy Debug spelling, so + /// rows written before this fix keep loading. An unrecognised value is an + /// error rather than a silent downgrade - a wrong asset type is worse than a + /// loud failure, because it misroutes every reader downstream. + pub fn from_stored_str(value: &str) -> Result { + if let Ok(parsed) = + serde_json::from_value::(serde_json::Value::String(value.to_string())) + { + return Ok(parsed); + } + + // Legacy Debug spellings, e.g. "IcebergTable". + for candidate in Self::all() { + if format!("{:?}", candidate) == value { + return Ok(candidate); + } + } + + Err(format!("unknown asset type {value:?}")) + } + + /// Every variant, for round-trip tests and legacy parsing. + pub fn all() -> Vec { + vec![ + Self::IcebergTable, + Self::DeltaTable, + Self::HudiTable, + Self::ParquetTable, + Self::CsvTable, + Self::JsonTable, + Self::View, + Self::MlModel, + Self::ApachePaimon, + Self::Vortex, + Self::Lance, + Self::Nimble, + Self::Directory, + Self::VideoFile, + Self::ImageFile, + Self::DbConnString, + Self::Other, + ] + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Asset { pub id: Uuid, diff --git a/pangolin/pangolin_core/src/user.rs b/pangolin/pangolin_core/src/user.rs index 3c6eaac..2a88a5b 100644 --- a/pangolin/pangolin_core/src/user.rs +++ b/pangolin/pangolin_core/src/user.rs @@ -56,6 +56,14 @@ pub struct UserSession { pub role: UserRole, pub issued_at: DateTime, pub expires_at: DateTime, + /// The `jti` of the bearer token that produced this session, when there was + /// one. Revocation is keyed by `jti`, so logout has to revoke *this* id - + /// revoking `user_id` (as it once did) blacklists an id no token ever + /// carries, and the token keeps working until it expires naturally. + /// + /// `None` for sessions with no underlying JWT (API keys, root basic auth). + #[serde(default)] + pub token_id: Option, } /// Service user for API key authentication diff --git a/pangolin/pangolin_store/Cargo.toml b/pangolin/pangolin_store/Cargo.toml index f8b001b..4f9ac2f 100644 --- a/pangolin/pangolin_store/Cargo.toml +++ b/pangolin/pangolin_store/Cargo.toml @@ -15,7 +15,7 @@ categories = ["database"] workspace = true [dependencies] -pangolin_core = { path = "../pangolin_core", version = "0.6.0", features = ["sqlx"] } +pangolin_core = { path = "../pangolin_core", version = "0.7.0", features = ["sqlx"] } object_store = { workspace = true } async-trait = { workspace = true } dashmap = { workspace = true } @@ -30,7 +30,7 @@ bytes = { workspace = true } tracing = { workspace = true } chrono = { workspace = true } aws-sdk-s3 = { version = "1.0", features = ["behavior-version-latest"] } -aws-config = { version = "1.0", features = ["behavior-version-latest"] } +aws-config = { version = "1.10", features = ["behavior-version-latest"] } aws-credential-types = "1.0" sqlx = { version = "0.8.6", features = ["runtime-tokio-rustls", "postgres", "sqlite", "uuid", "chrono", "json"] } mongodb = "3.4.1" diff --git a/pangolin/pangolin_store/migrations/20260810000000_add_business_metadata.sql b/pangolin/pangolin_store/migrations/20260810000000_add_business_metadata.sql new file mode 100644 index 0000000..cf2382e --- /dev/null +++ b/pangolin/pangolin_store/migrations/20260810000000_add_business_metadata.sql @@ -0,0 +1,41 @@ +-- Business metadata for assets: descriptions, tags, discoverability. +-- +-- This table was never created on Postgres. `search_assets` has always run +-- `LEFT JOIN business_metadata m ON a.id = m.asset_id`, so *every* asset search +-- on the Postgres backend failed outright with +-- `relation "business_metadata" does not exist` - not an empty result, a hard +-- SQL error. The three CRUD methods were not implemented either, so the trait's +-- "Operation not supported by this store" default answered them. +-- +-- Found by running the cross-backend parity suite against a live Postgres for +-- the first time. SQLite, Mongo and the memory backend all had it. +-- +-- Column types follow the Postgres conventions already in this schema: UUID for +-- ids, JSONB for structured values, TIMESTAMPTZ for times - rather than the +-- TEXT/INTEGER encoding SQLite uses. + +CREATE TABLE IF NOT EXISTS business_metadata ( + id UUID PRIMARY KEY, + asset_id UUID NOT NULL UNIQUE, + description TEXT, + tags JSONB NOT NULL DEFAULT '[]'::jsonb, + properties JSONB NOT NULL DEFAULT '{}'::jsonb, + discoverable BOOLEAN NOT NULL DEFAULT FALSE, + created_by UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_by UUID NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT fk_business_metadata_asset + FOREIGN KEY (asset_id) REFERENCES assets(id) ON DELETE CASCADE +); + +-- `search_assets` joins on asset_id and filters on description; the unique +-- constraint above already indexes asset_id. +CREATE INDEX IF NOT EXISTS idx_business_metadata_discoverable + ON business_metadata(discoverable) + WHERE discoverable; + +-- Tag filtering uses the containment operator (`tags @> $n::jsonb`), which GIN +-- serves directly. +CREATE INDEX IF NOT EXISTS idx_business_metadata_tags + ON business_metadata USING GIN (tags); diff --git a/pangolin/pangolin_store/sql/sqlite_schema.sql b/pangolin/pangolin_store/sql/sqlite_schema.sql index f3d7ab5..46bf7fb 100644 --- a/pangolin/pangolin_store/sql/sqlite_schema.sql +++ b/pangolin/pangolin_store/sql/sqlite_schema.sql @@ -113,14 +113,29 @@ CREATE TABLE IF NOT EXISTS commits ( CREATE INDEX IF NOT EXISTS idx_commits_tenant ON commits(tenant_id); -- Audit Logs +-- The columns here had drifted out of step with the code that writes them. +-- The table declared the original (actor, resource, details) shape while +-- `sqlite/audit_logs.rs` inserted the full AuditLogEntry - user_id, username, +-- resource_type, resource_id, resource_name, ip_address, user_agent, result, +-- error_message, metadata. Every `log_audit_event` on SQLite therefore failed +-- at runtime with "table audit_logs has no column named user_id", so the +-- backend recorded *no* audit trail at all. Found by the cross-backend parity +-- suite, which is the first thing to exercise audit logging on SQLite. CREATE TABLE IF NOT EXISTS audit_logs ( id TEXT PRIMARY KEY, tenant_id TEXT NOT NULL, - timestamp INTEGER NOT NULL, - actor TEXT NOT NULL, + user_id TEXT, + username TEXT NOT NULL, action TEXT NOT NULL, - resource TEXT NOT NULL, - details TEXT, -- JSON + resource_type TEXT NOT NULL, + resource_id TEXT, + resource_name TEXT NOT NULL, + timestamp INTEGER NOT NULL, + ip_address TEXT, + user_agent TEXT, + result TEXT NOT NULL, + error_message TEXT, + metadata TEXT, -- JSON FOREIGN KEY (tenant_id) REFERENCES tenants(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_audit_logs_tenant_ts ON audit_logs(tenant_id, timestamp DESC); diff --git a/pangolin/pangolin_store/src/azure_signer.rs b/pangolin/pangolin_store/src/azure_signer.rs index cdc1c38..f2c97bc 100644 --- a/pangolin/pangolin_store/src/azure_signer.rs +++ b/pangolin/pangolin_store/src/azure_signer.rs @@ -36,12 +36,16 @@ impl AzureSigner { pub fn parse_azure_path(path: &str) -> Result<(String, String)> { // Parse az://container/blob/path OR abfs://container@account/path OR abfss://container@account/path - let (scheme, rest) = if path.starts_with("az://") { - ("az", &path[5..]) - } else if path.starts_with("abfs://") { - ("abfs", &path[7..]) - } else if path.starts_with("abfss://") { - ("abfss", &path[8..]) + // `strip_prefix` rather than `starts_with` plus a hand-counted `&path[n..]`, + // where the index and the prefix have to be kept in step by hand. The three + // prefixes are mutually exclusive - `abfss://` does not start with + // `abfs://` - so the order of the arms does not matter. + let (scheme, rest) = if let Some(rest) = path.strip_prefix("az://") { + ("az", rest) + } else if let Some(rest) = path.strip_prefix("abfss://") { + ("abfss", rest) + } else if let Some(rest) = path.strip_prefix("abfs://") { + ("abfs", rest) } else { return Err(anyhow::anyhow!("Invalid Azure path: {}", path)); }; diff --git a/pangolin/pangolin_store/src/file_delete.rs b/pangolin/pangolin_store/src/file_delete.rs new file mode 100644 index 0000000..a2a59a0 --- /dev/null +++ b/pangolin/pangolin_store/src/file_delete.rs @@ -0,0 +1,64 @@ +//! Best-effort deletion of a single object at a warehouse location. +//! +//! Added for the metadata-orphan problem (B16d/B16g): the Iceberg commit loop +//! writes a full metadata file *before* attempting the compare-and-swap that +//! publishes it. On a lost CAS it retried and wrote a fresh file, abandoning the +//! previous one, and up to five orphans were left behind on a final give-up. +//! Orphaned metadata files are indistinguishable from live ones from the +//! outside, so they cannot be reaped later by inspection alone - they have to be +//! removed at the moment the writer knows they are unreferenced. +//! +//! Every backend routes here so the four of them cannot drift apart, which is +//! the failure mode most of the storage-layer audit findings share. + +use anyhow::Result; +use object_store::ObjectStore; +use std::collections::HashMap; + +/// Delete `location`, resolving credentials from `storage_config` when the +/// location points at object storage. +/// +/// A missing object is *not* an error: callers use this to clean up after a +/// failure, and the object may never have been written. +pub async fn delete_location( + storage_config: Option<&HashMap>, + location: &str, +) -> Result<()> { + if let Some(rest) = location.strip_prefix("file://") { + return match tokio::fs::remove_file(rest).await { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(anyhow::anyhow!("Failed to delete {}: {}", rest, e)), + }; + } + + let is_object_store = location.starts_with("s3://") + || location.starts_with("az://") + || location.starts_with("abfs://") + || location.starts_with("gs://"); + + if !is_object_store { + // A bare local path. + return match tokio::fs::remove_file(location).await { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(anyhow::anyhow!("Failed to delete {}: {}", location, e)), + }; + } + + let empty = HashMap::new(); + let config = storage_config.unwrap_or(&empty); + let store = crate::object_store_factory::create_object_store(config, location)?; + + let key = location + .split_once("://") + .and_then(|(_, rest)| rest.split_once('/')) + .map(|(_, key)| key) + .unwrap_or(location); + + match store.delete(&object_store::path::Path::from(key)).await { + Ok(()) => Ok(()), + Err(object_store::Error::NotFound { .. }) => Ok(()), + Err(e) => Err(anyhow::anyhow!("Failed to delete {}: {}", location, e)), + } +} diff --git a/pangolin/pangolin_store/src/lib.rs b/pangolin/pangolin_store/src/lib.rs index f645eba..05427c2 100644 --- a/pangolin/pangolin_store/src/lib.rs +++ b/pangolin/pangolin_store/src/lib.rs @@ -1,5 +1,6 @@ pub mod aws_utils; pub mod azure_signer; +pub mod file_delete; pub mod gcp_signer; pub mod memory; pub mod mongo; @@ -28,6 +29,7 @@ pub use sqlite::SqliteStore; pub mod metadata_cache; pub mod object_store_cache; pub mod object_store_factory; +pub mod search; pub use metadata_cache::MetadataCache; pub use object_store_cache::ObjectStoreCache; pub use signer::SignerImpl; @@ -118,6 +120,7 @@ pub trait CatalogStore: Send + Sync + Signer { catalog_name: &str, namespace: Vec, ) -> Result<()>; + /// Merge `properties` into the namespace's existing properties. async fn update_namespace_properties( &self, tenant_id: Uuid, @@ -125,6 +128,19 @@ pub trait CatalogStore: Send + Sync + Signer { namespace: Vec, properties: std::collections::HashMap, ) -> Result<()>; + /// Replace the namespace's properties with `properties`. + /// + /// The merge-only method above cannot express a removal, which is why the + /// Iceberg `updateProperties` handler silently dropped every `removals` + /// entry while reporting success (B16h). Errors when the namespace does not + /// exist, so "updated nothing" and "no such namespace" are distinguishable. + async fn replace_namespace_properties( + &self, + tenant_id: Uuid, + catalog_name: &str, + namespace: Vec, + properties: std::collections::HashMap, + ) -> Result<()>; // Asset Operations async fn create_asset( @@ -340,6 +356,13 @@ pub trait CatalogStore: Send + Sync + Signer { // Generic File IO (for metadata files) async fn read_file(&self, location: &str) -> Result>; async fn write_file(&self, location: &str, content: Vec) -> Result<()>; + /// Delete a single file, resolving warehouse credentials from the location. + /// + /// Needed so the commit path can reclaim the metadata file it wrote before + /// losing a compare-and-swap (B16d), and so `create_table` can clean up + /// after a failed registration (B16g). Deleting something that is not there + /// succeeds. + async fn delete_file(&self, location: &str) -> Result<()>; // Maintenance Operations async fn expire_snapshots( diff --git a/pangolin/pangolin_store/src/memory.rs.bak b/pangolin/pangolin_store/src/memory.rs.bak deleted file mode 100644 index 8ad28a6..0000000 --- a/pangolin/pangolin_store/src/memory.rs.bak +++ /dev/null @@ -1,1831 +0,0 @@ -use crate::CatalogStore; -use crate::signer::{Signer, Credentials}; -use async_trait::async_trait; -use dashmap::DashMap; -use chrono::Utc; -use pangolin_core::model::{ - Catalog, CatalogType, Namespace, Warehouse, Asset, Commit, Branch, Tag, BranchType, Tenant, - VendingStrategy, SystemSettings, SyncStats -}; -use pangolin_core::user::User; -use pangolin_core::permission::{Role, UserRole, Permission}; -use pangolin_core::audit::AuditLogEntry; -use uuid::Uuid; -use anyhow::Result; -use std::sync::Arc; -use pangolin_core::business_metadata::{BusinessMetadata, AccessRequest}; - -use tracing; - - -#[derive(Clone)] -pub struct MemoryStore { - tenants: Arc>, - warehouses: Arc>, // Key: (TenantId, WarehouseName) - catalogs: Arc>, // Key: (TenantId, CatalogName) - namespaces: Arc>, // Key: (TenantId, CatalogName, NamespaceString) - // Key: (TenantId, CatalogName, BranchName, NamespaceString, AssetName) - assets: Arc>, - branches: Arc>, // Key: (TenantId, CatalogName, BranchName) - tags: Arc>, // Key: (TenantId, CatalogName, TagName) - commits: Arc>, // Key: (TenantId, CommitId) - files: Arc>>, // Key: Location - audit_events: Arc>>, // Changed to DashMap for consistency, key tenant_id - // New fields - users: Arc>, - roles: Arc>, - signer: crate::signer::SignerImpl, - user_roles: Arc>, - permissions: Arc>, - business_metadata: Arc>, - access_requests: Arc>, - service_users: Arc>, - merge_operations: Arc>, - merge_conflicts: Arc>, - // Optimization: Direct lookup for assets by ID - // Key: AssetID, Value: (CatalogName, Namespace, Branch, AssetName) - assets_by_id: Arc, Option, String)>>, - // Token revocation - revoked_tokens: Arc>, - // Active tokens (for listing - in real DB this would be querying sessions/tokens table) - // We only store TokenInfo here. The actual validation is stateless JWT + Revocation Check. - // But to "List Tokens", we need to store them. - active_tokens: Arc>, - // System Settings: Tenant -> Settings - system_settings: Arc>, - // Federated Stats: (TenantId, CatalogName) -> SyncStats - federated_stats: Arc>, - // Performance optimizations - object_store_cache: crate::ObjectStoreCache, - metadata_cache: crate::MetadataCache, -} - -impl MemoryStore { - pub fn new() -> Self { - Self { - tenants: Arc::new(DashMap::new()), - warehouses: Arc::new(DashMap::new()), - catalogs: Arc::new(DashMap::new()), - namespaces: Arc::new(DashMap::new()), - assets: Arc::new(DashMap::new()), - branches: Arc::new(DashMap::new()), - tags: Arc::new(DashMap::new()), - commits: Arc::new(DashMap::new()), - files: Arc::new(DashMap::new()), - audit_events: Arc::new(DashMap::new()), - users: Arc::new(DashMap::new()), - roles: Arc::new(DashMap::new()), - signer: crate::signer::SignerImpl::new("memory_key".to_string()), - user_roles: Arc::new(DashMap::new()), - permissions: Arc::new(DashMap::new()), - business_metadata: Arc::new(DashMap::new()), - access_requests: Arc::new(DashMap::new()), - service_users: Arc::new(DashMap::new()), - merge_operations: Arc::new(DashMap::new()), - merge_conflicts: Arc::new(DashMap::new()), - assets_by_id: Arc::new(DashMap::new()), - revoked_tokens: Arc::new(DashMap::new()), - active_tokens: Arc::new(DashMap::new()), - system_settings: Arc::new(DashMap::new()), - federated_stats: Arc::new(DashMap::new()), - object_store_cache: crate::ObjectStoreCache::new(), - metadata_cache: crate::MetadataCache::default(), - } - } -} - - -#[async_trait] -impl CatalogStore for MemoryStore { - async fn create_tenant(&self, tenant: Tenant) -> Result<()> { - self.tenants.insert(tenant.id, tenant); - Ok(()) - } - - async fn get_tenant(&self, tenant_id: Uuid) -> Result> { - if let Some(t) = self.tenants.get(&tenant_id) { - Ok(Some(t.value().clone())) - } else { - Ok(None) - } - } - - async fn list_tenants(&self) -> Result> { - let tenants = self.tenants.iter().map(|t| t.value().clone()).collect(); - Ok(tenants) - } - - async fn update_tenant(&self, tenant_id: Uuid, updates: pangolin_core::model::TenantUpdate) -> Result { - if let Some(mut tenant) = self.tenants.get_mut(&tenant_id) { - if let Some(name) = updates.name { - tenant.name = name; - } - if let Some(properties) = updates.properties { - tenant.properties.extend(properties); - } - Ok(tenant.clone()) - } else { - Err(anyhow::anyhow!("Tenant not found")) - } - } - - async fn delete_tenant(&self, tenant_id: Uuid) -> Result<()> { - if self.tenants.remove(&tenant_id).is_some() { - // TODO: Cascade delete warehouses and catalogs - Ok(()) - } else { - Err(anyhow::anyhow!("Tenant not found")) - } - } - - async fn create_warehouse(&self, tenant_id: Uuid, warehouse: Warehouse) -> Result<()> { - let key = (tenant_id, warehouse.name.clone()); - self.warehouses.insert(key, warehouse); - Ok(()) - } - - async fn get_warehouse(&self, tenant_id: Uuid, name: String) -> Result> { - let key = (tenant_id, name); - if let Some(w) = self.warehouses.get(&key) { - Ok(Some(w.value().clone())) - } else { - Ok(None) - } - } - - async fn list_warehouses(&self, tenant_id: Uuid) -> Result> { - let warehouses = self.warehouses.iter() - .filter(|r| r.key().0 == tenant_id) - .map(|r| r.value().clone()) - .collect(); - Ok(warehouses) - } - - async fn update_warehouse(&self, tenant_id: Uuid, name: String, updates: pangolin_core::model::WarehouseUpdate) -> Result { - let key = (tenant_id, name.clone()); - if let Some(mut warehouse) = self.warehouses.get_mut(&key) { - if let Some(new_name) = updates.name { - // If name is changing, we need to remove old key and insert with new key - let mut w = warehouse.clone(); - w.name = new_name.clone(); - drop(warehouse); // Release the mutable reference - self.warehouses.remove(&key); - let new_key = (tenant_id, new_name); - self.warehouses.insert(new_key, w.clone()); - return Ok(w); - } - if let Some(config) = updates.storage_config { - warehouse.storage_config.extend(config); - } - if let Some(use_sts) = updates.use_sts { - warehouse.use_sts = use_sts; - } - Ok(warehouse.clone()) - } else { - Err(anyhow::anyhow!("Warehouse '{}' not found", name)) - } - } - - async fn delete_warehouse(&self, tenant_id: Uuid, name: String) -> Result<()> { - let key = (tenant_id, name.clone()); - - if self.warehouses.remove(&key).is_some() { - Ok(()) - } else { - Err(anyhow::anyhow!("Warehouse '{}' not found", name)) - } - } - - async fn create_catalog(&self, tenant_id: Uuid, catalog: Catalog) -> Result<()> { - let key = (tenant_id, catalog.name.clone()); - self.catalogs.insert(key, catalog); - Ok(()) - } - - async fn get_catalog(&self, tenant_id: Uuid, name: String) -> Result> { - let key = (tenant_id, name); - if let Some(c) = self.catalogs.get(&key) { - Ok(Some(c.value().clone())) - } else { - Ok(None) - } - } - - async fn list_catalogs(&self, tenant_id: Uuid) -> Result> { - let mut catalogs = Vec::new(); - for entry in self.catalogs.iter() { - let (tid, _) = entry.key(); - if *tid == tenant_id { - catalogs.push(entry.value().clone()); - } - } - Ok(catalogs) - } - - async fn update_catalog(&self, tenant_id: Uuid, name: String, updates: pangolin_core::model::CatalogUpdate) -> Result { - let key = (tenant_id, name.clone()); - if let Some(mut catalog) = self.catalogs.get_mut(&key) { - if let Some(warehouse_name) = updates.warehouse_name { - catalog.warehouse_name = Some(warehouse_name); - } - if let Some(storage_location) = updates.storage_location { - catalog.storage_location = Some(storage_location); - } - if let Some(properties) = updates.properties { - catalog.properties.extend(properties); - } - Ok(catalog.clone()) - } else { - Err(anyhow::anyhow!("Catalog '{}' not found", name)) - } - } - - async fn delete_catalog(&self, tenant_id: Uuid, name: String) -> Result<()> { - let key = (tenant_id, name.clone()); - if self.catalogs.remove(&key).is_some() { - // Cascade delete: Remove all associated resources - // Note: In a real database, this would be handled by foreign keys. - // In MemoryStore, we must manually iterate and remove. - - // Remove Namespaces - self.namespaces.retain(|k, _| !(k.0 == tenant_id && k.1 == name)); - - // Remove Assets - self.assets.retain(|k, _| !(k.0 == tenant_id && k.1 == name)); - - // Remove Branches - self.branches.retain(|k, _| !(k.0 == tenant_id && k.1 == name)); - - // Remove Tags - self.tags.retain(|k, _| !(k.0 == tenant_id && k.1 == name)); - - // Clean up assets_by_id index - // This is expensive O(N) since we have to scan the whole index - // But deletion is rare. - self.assets_by_id.retain(|_, v| v.0 != name); - - Ok(()) - } else { - Err(anyhow::anyhow!("Catalog not found")) - } - } - - async fn create_namespace(&self, tenant_id: Uuid, catalog_name: &str, namespace: Namespace) -> Result<()> { - let key = (tenant_id, catalog_name.to_string(), namespace.to_string()); - self.namespaces.insert(key, namespace); - Ok(()) - } - - async fn list_namespaces(&self, tenant_id: Uuid, catalog_name: &str, parent: Option) -> Result> { - let parent_prefix = parent.unwrap_or_default(); - tracing::info!("DEBUG_MEM: list_namespaces tid={} cat={} parent='{}'", tenant_id, catalog_name, parent_prefix); - let mut namespaces = Vec::new(); - for entry in self.namespaces.iter() { - let (tid, cat, ns_str) = entry.key(); - tracing::info!("DEBUG_MEM: Checking entry tid={} cat={} ns={}", tid, cat, ns_str); - if *tid == tenant_id && cat == catalog_name && (parent_prefix.is_empty() || ns_str.starts_with(&parent_prefix)) { - namespaces.push(entry.value().clone()); - } - } - tracing::info!("DEBUG_MEM: Found {} namespaces", namespaces.len()); - Ok(namespaces) - } - - async fn get_namespace(&self, tenant_id: Uuid, catalog_name: &str, namespace: Vec) -> Result> { - let key = (tenant_id, catalog_name.to_string(), namespace.join(".")); - if let Some(n) = self.namespaces.get(&key) { - Ok(Some(n.value().clone())) - } else { - Ok(None) - } - } - - async fn delete_namespace(&self, tenant_id: Uuid, catalog_name: &str, namespace: Vec) -> Result<()> { - let ns_str = namespace.join("\x1F"); - let key = (tenant_id, catalog_name.to_string(), ns_str); - if self.namespaces.remove(&key).is_some() { - Ok(()) - } else { - Err(anyhow::anyhow!("Namespace not found")) - } - } - - async fn update_namespace_properties(&self, tenant_id: Uuid, catalog_name: &str, namespace: Vec, properties: std::collections::HashMap) -> Result<()> { - let ns_str = namespace.join("\x1F"); - let key = (tenant_id, catalog_name.to_string(), ns_str); - - if let Some(mut ns) = self.namespaces.get_mut(&key) { - ns.properties.extend(properties); - Ok(()) - } else { - Err(anyhow::anyhow!("Namespace not found")) - } - } - - async fn create_asset(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec, asset: Asset) -> Result<()> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - - // 1. Insert Asset - let asset_full_name = format!("{}.{}", namespace.join("."), asset.name); - let key = (tenant_id, catalog_name.to_string(), branch_name.clone(), namespace.join("\x1F"), asset.name.clone()); - self.assets.insert(key, asset.clone()); - - // 2. Update optimized lookups - self.assets_by_id.insert(asset.id, (catalog_name.to_string(), namespace.clone(), Some(branch_name.clone()), asset.name.clone())); - - // 2. Ensure Branch Exists and Update Asset List - let mut branch_obj = self.get_branch(tenant_id, catalog_name, branch_name.clone()).await? - .unwrap_or_else(|| { - Branch { - name: branch_name.clone(), - head_commit_id: None, - branch_type: BranchType::Experimental, - assets: vec![], - } - }); - - if !branch_obj.assets.contains(&asset_full_name) { - branch_obj.assets.push(asset_full_name); - self.create_branch(tenant_id, catalog_name, branch_obj).await?; - } - - Ok(()) - } - - async fn get_asset_by_id(&self, tenant_id: Uuid, asset_id: Uuid) -> Result)>> { - if let Some(entry) = self.assets_by_id.get(&asset_id) { - let (catalog_name, namespace, branch, name) = entry.value().clone(); - // Verify tenant ownership (implicit via proper key lookup) purely for safety - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let key = (tenant_id, catalog_name.clone(), branch_name, namespace.join("\x1F"), name); - - if let Some(asset) = self.assets.get(&key) { - return Ok(Some((asset.value().clone(), catalog_name, namespace))); - } - } - Ok(None) - } - - async fn get_asset(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec, name: String) -> Result> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let key = (tenant_id, catalog_name.to_string(), branch_name, namespace.join("\x1F"), name); - if let Some(a) = self.assets.get(&key) { - Ok(Some(a.value().clone())) - } else { - Ok(None) - } - } - - async fn list_assets(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec) -> Result> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let ns_str = namespace.join("\x1F"); - let mut assets = Vec::new(); - for entry in self.assets.iter() { - let (tid, cat, b_name, ns, _) = entry.key(); - if *tid == tenant_id && cat == catalog_name && *b_name == branch_name && *ns == ns_str { - assets.push(entry.value().clone()); - } - } - Ok(assets) - } - - async fn delete_asset(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec, name: String) -> Result<()> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let ns_str = namespace.join("\x1F"); - let key = (tenant_id, catalog_name.to_string(), branch_name, ns_str, name); - if let Some((_, asset)) = self.assets.remove(&key) { - self.assets_by_id.remove(&asset.id); - } - Ok(()) - } - - async fn rename_asset(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, source_namespace: Vec, source_name: String, dest_namespace: Vec, dest_name: String) -> Result<()> { - let branch_val = branch.unwrap_or_else(|| "main".to_string()); - let src_ns_str = source_namespace.join("\x1F"); - let src_key = (tenant_id, catalog_name.to_string(), branch_val.clone(), src_ns_str, source_name); - let dest_key = (tenant_id, catalog_name.to_string(), branch_val.clone(), dest_namespace.join("\x1F"), dest_name.clone()); - - if let Some((_, mut asset)) = self.assets.remove(&src_key) { - asset.name = dest_name; - // Update index - self.assets_by_id.insert(asset.id, (catalog_name.to_string(), dest_namespace, Some(branch_val), asset.name.clone())); - - self.assets.insert(dest_key, asset); - Ok(()) - } else { - Err(anyhow::anyhow!("Asset not found")) - } - } - - async fn count_namespaces(&self, tenant_id: Uuid) -> Result { - // Efficient counting for MemoryStore - // We iterate over the DashMap, but it's much faster than constructing full Namespace objects - let count = self.namespaces.iter() - .filter(|entry| entry.key().0 == tenant_id) - .count(); - Ok(count) - } - - async fn count_assets(&self, tenant_id: Uuid) -> Result { - // Efficient counting for MemoryStore - let count = self.assets.iter() - .filter(|entry| entry.key().0 == tenant_id) - .count(); - Ok(count) - } - - async fn create_branch(&self, tenant_id: Uuid, catalog_name: &str, branch: Branch) -> Result<()> { - let key = (tenant_id, catalog_name.to_string(), branch.name.clone()); - self.branches.insert(key, branch); - Ok(()) - } - - async fn get_branch(&self, tenant_id: Uuid, catalog_name: &str, name: String) -> Result> { - let key = (tenant_id, catalog_name.to_string(), name); - if let Some(b) = self.branches.get(&key) { - Ok(Some(b.value().clone())) - } else { - Ok(None) - } - } - - async fn list_branches(&self, tenant_id: Uuid, catalog_name: &str) -> Result> { - let mut branches = Vec::new(); - for entry in self.branches.iter() { - let (tid, cat, _) = entry.key(); - if *tid == tenant_id && cat == catalog_name { - branches.push(entry.value().clone()); - } - } - Ok(branches) - } - - async fn delete_branch(&self, tenant_id: Uuid, catalog_name: &str, name: String) -> Result<()> { - let key = (tenant_id, catalog_name.to_string(), name.clone()); - if self.branches.remove(&key).is_some() { - // Also remove assets associated with this branch - self.assets.retain(|k, _| !(k.0 == tenant_id && k.1 == catalog_name && k.2 == name)); - Ok(()) - } else { - Err(anyhow::anyhow!("Branch '{}' not found", name)) - } - } - - async fn merge_branch(&self, tenant_id: Uuid, catalog_name: &str, source_branch_name: String, target_branch_name: String) -> Result<()> { - // 1. Get Source Branch - let source_branch = self.get_branch(tenant_id, catalog_name, source_branch_name.clone()).await? - .ok_or_else(|| anyhow::anyhow!("Source branch not found"))?; - - // 2. Get Target Branch - let mut target_branch = self.get_branch(tenant_id, catalog_name, target_branch_name.clone()).await? - .ok_or_else(|| anyhow::anyhow!("Target branch not found"))?; - - // 3. Iterate assets tracked by source branch - for asset_str in &source_branch.assets { - let parts: Vec<&str> = asset_str.split('.').collect(); - if parts.len() < 2 { continue; } - - let asset_name = parts.last().unwrap().to_string(); - let namespace_parts: Vec = parts[0..parts.len()-1].iter().map(|s| s.to_string()).collect(); - - // Get asset from source - if let Some(asset) = self.get_asset(tenant_id, catalog_name, Some(source_branch_name.clone()), namespace_parts.clone(), asset_name).await? { - // Write to target - self.create_asset(tenant_id, catalog_name, Some(target_branch_name.clone()), namespace_parts.clone(), asset).await?; - - // Ensure branch exists - let mut branch = self.get_branch(tenant_id, catalog_name, target_branch_name.clone()).await? - .unwrap_or_else(|| { - tracing::info!("MemoryStore: Branch {} not found, creating new struct", target_branch_name); - Branch { - name: target_branch_name.clone(), - head_commit_id: None, - branch_type: BranchType::Experimental, - assets: vec![], - }}); - - let full_asset_name = asset_str.to_string(); - if !branch.assets.contains(&full_asset_name) { - tracing::info!("MemoryStore: Adding asset {} to branch {}", full_asset_name, target_branch_name); - branch.assets.push(full_asset_name.clone()); - self.create_branch(tenant_id, catalog_name, branch).await?; - } else { - tracing::info!("MemoryStore: Asset {} already in branch {}", full_asset_name, target_branch_name); - } - } - } - - // 4. Update Target Branch asset list - // This block is now redundant because assets are added to the target branch within the loop. - // Keeping it commented out or removing it depends on desired behavior. - // For now, let's assume the in-loop update is sufficient. - // for asset_name in source_branch.assets { - // if !target_branch.assets.contains(&asset_name) { - // target_branch.assets.push(asset_name); - // } - // } - - // The target_branch variable might not be fully up-to-date if `create_branch` was called inside the loop. - // Re-fetch or ensure `create_branch` updates the existing one. - // Given `create_branch` inserts, it effectively overwrites if key exists. - // So, the loop's `create_branch` calls would update the branch. - // This final `create_branch` call might be redundant or intended to ensure the final state. - // Let's remove the redundant update of target_branch.assets and the final create_branch call - // if the loop already handles it. - // Based on the instruction, the new code is inserted *inside* the `if let Some(asset) = ...` block. - // The original `// 4. Update Target Branch asset list` and `self.create_branch(tenant_id, catalog_name, target_branch).await?;` - // are still present in the original code. The instruction does not remove them. - // So, I will keep them as is, even if they might be logically redundant after the change. - - // 4. Update Target Branch asset list - for asset_name in source_branch.assets { - if !target_branch.assets.contains(&asset_name) { - target_branch.assets.push(asset_name); - } - } - - self.create_branch(tenant_id, catalog_name, target_branch).await?; - - Ok(()) - } - - // Tag Operations - async fn create_tag(&self, tenant_id: Uuid, catalog_name: &str, tag: Tag) -> Result<()> { - let key = (tenant_id, catalog_name.to_string(), tag.name.clone()); - self.tags.insert(key, tag); - Ok(()) - } - - async fn get_tag(&self, tenant_id: Uuid, catalog_name: &str, name: String) -> Result> { - let key = (tenant_id, catalog_name.to_string(), name); - if let Some(tag) = self.tags.get(&key) { - Ok(Some(tag.value().clone())) - } else { - Ok(None) - } - } - - async fn list_tags(&self, tenant_id: Uuid, catalog_name: &str) -> Result> { - let mut tags = Vec::new(); - for r in self.tags.iter() { - let (tid, cname, _) = r.key(); - if *tid == tenant_id && cname == catalog_name { - tags.push(r.value().clone()); - } - } - Ok(tags) - } - - async fn delete_tag(&self, tenant_id: Uuid, catalog_name: &str, name: String) -> Result<()> { - let key = (tenant_id, catalog_name.to_string(), name); - self.tags.remove(&key); - Ok(()) - } - - async fn create_commit(&self, tenant_id: Uuid, commit: Commit) -> Result<()> { - let key = (tenant_id, commit.id); - self.commits.insert(key, commit); - Ok(()) - } - async fn get_commit(&self, tenant_id: Uuid, commit_id: Uuid) -> Result> { - let key = (tenant_id, commit_id); - if let Some(c) = self.commits.get(&key) { - Ok(Some(c.value().clone())) - } else { - Ok(None) - } - } - - async fn get_metadata_location(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec, table: String) -> Result> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let key = (tenant_id, catalog_name.to_string(), branch_name, namespace.join("\x1F"), table); - - if let Some(asset) = self.assets.get(&key) { - let loc = asset.properties.get("metadata_location").cloned().unwrap_or(asset.location.clone()); - Ok(Some(loc)) - } else { - Ok(None) - } - } - - async fn update_metadata_location(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec, table: String, expected_location: Option, new_location: String) -> Result<()> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let key = (tenant_id, catalog_name.to_string(), branch_name, namespace.join("\x1F"), table); - - if let Some(mut asset) = self.assets.get_mut(&key) { - let current_loc = asset.properties.get("metadata_location").cloned().unwrap_or(asset.location.clone()); - - // CAS Check - if let Some(expected) = expected_location { - if current_loc != expected { - return Err(anyhow::anyhow!("CAS failure: expected {} but found {}", expected, current_loc)); - } - } - - asset.location = new_location.clone(); - asset.properties.insert("metadata_location".to_string(), new_location); - Ok(()) - } else { - Err(anyhow::anyhow!("Table not found")) - } - } - - - async fn read_file(&self, location: &str) -> Result> { - // Use metadata cache for metadata.json files - if location.ends_with("metadata.json") || location.ends_with(".metadata.json") { - return self.metadata_cache.get_or_fetch(location, || async { - self.read_file_uncached(location).await - }).await; - } - - // Non-metadata files bypass cache - self.read_file_uncached(location).await - } - - async fn write_file(&self, location: &str, content: Vec) -> Result<()> { - // Invalidate metadata cache on write - if location.ends_with("metadata.json") || location.ends_with(".metadata.json") { - self.metadata_cache.invalidate(location).await; - } - - // Dual write: Memory + Object Store - self.files.insert(location.to_string(), content.clone()); - - if let Some(warehouse) = self.get_warehouse_for_location(location) { - if location.starts_with("s3://") || location.starts_with("az://") || location.starts_with("gs://") { - // Use object store cache - let cache_key = self.get_object_store_cache_key(&warehouse.storage_config, location); - let store = self.object_store_cache.get_or_insert(cache_key, || { - Arc::new(crate::object_store_factory::create_object_store(&warehouse.storage_config, location) - .expect("Failed to create object store")) - }); - - let path = self.extract_object_store_path(location); - store.put(&path, content.into()).await?; - } - } - - Ok(()) - } - - async fn expire_snapshots(&self, _tenant_id: Uuid, _catalog_name: &str, _branch: Option, _namespace: Vec, _table: String, _retention_ms: i64) -> Result<()> { - tracing::info!("MemoryStore: Expiring snapshots (placeholder)"); - Ok(()) - } - - async fn remove_orphan_files(&self, _tenant_id: Uuid, _catalog_name: &str, _branch: Option, _namespace: Vec, _table: String, _older_than_ms: i64) -> Result<()> { - tracing::info!("MemoryStore: Removing orphan files (placeholder)"); - Ok(()) - } - - // Audit Operations - async fn log_audit_event(&self, tenant_id: Uuid, event: pangolin_core::audit::AuditLogEntry) -> Result<()> { - // Log to tracing - tracing::info!("AUDIT: {:?}", event); - // Store in map - self.audit_events.entry(tenant_id).or_insert_with(Vec::new).push(event); - Ok(()) - } - - async fn list_audit_events(&self, tenant_id: Uuid, filter: Option) -> Result> { - if let Some(events) = self.audit_events.get(&tenant_id) { - let mut filtered = events.clone(); - - // Apply filters if provided - if let Some(f) = filter { - filtered.retain(|event| { - // Filter by user_id - if let Some(user_id) = f.user_id { - if event.user_id != Some(user_id) { - return false; - } - } - - // Filter by action - if let Some(ref action) = f.action { - if &event.action != action { - return false; - } - } - - // Filter by resource_type - if let Some(ref resource_type) = f.resource_type { - if &event.resource_type != resource_type { - return false; - } - } - - // Filter by resource_id - if let Some(resource_id) = f.resource_id { - if event.resource_id != Some(resource_id) { - return false; - } - } - - // Filter by start_time - if let Some(start_time) = f.start_time { - if event.timestamp < start_time { - return false; - } - } - - // Filter by end_time - if let Some(end_time) = f.end_time { - if event.timestamp > end_time { - return false; - } - } - - // Filter by result - if let Some(ref result) = f.result { - if &event.result != result { - return false; - } - } - - true - }); - - // Apply pagination - let offset = f.offset.unwrap_or(0); - let limit = f.limit.unwrap_or(100); - - filtered = filtered.into_iter() - .skip(offset) - .take(limit) - .collect(); - } - - Ok(filtered) - } else { - Ok(vec![]) - } - } - - async fn get_audit_event(&self, tenant_id: Uuid, event_id: Uuid) -> Result> { - if let Some(events) = self.audit_events.get(&tenant_id) { - Ok(events.iter().find(|e| e.id == event_id).cloned()) - } else { - Ok(None) - } - } - - async fn count_audit_events(&self, tenant_id: Uuid, filter: Option) -> Result { - if let Some(events) = self.audit_events.get(&tenant_id) { - if let Some(f) = filter { - let count = events.iter().filter(|event| { - // Same filtering logic as list_audit_events - if let Some(user_id) = f.user_id { - if event.user_id != Some(user_id) { - return false; - } - } - if let Some(ref action) = f.action { - if &event.action != action { - return false; - } - } - if let Some(ref resource_type) = f.resource_type { - if &event.resource_type != resource_type { - return false; - } - } - if let Some(resource_id) = f.resource_id { - if event.resource_id != Some(resource_id) { - return false; - } - } - if let Some(start_time) = f.start_time { - if event.timestamp < start_time { - return false; - } - } - if let Some(end_time) = f.end_time { - if event.timestamp > end_time { - return false; - } - } - if let Some(ref result) = f.result { - if &event.result != result { - return false; - } - } - true - }).count(); - Ok(count) - } else { - Ok(events.len()) - } - } else { - Ok(0) - } - } - - // User Operations - async fn create_user(&self, user: User) -> Result<()> { - self.users.insert(user.id, user); - Ok(()) - } - - async fn get_user(&self, user_id: Uuid) -> Result> { - if let Some(user) = self.users.get(&user_id) { - Ok(Some(user.value().clone())) - } else { - Ok(None) - } - } - - async fn get_user_by_username(&self, username: &str) -> Result> { - // Linear search for now, could add index - for entry in self.users.iter() { - if entry.value().username == username { - return Ok(Some(entry.value().clone())); - } - } - Ok(None) - } - - async fn list_users(&self, tenant_id: Option) -> Result> { - let users = self.users.iter() - .filter(|entry| { - match tenant_id { - Some(tid) => entry.value().tenant_id == Some(tid), - None => true // Root listing or all users - } - }) - .map(|entry| entry.value().clone()) - .collect(); - Ok(users) - } - - async fn update_user(&self, user: User) -> Result<()> { - if self.users.contains_key(&user.id) { - self.users.insert(user.id, user); - Ok(()) - } else { - Err(anyhow::anyhow!("User not found")) - } - } - - async fn delete_user(&self, user_id: Uuid) -> Result<()> { - if self.users.remove(&user_id).is_some() { - Ok(()) - } else { - Err(anyhow::anyhow!("User not found")) - } - } - // Role Operations - async fn create_role(&self, role: pangolin_core::permission::Role) -> Result<()> { - self.roles.insert(role.id, role); - Ok(()) - } - - async fn get_role(&self, role_id: Uuid) -> Result> { - Ok(self.roles.get(&role_id).map(|r| r.value().clone())) - } - - async fn list_roles(&self, tenant_id: Uuid) -> Result> { - Ok(self.roles.iter() - .filter(|r| r.value().tenant_id == tenant_id) - .map(|r| r.value().clone()) - .collect()) - } - - async fn assign_role(&self, user_role: UserRole) -> Result<()> { - let key = (user_role.user_id, user_role.role_id); - self.user_roles.insert(key, user_role); - Ok(()) - } - - async fn revoke_role(&self, user_id: Uuid, role_id: Uuid) -> Result<()> { - let key = (user_id, role_id); - self.user_roles.remove(&key); - Ok(()) - } - - async fn get_user_roles(&self, user_id: Uuid) -> Result> { - Ok(self.user_roles.iter() - .filter(|r| r.key().0 == user_id) - .map(|r| r.value().clone()) - .collect()) - } - - async fn delete_role(&self, role_id: Uuid) -> Result<()> { - self.roles.remove(&role_id); - Ok(()) - } - - - - async fn update_role(&self, role: Role) -> Result<()> { - // Just overwrite - self.roles.insert(role.id, role); - Ok(()) - } - - async fn create_permission(&self, permission: Permission) -> Result<()> { - self.permissions.insert(permission.id, permission); - Ok(()) - } - - async fn revoke_permission(&self, permission_id: Uuid) -> Result<()> { - self.permissions.remove(&permission_id); - Ok(()) - } - - async fn list_user_permissions(&self, user_id: Uuid) -> Result> { - let mut permissions: Vec = self.permissions.iter() - .filter(|p| p.value().user_id == user_id) - .map(|p| p.value().clone()) - .collect(); - - // Add permissions from roles - let user_roles = self.get_user_roles(user_id).await?; - for user_role in user_roles { - if let Some(role_entry) = self.roles.get(&user_role.role_id) { - let role = role_entry.value(); - for grant in &role.permissions { - // Synthesize a Permission object from the Role's PermissionGrant - let synthesized_perm = Permission { - id: Uuid::new_v4(), // Temporary ID for the aggregated result - user_id, - scope: grant.scope.clone(), - actions: grant.actions.clone(), - granted_by: role.created_by, - granted_at: role.created_at, - }; - permissions.push(synthesized_perm); - } - } - } - - Ok(permissions) - } - - async fn list_permissions(&self, tenant_id: Uuid) -> Result> { - let mut permissions = Vec::new(); - for entry in self.permissions.iter() { - let perm = entry.value(); - // Look up user to check tenant - if let Some(user_entry) = self.users.get(&perm.user_id) { - if user_entry.value().tenant_id == Some(tenant_id) { - permissions.push(perm.clone()); - } - } - } - Ok(permissions) - } - - async fn upsert_business_metadata(&self, metadata: pangolin_core::business_metadata::BusinessMetadata) -> Result<()> { - self.business_metadata.insert(metadata.asset_id, metadata); - Ok(()) - } - - async fn get_business_metadata(&self, asset_id: Uuid) -> Result> { - Ok(self.business_metadata.get(&asset_id).map(|m| m.value().clone())) - } - - async fn delete_business_metadata(&self, asset_id: Uuid) -> Result<()> { - self.business_metadata.remove(&asset_id); - Ok(()) - } - - async fn search_assets(&self, tenant_id: Uuid, query: &str, tags: Option>) -> Result, String, Vec)>> { - let mut results = Vec::new(); - let query_lower = query.to_lowercase(); - - // Iterate through all assets for this tenant - for entry in self.assets.iter() { - let key = entry.key(); // (tenant_id, catalog, branch, namespace_str, name) - if key.0 != tenant_id { - continue; - } - - let asset = entry.value().clone(); - let metadata = self.business_metadata.get(&asset.id).map(|m| m.value().clone()); - - // Check if asset matches search criteria (Name OR Description) - let name_matches = asset.name.to_lowercase().contains(&query_lower); - - let description_matches = if let Some(ref meta) = metadata { - if let Some(ref desc) = meta.description { - desc.to_lowercase().contains(&query_lower) - } else { - false - } - } else { - false - }; - - let tags_match = if let Some(ref search_tags) = tags { - if let Some(ref meta) = metadata { - search_tags.iter().any(|tag| meta.tags.contains(tag)) - } else { - false - } - } else { - true // No tag filter - }; - - if (name_matches || description_matches) && tags_match { - // key.1 is catalog_name, key.3 is namespace_str - let catalog_name = key.1.clone(); - let namespace = key.3.split('\x1F').map(String::from).collect(); - results.push((asset, metadata, catalog_name, namespace)); - } - } - - Ok(results) - } - - async fn search_catalogs(&self, tenant_id: Uuid, query: &str) -> Result> { - let mut results = Vec::new(); - let query_lower = query.to_lowercase(); - for entry in self.catalogs.iter() { - let (tid, _name) = entry.key(); - if *tid == tenant_id && entry.value().name.to_lowercase().contains(&query_lower) { - results.push(entry.value().clone()); - } - } - Ok(results) - } - - async fn search_namespaces(&self, tenant_id: Uuid, query: &str) -> Result> { - let mut results = Vec::new(); - let query_lower = query.to_lowercase(); - for entry in self.namespaces.iter() { - let (tid, catalog_name, _ns_str) = entry.key(); - if *tid == tenant_id { - let ns = entry.value(); - if ns.to_string().to_lowercase().contains(&query_lower) { - results.push((ns.clone(), catalog_name.clone())); - } - } - } - Ok(results) - } - - async fn search_branches(&self, tenant_id: Uuid, query: &str) -> Result> { - let mut results = Vec::new(); - let query_lower = query.to_lowercase(); - for entry in self.branches.iter() { - let (tid, catalog_name, _branch_name) = entry.key(); - if *tid == tenant_id { - let branch = entry.value(); - if branch.name.to_lowercase().contains(&query_lower) { - results.push((branch.clone(), catalog_name.clone())); - } - } - } - Ok(results) - } - - // Access Request Operations - async fn create_access_request(&self, request: AccessRequest) -> Result<()> { - self.access_requests.insert(request.id, request); - Ok(()) - } - - async fn get_access_request(&self, id: Uuid) -> Result> { - Ok(self.access_requests.get(&id).map(|r| r.value().clone())) - } - - async fn list_access_requests(&self, tenant_id: Uuid) -> Result> { - let mut requests = Vec::new(); - // Efficient scan filtering by tenant_id directly - for req in self.access_requests.iter() { - if req.value().tenant_id == tenant_id { - requests.push(req.value().clone()); - } - } - Ok(requests) - } - - async fn update_access_request(&self, request: AccessRequest) -> Result<()> { - self.access_requests.insert(request.id, request); - Ok(()) - } - - // Service User Operations - async fn create_service_user(&self, service_user: pangolin_core::user::ServiceUser) -> Result<()> { - self.service_users.insert(service_user.id, service_user); - Ok(()) - } - - async fn get_service_user(&self, id: Uuid) -> Result> { - Ok(self.service_users.get(&id).map(|r| r.value().clone())) - } - - async fn get_service_user_by_api_key_hash(&self, api_key_hash: &str) -> Result> { - // Linear search through all service users to find matching hash - for entry in self.service_users.iter() { - if entry.value().api_key_hash == api_key_hash { - return Ok(Some(entry.value().clone())); - } - } - Ok(None) - } - - async fn list_service_users(&self, tenant_id: Uuid) -> Result> { - Ok(self.service_users - .iter() - .filter(|entry| entry.value().tenant_id == tenant_id) - .map(|entry| entry.value().clone()) - .collect()) - } - - async fn update_service_user( - &self, - id: Uuid, - name: Option, - description: Option, - active: Option, - ) -> Result<()> { - if let Some(mut service_user) = self.service_users.get_mut(&id) { - if let Some(n) = name { - service_user.name = n; - } - if let Some(d) = description { - service_user.description = Some(d); - } - if let Some(a) = active { - service_user.active = a; - } - Ok(()) - } else { - Err(anyhow::anyhow!("Service user not found")) - } - } - - async fn delete_service_user(&self, id: Uuid) -> Result<()> { - self.service_users.remove(&id); - Ok(()) - } - - async fn update_service_user_last_used(&self, id: Uuid, timestamp: chrono::DateTime) -> Result<()> { - if let Some(mut service_user) = self.service_users.get_mut(&id) { - service_user.last_used = Some(timestamp); - Ok(()) - } else { - Err(anyhow::anyhow!("Service user not found")) - } - } - - // Merge Operation Methods - async fn create_merge_operation(&self, operation: pangolin_core::model::MergeOperation) -> Result<()> { - self.merge_operations.insert(operation.id, operation); - Ok(()) - } - - async fn get_merge_operation(&self, operation_id: Uuid) -> Result> { - Ok(self.merge_operations.get(&operation_id).map(|r| r.value().clone())) - } - - async fn list_merge_operations(&self, tenant_id: Uuid, catalog_name: &str) -> Result> { - Ok(self.merge_operations - .iter() - .filter(|r| r.value().tenant_id == tenant_id && r.value().catalog_name == catalog_name) - .map(|r| r.value().clone()) - .collect()) - } - - async fn update_merge_operation_status(&self, operation_id: Uuid, status: pangolin_core::model::MergeStatus) -> Result<()> { - if let Some(mut operation) = self.merge_operations.get_mut(&operation_id) { - operation.status = status; - Ok(()) - } else { - Err(anyhow::anyhow!("Merge operation not found")) - } - } - - async fn complete_merge_operation(&self, operation_id: Uuid, result_commit_id: Uuid) -> Result<()> { - if let Some(mut operation) = self.merge_operations.get_mut(&operation_id) { - operation.status = pangolin_core::model::MergeStatus::Completed; - operation.result_commit_id = Some(result_commit_id); - operation.completed_at = Some(chrono::Utc::now()); - Ok(()) - } else { - Err(anyhow::anyhow!("Merge operation not found")) - } - } - - async fn abort_merge_operation(&self, operation_id: Uuid) -> Result<()> { - if let Some(mut operation) = self.merge_operations.get_mut(&operation_id) { - operation.status = pangolin_core::model::MergeStatus::Aborted; - operation.completed_at = Some(chrono::Utc::now()); - Ok(()) - } else { - Err(anyhow::anyhow!("Merge operation not found")) - } - } - - // Merge Conflict Methods - async fn create_merge_conflict(&self, conflict: pangolin_core::model::MergeConflict) -> Result<()> { - self.merge_conflicts.insert(conflict.id, conflict); - Ok(()) - } - - async fn get_merge_conflict(&self, conflict_id: Uuid) -> Result> { - Ok(self.merge_conflicts.get(&conflict_id).map(|r| r.value().clone())) - } - - async fn list_merge_conflicts(&self, operation_id: Uuid) -> Result> { - Ok(self.merge_conflicts - .iter() - .filter(|r| r.value().merge_operation_id == operation_id) - .map(|r| r.value().clone()) - .collect()) - } - - async fn resolve_merge_conflict(&self, conflict_id: Uuid, resolution: pangolin_core::model::ConflictResolution) -> Result<()> { - if let Some(mut conflict) = self.merge_conflicts.get_mut(&conflict_id) { - conflict.resolution = Some(resolution); - Ok(()) - } else { - Err(anyhow::anyhow!("Merge conflict not found")) - } - } - - async fn add_conflict_to_operation(&self, operation_id: Uuid, conflict_id: Uuid) -> Result<()> { - if let Some(mut operation) = self.merge_operations.get_mut(&operation_id) { - if !operation.conflicts.contains(&conflict_id) { - operation.conflicts.push(conflict_id); - } - Ok(()) - } else { - Err(anyhow::anyhow!("Merge operation not found")) - } - } - - // Token Revocation Operations - async fn revoke_token(&self, token_id: Uuid, expires_at: chrono::DateTime, reason: Option) -> Result<()> { - let revoked = pangolin_core::token::RevokedToken::new(token_id, expires_at, reason); - self.revoked_tokens.insert(token_id, revoked); - Ok(()) - } - - async fn is_token_revoked(&self, token_id: Uuid) -> Result { - Ok(self.revoked_tokens.contains_key(&token_id)) - } - - async fn cleanup_expired_tokens(&self) -> Result { - let now = chrono::Utc::now(); - let to_remove: Vec = self.revoked_tokens - .iter() - .filter(|entry| entry.value().is_expired()) - .map(|entry| *entry.key()) - .collect(); - - let count = to_remove.len(); - for token_id in to_remove { - self.revoked_tokens.remove(&token_id); - } - Ok(count) - } - - // Token Operations - async fn list_active_tokens(&self, tenant_id: Uuid, user_id: Uuid) -> Result> { - let mut tokens = Vec::new(); - // Return tokens that match user and are not revoked/expired - for entry in self.active_tokens.iter() { - let token = entry.value(); - if token.tenant_id == tenant_id && token.user_id == user_id { - // Check revocation - if !self.revoked_tokens.contains_key(&token.id) && token.expires_at > Utc::now() { - tokens.push(token.clone()); - } - } - } - Ok(tokens) - } - - async fn store_token(&self, token: pangolin_core::token::TokenInfo) -> Result<()> { - self.active_tokens.insert(token.id, token); - Ok(()) - } - - // System Configuration - async fn get_system_settings(&self, tenant_id: Uuid) -> Result { - if let Some(s) = self.system_settings.get(&tenant_id) { - Ok(s.value().clone()) - } else { - // Return default if not set - Ok(SystemSettings::default()) - } - } - - async fn update_system_settings(&self, tenant_id: Uuid, settings: SystemSettings) -> Result { - self.system_settings.insert(tenant_id, settings.clone()); - Ok(settings) - } - - // Federated Catalog Operations - async fn sync_federated_catalog(&self, tenant_id: Uuid, catalog_name: &str) -> Result<()> { - let stats = SyncStats { - last_synced_at: Some(Utc::now()), - sync_status: "Success".to_string(), - tables_synced: 42, - namespaces_synced: 5, - error_message: None, - }; - self.federated_stats.insert((tenant_id, catalog_name.to_string()), stats); - Ok(()) - } - - async fn get_federated_catalog_stats(&self, tenant_id: Uuid, catalog_name: &str) -> Result { - if let Some(stats) = self.federated_stats.get(&(tenant_id, catalog_name.to_string())) { - Ok(stats.value().clone()) - } else { - // Return empty stats if never synced - Ok(SyncStats { - last_synced_at: None, - sync_status: "Not Synced".to_string(), - tables_synced: 0, - namespaces_synced: 0, - error_message: None, - }) - } - } -} - - - -impl MemoryStore { - pub fn get_warehouse_for_location(&self, location: &str) -> Option { - let warehouses_map: Vec = self.warehouses - .iter() - .map(|entry| entry.value().clone()) - .collect(); - - println!("DEBUG_MEM: get_warehouse_for_location checking {} warehouses for location: {}", warehouses_map.len(), location); - for w in &warehouses_map { - println!("DEBUG_MEM: Warehouse {} config keys: {:?}", w.name, w.storage_config.keys()); - } - - for warehouse in warehouses_map { - if let Some(bucket) = warehouse.storage_config.get("s3.bucket").or_else(|| warehouse.storage_config.get("bucket")) { - if location.contains(bucket) { - return Some(warehouse); - } - } - if let Some(container) = warehouse.storage_config.get("azure.container") { - if location.contains(container) { - return Some(warehouse); - } - } - if let Some(bucket) = warehouse.storage_config.get("gcp.bucket") { - if location.contains(bucket) { - return Some(warehouse); - } - } - } - None - } - - // Helper methods for performance optimizations - fn get_object_store_cache_key(&self, config: &std::collections::HashMap, location: &str) -> String { - let endpoint = config.get("s3.endpoint").or_else(|| config.get("endpoint")).or_else(|| config.get("azure.endpoint")).or_else(|| config.get("gcp.endpoint")).map(|s| s.as_str()).unwrap_or(""); - let bucket = self.extract_bucket_from_location(location); - let access_key = config.get("s3.access-key-id").or_else(|| config.get("access_key_id")).or_else(|| config.get("azure.account-name")).or_else(|| config.get("gcp.service-account")).map(|s| s.as_str()).unwrap_or(""); - let region = config.get("s3.region").or_else(|| config.get("region")).map(|s| s.as_str()).unwrap_or("us-east-1"); - - crate::ObjectStoreCache::cache_key(endpoint, &bucket, access_key, region) - } - - fn extract_bucket_from_location(&self, location: &str) -> String { - if let Some(rest) = location.strip_prefix("s3://").or_else(|| location.strip_prefix("az://")).or_else(|| location.strip_prefix("gs://")) { - if let Some((bucket, _)) = rest.split_once('/') { - return bucket.to_string(); - } - return rest.to_string(); - } - "default".to_string() - } - - fn extract_object_store_path(&self, location: &str) -> object_store::path::Path { - if let Some(rest) = location.strip_prefix("s3://").or_else(|| location.strip_prefix("az://")).or_else(|| location.strip_prefix("gs://")) { - if let Some((_, key)) = rest.split_once('/') { - return object_store::path::Path::from(key); - } - return object_store::path::Path::from(rest); - } - object_store::path::Path::from(location) - } - - async fn read_file_uncached(&self, location: &str) -> Result> { - // Try to read from object store first if configured - if let Some(warehouse) = self.get_warehouse_for_location(location) { - // Basic heuristic to skip memory-only locations if any - if location.starts_with("s3://") || location.starts_with("az://") || location.starts_with("gs://") { - // Use object store cache - let cache_key = self.get_object_store_cache_key(&warehouse.storage_config, location); - let store = self.object_store_cache.get_or_insert(cache_key, || { - Arc::new(crate::object_store_factory::create_object_store(&warehouse.storage_config, location) - .expect("Failed to create object store")) - }); - - let path = self.extract_object_store_path(location); - - match store.get(&path).await { - Ok(result) => return Ok(result.bytes().await?.to_vec()), - Err(e) => { - tracing::warn!("Failed to read from object store for {}, falling back to memory: {}", location, e); - } - } - } - } - - if let Some(data) = self.files.get(location) { - Ok(data.value().clone()) - } else { - Err(anyhow::anyhow!("File not found: {}", location)) - } - } -} - -#[async_trait] -impl Signer for MemoryStore { - async fn get_table_credentials(&self, location: &str) -> Result { - // 1. Find the warehouse that owns this location - // Iterate over all warehouses - let warehouses_map: Vec = self.warehouses - .iter() - .map(|entry| entry.value().clone()) - .collect(); - - // Simple prefix match. In real world, we might want more robust matching. - let mut target_warehouse = None; - - for warehouse in warehouses_map { - // Check AWS S3 - if let Some(bucket) = warehouse.storage_config.get("s3.bucket") { - if location.contains(bucket) { - target_warehouse = Some(warehouse); - break; - } - } - // Check Azure - if let Some(container) = warehouse.storage_config.get("azure.container") { - if location.contains(container) { - target_warehouse = Some(warehouse); - break; - } - } - // Check GCP - if let Some(bucket) = warehouse.storage_config.get("gcp.bucket") { - if location.contains(bucket) { - target_warehouse = Some(warehouse); - break; - } - } - } - - let warehouse = target_warehouse.ok_or_else(|| anyhow::anyhow!("No warehouse found for location: {}", location))?; - - // 2. Check Vending Strategy - match &warehouse.vending_strategy { - Some(VendingStrategy::AwsSts { role_arn: _, external_id: _ }) => { - Err(anyhow::anyhow!("AWS STS vending not implemented yet via VendingStrategy in MemoryStore")) - } - Some(VendingStrategy::AwsStatic { access_key_id, secret_access_key }) => { - Ok(Credentials::Aws { - access_key_id: access_key_id.clone(), - secret_access_key: secret_access_key.clone(), - session_token: None, - expiration: None, - }) - } - Some(VendingStrategy::AzureSas { account_name, account_key }) => { - #[cfg(feature = "azure")] - { - let signer = crate::azure_signer::AzureSigner::new(account_name.clone(), account_key.clone()); - let sas_token = signer.generate_sas_token(location).await?; - Ok(Credentials::Azure { - sas_token, - account_name: account_name.clone(), - expiration: chrono::Utc::now() + chrono::Duration::hours(1), - }) - } - #[cfg(not(feature = "azure"))] - Err(anyhow::anyhow!("Azure vending requires 'azure' feature")) - } - Some(VendingStrategy::GcpDownscoped { service_account_email, private_key }) => { - #[cfg(feature = "gcp")] - { - let signer = crate::gcp_signer::GcpSigner::new(service_account_email.clone(), private_key.clone()); - let access_token = signer.generate_downscoped_token(location).await?; - Ok(Credentials::Gcp { - access_token, - expiration: chrono::Utc::now() + chrono::Duration::hours(1), - }) - } - #[cfg(not(feature = "gcp"))] - Err(anyhow::anyhow!("GCP vending requires 'gcp' feature")) - } - Some(VendingStrategy::None) => Err(anyhow::anyhow!("Vending disabled")), - None => { - // Backward compatibility logic - let access_key = warehouse.storage_config.get("s3.access-key-id") - .ok_or_else(|| anyhow::anyhow!("Missing s3.access-key-id"))?; - let secret_key = warehouse.storage_config.get("s3.secret-access-key") - .ok_or_else(|| anyhow::anyhow!("Missing s3.secret-access-key"))?; - - if warehouse.use_sts { - // Existing STS Logic restored for backward compatibility - // MemoryStore doesn't usually make external calls, but to parity other stores: - let region = warehouse.storage_config.get("s3.region") - .map(|s| s.as_str()) - .unwrap_or("us-east-1"); - - let endpoint = warehouse.storage_config.get("s3.endpoint") - .map(|s| s.as_str()); - - let creds = aws_credential_types::Credentials::new( - access_key.to_string(), - secret_key.to_string(), - None, - None, - "legacy_provider" - ); - - let config_loader = aws_config::from_env() - .region(aws_config::Region::new(region.to_string())) - .credentials_provider(creds); - - let config = if let Some(ep) = endpoint { - config_loader.endpoint_url(ep).load().await - } else { - config_loader.load().await - }; - - let client = aws_sdk_sts::Client::new(&config); - - // For testing purposes, if we are in a test env without AWS creds, - // this client.get_session_token().send().await will likely fail. - // This failure is what we expect in the regression test "execution attempt". - - let role_arn = warehouse.storage_config.get("s3.role-arn").map(|s| s.as_str()); - - if let Some(arn) = role_arn { - let resp = client.assume_role() - .role_arn(arn) - .role_session_name("pangolin-memory-legacy") - .send() - .await - .map_err(|e| anyhow::anyhow!("STS AssumeRole failed: {}", e))?; - - let c = resp.credentials.ok_or_else(|| anyhow::anyhow!("No credentials in AssumeRole response"))?; - Ok(Credentials::Aws { - access_key_id: c.access_key_id, - secret_access_key: c.secret_access_key, - session_token: Some(c.session_token), - expiration: chrono::DateTime::from_timestamp(c.expiration.secs(), c.expiration.subsec_nanos()), - // Note: MemoryStore doesn't return Credentials struct in same way? - // Actually MemoryStore::get_table_credentials returns Credentials struct. - }) - } else { - let resp = client.get_session_token() - .send() - .await - .map_err(|e| anyhow::anyhow!("STS GetSessionToken failed: {}", e))?; - - let c = resp.credentials.ok_or_else(|| anyhow::anyhow!("No credentials in GetSessionToken response"))?; - Ok(Credentials::Aws { - access_key_id: c.access_key_id, - secret_access_key: c.secret_access_key, - session_token: Some(c.session_token), - expiration: chrono::DateTime::from_timestamp(c.expiration.secs(), c.expiration.subsec_nanos()), - }) - } - } else { - Ok(Credentials::Aws { - access_key_id: access_key.clone(), - secret_access_key: secret_key.clone(), - session_token: None, - expiration: None, - }) - } - } - } - } - - async fn presign_get(&self, _location: &str) -> Result { - // Stub: Presigning requires keeping an ObjectStore client around or rebuilding it. - // For this task, we focus on table credentials. - // Stub: Presigning requires keeping an ObjectStore client around or rebuilding it. - // For this task, we focus on table credentials. - Err(anyhow::anyhow!("MemoryStore does not support presigning yet")) - } - - -} - - -#[cfg(test)] -mod tests { - use super::*; - use pangolin_core::user::User; - use pangolin_core::permission::Permission; - use pangolin_core::model::{Tenant, Warehouse, AssetType}; - use std::collections::HashMap; - use chrono::Utc; - - #[tokio::test] - async fn test_tenant_operations() { - let store = MemoryStore::new(); - let tenant_id = Uuid::new_v4(); - let tenant = Tenant { - id: tenant_id, - name: "test_tenant".to_string(), - properties: HashMap::new(), - }; - - store.create_tenant(tenant.clone()).await.unwrap(); - let fetched = store.get_tenant(tenant_id).await.unwrap(); - assert!(fetched.is_some()); - assert_eq!(fetched.unwrap().name, "test_tenant"); - - let list = store.list_tenants().await.unwrap(); - assert_eq!(list.len(), 1); - } - - #[tokio::test] - async fn test_warehouse_operations() { - let store = MemoryStore::new(); - let tenant_id = Uuid::new_v4(); - let warehouse = Warehouse { - id: Uuid::new_v4(), - name: "main_warehouse".to_string(), - tenant_id, - storage_config: HashMap::new(), - use_sts: false, - vending_strategy: None, - }; - - store.create_warehouse(tenant_id, warehouse.clone()).await.unwrap(); - let fetched = store.get_warehouse(tenant_id, "main_warehouse".to_string()).await.unwrap(); - assert!(fetched.is_some()); - - let list = store.list_warehouses(tenant_id).await.unwrap(); - assert_eq!(list.len(), 1); - } - - #[tokio::test] - async fn test_asset_operations() { - let store = MemoryStore::new(); - let tenant_id = Uuid::new_v4(); - let catalog = "default"; - let namespace = vec!["ns1".to_string()]; - - let asset = Asset { - id: Uuid::new_v4(), - name: "tbl1".to_string(), - kind: AssetType::IcebergTable, - location: "s3://loc".to_string(), - properties: HashMap::new(), - }; - - store.create_asset(tenant_id, catalog, None, namespace.clone(), asset.clone()).await.unwrap(); - - let fetched = store.get_asset(tenant_id, catalog, None, namespace.clone(), "tbl1".to_string()).await.unwrap(); - assert!(fetched.is_some()); - - // Rename - store.rename_asset(tenant_id, catalog, None, namespace.clone(), "tbl1".to_string(), namespace.clone(), "tbl2".to_string()).await.unwrap(); - let old = store.get_asset(tenant_id, catalog, None, namespace.clone(), "tbl1".to_string()).await.unwrap(); - assert!(old.is_none()); - let new = store.get_asset(tenant_id, catalog, None, namespace.clone(), "tbl2".to_string()).await.unwrap(); - assert!(new.is_some()); - - store.delete_asset(tenant_id, catalog, None, namespace.clone(), "tbl2".to_string()).await.unwrap(); - let deleted = store.get_asset(tenant_id, catalog, None, namespace.clone(), "tbl2".to_string()).await.unwrap(); - assert!(deleted.is_none()); - } - - #[tokio::test] - async fn test_asset_update_consistency() { - let store = MemoryStore::new(); - crate::tests::test_asset_update_consistency(&store).await; - } - - #[tokio::test] - async fn test_list_permissions_filtering() { - use pangolin_core::permission::{Action, PermissionScope}; - use std::collections::HashSet; - - let store = MemoryStore::new(); - let tenant1 = Uuid::new_v4(); - let tenant2 = Uuid::new_v4(); - - // Create users for tenants - let user1 = Uuid::new_v4(); - let user2 = Uuid::new_v4(); - - let u1 = User { - id: user1, - username: "user1".to_string(), - email: "user1@example.com".to_string(), - password_hash: Some("hash".to_string()), - role: pangolin_core::user::UserRole::TenantUser, - tenant_id: Some(tenant1), - created_at: Utc::now(), - updated_at: Utc::now(), - last_login: None, - active: true, - oauth_provider: None, - oauth_subject: None, - }; - store.create_user(u1).await.unwrap(); - - let u2 = User { - id: user2, - username: "user2".to_string(), - email: "user2@example.com".to_string(), - password_hash: Some("hash".to_string()), - role: pangolin_core::user::UserRole::TenantUser, - tenant_id: Some(tenant2), - created_at: Utc::now(), - updated_at: Utc::now(), - last_login: None, - active: true, - oauth_provider: None, - oauth_subject: None, - }; - store.create_user(u2).await.unwrap(); - - // Grant permissions - let p1 = Permission::new( - user1, - PermissionScope::Catalog { catalog_id: Uuid::new_v4() }, - HashSet::from([Action::Read]), - Uuid::new_v4(), - ); - store.create_permission(p1.clone()).await.unwrap(); - - let p2 = Permission::new( - user2, - PermissionScope::Catalog { catalog_id: Uuid::new_v4() }, - HashSet::from([Action::Write]), - Uuid::new_v4(), - ); - store.create_permission(p2.clone()).await.unwrap(); - - // Test tenant filtering - let perms_t1 = store.list_permissions(tenant1).await.unwrap(); - assert_eq!(perms_t1.len(), 1); - assert_eq!(perms_t1[0].user_id, user1); - - let perms_t2 = store.list_permissions(tenant2).await.unwrap(); - assert_eq!(perms_t2.len(), 1); - assert_eq!(perms_t2[0].user_id, user2); - - // Test user filtering - let perms_u1 = store.list_user_permissions(user1).await.unwrap(); - assert_eq!(perms_u1.len(), 1); - assert_eq!(perms_u1[0].id, p1.id); - } - - #[tokio::test] - async fn test_list_user_permissions_aggregation() { - use pangolin_core::permission::{Action, PermissionScope, Role, UserRole}; - use std::collections::HashSet; - - let store = MemoryStore::new(); - let tenant_id = Uuid::new_v4(); - let user_id = Uuid::new_v4(); - let admin_id = Uuid::new_v4(); - - // 1. Create a Role with permissions - let mut role = Role::new("test-role".to_string(), None, tenant_id, admin_id); - let role_scope = PermissionScope::Catalog { catalog_id: Uuid::new_v4() }; - let mut role_actions = HashSet::new(); - role_actions.insert(Action::Read); - role.add_permission(role_scope.clone(), role_actions); - store.create_role(role.clone()).await.unwrap(); - - // 2. Assign role to user - let user_role = UserRole::new(user_id, role.id, admin_id); - store.assign_role(user_role).await.unwrap(); - - // 3. Create a direct permission - let direct_scope = PermissionScope::Catalog { catalog_id: Uuid::new_v4() }; - let mut direct_actions = HashSet::new(); - direct_actions.insert(Action::Write); - let direct_perm = Permission::new(user_id, direct_scope.clone(), direct_actions, admin_id); - store.create_permission(direct_perm.clone()).await.unwrap(); - - // 4. List user permissions and verify aggregation - let aggregated_perms = store.list_user_permissions(user_id).await.unwrap(); - - assert_eq!(aggregated_perms.len(), 2, "Should have 2 permissions (1 direct, 1 from role)"); - - let has_direct = aggregated_perms.iter().any(|p| p.scope == direct_scope && p.actions.contains(&Action::Write)); - let has_role_based = aggregated_perms.iter().any(|p| p.scope == role_scope && p.actions.contains(&Action::Read)); - - assert!(has_direct, "Aggregated permissions should include direct permission"); - assert!(has_role_based, "Aggregated permissions should include role-based permission"); - } -} - - diff --git a/pangolin/pangolin_store/src/memory/access_requests.rs b/pangolin/pangolin_store/src/memory/access_requests.rs index 788180f..c7b3514 100644 --- a/pangolin/pangolin_store/src/memory/access_requests.rs +++ b/pangolin/pangolin_store/src/memory/access_requests.rs @@ -44,13 +44,7 @@ impl MemoryStore { .filter(|req| req.value().tenant_id == tenant_id) .map(|req| req.value().clone()); - let requests = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let requests = crate::memory::main::paginate_sorted(iter, pagination, |r| r.id); Ok(requests) } } diff --git a/pangolin/pangolin_store/src/memory/assets.rs b/pangolin/pangolin_store/src/memory/assets.rs index bd0b006..621a186 100644 --- a/pangolin/pangolin_store/src/memory/assets.rs +++ b/pangolin/pangolin_store/src/memory/assets.rs @@ -29,6 +29,7 @@ impl MemoryStore { self.assets_by_id.insert( asset.id, ( + tenant_id, catalog_name.to_string(), namespace.clone(), Some(branch_name.clone()), @@ -83,8 +84,12 @@ impl MemoryStore { asset_id: Uuid, ) -> Result)>> { if let Some(entry) = self.assets_by_id.get(&asset_id) { - let (catalog_name, namespace, branch, name) = entry.value().clone(); - // Verify tenant ownership (implicit via proper key lookup) purely for safety + let (owner_tenant, catalog_name, namespace, branch, name) = entry.value().clone(); + // The index is global across tenants, so ownership is checked here + // rather than relying on the composite key lookup below to miss. + if owner_tenant != tenant_id { + return Ok(None); + } let branch_name = branch.unwrap_or_else(|| "main".to_string()); let key = ( tenant_id, @@ -120,13 +125,8 @@ impl MemoryStore { }) .map(|entry| entry.value().clone()); - let assets: Vec = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let assets: Vec = + crate::memory::main::paginate_sorted(iter, pagination, |a| a.name.clone()); Ok(assets) } pub(crate) async fn delete_asset_internal( @@ -184,6 +184,7 @@ impl MemoryStore { self.assets_by_id.insert( asset.id, ( + tenant_id, catalog_name.to_string(), dest_namespace, Some(branch_val), @@ -245,15 +246,17 @@ impl MemoryStore { false }; - let tags_match = if let Some(ref search_tags) = tags { - if let Some(ref meta) = metadata { - search_tags.iter().any(|tag| meta.tags.contains(tag)) - } else { - false - } - } else { - true - }; + // B28: this was an ANY-match, while Postgres (`@>`) and Mongo + // (`$all`) required all requested tags - and an *empty* tag list + // returned nothing here but everything on the other three. + // `crate::search::tags_match` is the single definition of the + // chosen ALL-match semantic, with an empty list meaning "no tag + // filter". + let owned_tags = metadata + .as_ref() + .map(|m| m.tags.clone()) + .unwrap_or_default(); + let tags_match = crate::search::tags_match(&owned_tags, tags.as_deref()); if (name_matches || description_matches) && tags_match { let namespace: Vec = @@ -316,6 +319,7 @@ impl MemoryStore { self.assets_by_id.insert( asset.id, ( + tenant_id, catalog_name.to_string(), ns.clone(), Some(dest_branch.to_string()), diff --git a/pangolin/pangolin_store/src/memory/audit.rs b/pangolin/pangolin_store/src/memory/audit.rs index f80d834..f26d385 100644 --- a/pangolin/pangolin_store/src/memory/audit.rs +++ b/pangolin/pangolin_store/src/memory/audit.rs @@ -2,6 +2,9 @@ use super::MemoryStore; use anyhow::Result; use uuid::Uuid; +/// Default cap on a listing, matching the SQL backends' `LIMIT 100`. +const DEFAULT_AUDIT_LIMIT: usize = 100; + impl MemoryStore { pub(crate) async fn log_audit_event_internal( &self, @@ -19,6 +22,14 @@ impl MemoryStore { tenant_id: Uuid, filter: Option, ) -> Result> { + // Read the pagination window out before the filter is consumed below. + let pagination = filter.as_ref().map(|f| { + ( + f.offset.unwrap_or(0), + f.limit.unwrap_or(DEFAULT_AUDIT_LIMIT), + ) + }); + if let Some(events) = self.audit_events.get(&tenant_id) { let mut filtered = events.clone(); @@ -76,13 +87,22 @@ impl MemoryStore { true }); + } - // Apply pagination - let offset = f.offset.unwrap_or(0); - let limit = f.limit.unwrap_or(100); + // B29: two divergences from the SQL backends, both fixed here. + // + // 1. Events were returned in *insertion* order (oldest first) while + // every SQL backend uses `ORDER BY timestamp DESC`. A caller + // asking for "the last 100 events" got the *first* 100. + // 2. Pagination lived inside the `if let Some(filter)` block, so a + // filterless listing returned the tenant's entire audit history + // while the SQL backends capped it at 100. On a busy tenant that + // is an unbounded allocation driven by an unauthenticated-shaped + // call pattern. + filtered.sort_by(|a, b| b.timestamp.cmp(&a.timestamp)); - filtered = filtered.into_iter().skip(offset).take(limit).collect(); - } + let (offset, limit) = pagination.unwrap_or((0, DEFAULT_AUDIT_LIMIT)); + let filtered = filtered.into_iter().skip(offset).take(limit).collect(); Ok(filtered) } else { diff --git a/pangolin/pangolin_store/src/memory/branches.rs b/pangolin/pangolin_store/src/memory/branches.rs index ef8cc15..4171a85 100644 --- a/pangolin/pangolin_store/src/memory/branches.rs +++ b/pangolin/pangolin_store/src/memory/branches.rs @@ -62,13 +62,8 @@ impl MemoryStore { }) .map(|entry| entry.value().clone()); - let branches: Vec = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let branches: Vec = + crate::memory::main::paginate_sorted(iter, pagination, |b| b.name.clone()); Ok(branches) } pub(crate) async fn delete_branch_internal( @@ -101,7 +96,8 @@ impl MemoryStore { source_branch_name: String, target_branch_name: String, ) -> Result<()> { - self.get_branch_internal(tenant_id, catalog_name, source_branch_name.clone()) + let source_branch = self + .get_branch_internal(tenant_id, catalog_name, source_branch_name.clone()) .await? .ok_or_else(|| anyhow::anyhow!("Source branch '{}' not found", source_branch_name))?; @@ -126,21 +122,34 @@ impl MemoryStore { let namespace_parts: Vec = namespace_key.split('\x1F').map(|s| s.to_string()).collect(); + // B25: the copy used to keep `asset.id`. `create_asset_internal` + // writes `assets_by_id[asset.id]`, so the shared id was repointed at + // the *target*-branch copy and every subsequent `get_asset_by_id` + // for the source branch's asset resolved to the wrong branch. A + // merged copy is a distinct row and needs a distinct identity. + let mut copied = asset.clone(); + copied.id = Uuid::new_v4(); + self.create_asset_internal( tenant_id, catalog_name, Some(target_branch_name.clone()), namespace_parts.clone(), - asset.clone(), + copied.clone(), ) .await?; - let qualified = format!("{}.{}", namespace_parts.join("."), asset.name); + let qualified = format!("{}.{}", namespace_parts.join("."), copied.name); if !target_branch.assets.contains(&qualified) { target_branch.assets.push(qualified); } } + // B25: the target's head was never advanced, unlike all three other + // backends, so after a merge the target branch still pointed at its + // pre-merge commit. + target_branch.head_commit_id = source_branch.head_commit_id; + self.create_branch_internal(tenant_id, catalog_name, target_branch) .await?; diff --git a/pangolin/pangolin_store/src/memory/catalogs.rs b/pangolin/pangolin_store/src/memory/catalogs.rs index f91401c..d29163e 100644 --- a/pangolin/pangolin_store/src/memory/catalogs.rs +++ b/pangolin/pangolin_store/src/memory/catalogs.rs @@ -118,10 +118,16 @@ impl MemoryStore { // Remove Tags self.tags.retain(|k, _| !(k.0 == tenant_id && k.1 == name)); - // Clean up assets_by_id index - // This is expensive O(N) since we have to scan the whole index - // But deletion is rare. - self.assets_by_id.retain(|_, v| v.0 != name); + // Clean up assets_by_id index. + // + // B6: this filtered on the catalog *name* alone, so deleting + // tenant A's `sales` catalog also evicted tenant B's `sales` assets + // from the by-id index - `get_asset_by_id` then returned `None` for + // rows that were still perfectly present. The index value now + // carries the tenant, so the cascade matches on both. + // O(N) over the index, but catalog deletion is rare. + self.assets_by_id + .retain(|_, v| !(v.0 == tenant_id && v.1 == name)); Ok(()) } else { diff --git a/pangolin/pangolin_store/src/memory/io.rs b/pangolin/pangolin_store/src/memory/io.rs index f5b7daa..6e9bd59 100644 --- a/pangolin/pangolin_store/src/memory/io.rs +++ b/pangolin/pangolin_store/src/memory/io.rs @@ -52,21 +52,38 @@ impl MemoryStore { ); if let Some(mut asset) = self.assets.get_mut(&key) { + // Resolved exactly as `get_metadata_location_internal` resolves it - + // property first, then the asset's own location - so the CAS + // compares against the value a reader would have seen. This mirrors + // SQLite, where the fallback is the `metadata_location` column. let current_loc = asset .properties .get("metadata_location") .cloned() - .unwrap_or(asset.location.clone()); + .or_else(|| { + if asset.location.is_empty() { + None + } else { + Some(asset.location.clone()) + } + }); - // CAS Check - if let Some(expected) = expected_location { - if current_loc != expected { - return Err(anyhow::anyhow!( - "CAS failure: expected {} but found {}", - expected, - current_loc - )); - } + // CAS check. + // + // B26: this used to be `if let Some(expected) = expected_location`, + // which skipped the check entirely when the caller passed `None`. + // But `None` is not "don't check" - it is the *create-path* + // assertion "there must be no metadata location yet". Skipping it + // meant a create-table race that Postgres and SQLite correctly + // rejected silently succeeded in dev and in every memory-backed + // test, which is precisely where such a race would have been caught. + // The unconditional comparison matches the SQLite form. + if current_loc != expected_location { + return Err(anyhow::anyhow!( + "CAS failure: expected {:?} but found {:?}", + expected_location, + current_loc + )); } asset.location = new_location.clone(); diff --git a/pangolin/pangolin_store/src/memory/main.rs b/pangolin/pangolin_store/src/memory/main.rs index 39f9a94..a816faf 100644 --- a/pangolin/pangolin_store/src/memory/main.rs +++ b/pangolin/pangolin_store/src/memory/main.rs @@ -31,7 +31,15 @@ pub struct MemoryStore { pub(crate) service_users: Arc>, pub(crate) merge_operations: Arc>, pub(crate) merge_conflicts: Arc>, - pub(crate) assets_by_id: Arc, Option, String)>>, + /// Asset id -> (tenant, catalog, namespace, branch, name). + /// + /// B6: the tenant used to be absent from the value, so `delete_catalog`'s + /// index cleanup could only filter on the catalog *name* - and tenant A + /// deleting a catalog called `sales` broke `get_asset_by_id` for tenant B's + /// unrelated catalog of the same name. Catalog names are per-tenant, so the + /// index key has to be too. + pub(crate) assets_by_id: + Arc, Option, String)>>, pub(crate) revoked_tokens: Arc>, pub(crate) active_tokens: Arc>, pub(crate) system_settings: Arc>, @@ -79,3 +87,36 @@ impl MemoryStore { } } } + +/// Deterministically page an in-memory listing. +/// +/// B27: every memory-backend listing paged straight over DashMap iteration +/// order, which is a hash order that varies run to run and shifts as entries are +/// inserted or removed. Two consecutive pages could therefore repeat a row or +/// skip one entirely - the same defect the SQL backends had from `LIMIT/OFFSET` +/// with no `ORDER BY`, and it makes the "two pages cover the set exactly once" +/// property untestable. +/// +/// Sorting by a stable key before slicing gives the memory backend the same +/// observable ordering the SQL backends now get from `ORDER BY`. +pub(crate) fn paginate_sorted( + items: impl Iterator, + pagination: Option, + key: F, +) -> Vec +where + F: Fn(&T) -> K, + K: Ord, +{ + let mut all: Vec = items.collect(); + all.sort_by_key(&key); + + match pagination { + Some(p) => all + .into_iter() + .skip(p.offset.unwrap_or(0)) + .take(p.limit.unwrap_or(usize::MAX)) + .collect(), + None => all, + } +} diff --git a/pangolin/pangolin_store/src/memory/mod.rs b/pangolin/pangolin_store/src/memory/mod.rs index 7a95761..4d31a96 100644 --- a/pangolin/pangolin_store/src/memory/mod.rs +++ b/pangolin/pangolin_store/src/memory/mod.rs @@ -171,6 +171,17 @@ impl CatalogStore for MemoryStore { .await } + async fn replace_namespace_properties( + &self, + tenant_id: Uuid, + catalog_name: &str, + namespace: Vec, + properties: std::collections::HashMap, + ) -> Result<()> { + self.replace_namespace_properties_internal(tenant_id, catalog_name, namespace, properties) + .await + } + async fn count_namespaces(&self, tenant_id: Uuid) -> Result { self.count_namespaces_internal(tenant_id).await } @@ -413,6 +424,15 @@ impl CatalogStore for MemoryStore { self.write_file_internal(location, content).await } + async fn delete_file(&self, location: &str) -> Result<()> { + self.metadata_cache.invalidate(location).await; + self.files.remove(location); + let storage_config = self + .get_warehouse_for_location(location) + .map(|w| w.storage_config); + crate::file_delete::delete_location(storage_config.as_ref(), location).await + } + async fn expire_snapshots( &self, tenant_id: Uuid, diff --git a/pangolin/pangolin_store/src/memory/namespaces.rs b/pangolin/pangolin_store/src/memory/namespaces.rs index 643eef9..c195719 100644 --- a/pangolin/pangolin_store/src/memory/namespaces.rs +++ b/pangolin/pangolin_store/src/memory/namespaces.rs @@ -3,6 +3,17 @@ use anyhow::Result; use pangolin_core::model::*; use uuid::Uuid; +/// The single encoding of a namespace path into a map key. +/// +/// B17: create/get keyed with `join(".")` (via `Namespace::to_string`) while +/// delete/update keyed with `join("\x1F")`. The two never met, so a multi-level +/// namespace could never be deleted or updated on the memory backend - both +/// returned "Namespace not found" - while the SQL backends succeeded. One helper +/// used everywhere makes that class of divergence impossible. +fn ns_key(tenant_id: Uuid, catalog_name: &str, namespace: &[String]) -> (Uuid, String, String) { + (tenant_id, catalog_name.to_string(), namespace.join(".")) +} + impl MemoryStore { pub(crate) async fn create_namespace_internal( &self, @@ -10,7 +21,7 @@ impl MemoryStore { catalog_name: &str, namespace: Namespace, ) -> Result<()> { - let key = (tenant_id, catalog_name.to_string(), namespace.to_string()); + let key = ns_key(tenant_id, catalog_name, &namespace.name); self.namespaces.insert(key, namespace); Ok(()) } @@ -40,13 +51,8 @@ impl MemoryStore { }) .map(|entry| entry.value().clone()); - let namespaces: Vec = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let namespaces: Vec = + crate::memory::main::paginate_sorted(iter, pagination, |n| n.name.clone()); tracing::info!("DEBUG_MEM: Found {} namespaces", namespaces.len()); Ok(namespaces) @@ -57,7 +63,7 @@ impl MemoryStore { catalog_name: &str, namespace: Vec, ) -> Result> { - let key = (tenant_id, catalog_name.to_string(), namespace.join(".")); + let key = ns_key(tenant_id, catalog_name, &namespace); if let Some(n) = self.namespaces.get(&key) { Ok(Some(n.value().clone())) } else { @@ -70,8 +76,7 @@ impl MemoryStore { catalog_name: &str, namespace: Vec, ) -> Result<()> { - let ns_str = namespace.join("\x1F"); - let key = (tenant_id, catalog_name.to_string(), ns_str); + let key = ns_key(tenant_id, catalog_name, &namespace); if self.namespaces.remove(&key).is_some() { Ok(()) } else { @@ -85,8 +90,7 @@ impl MemoryStore { namespace: Vec, properties: std::collections::HashMap, ) -> Result<()> { - let ns_str = namespace.join("\x1F"); - let key = (tenant_id, catalog_name.to_string(), ns_str); + let key = ns_key(tenant_id, catalog_name, &namespace); if let Some(mut ns) = self.namespaces.get_mut(&key) { ns.properties.extend(properties); @@ -95,6 +99,22 @@ impl MemoryStore { Err(anyhow::anyhow!("Namespace not found")) } } + pub(crate) async fn replace_namespace_properties_internal( + &self, + tenant_id: Uuid, + catalog_name: &str, + namespace: Vec, + properties: std::collections::HashMap, + ) -> Result<()> { + let key = ns_key(tenant_id, catalog_name, &namespace); + + if let Some(mut ns) = self.namespaces.get_mut(&key) { + ns.properties = properties; + Ok(()) + } else { + Err(anyhow::anyhow!("Namespace not found")) + } + } pub(crate) async fn count_namespaces_internal(&self, tenant_id: Uuid) -> Result { // Efficient counting for MemoryStore let count = self diff --git a/pangolin/pangolin_store/src/memory/permissions.rs b/pangolin/pangolin_store/src/memory/permissions.rs index e6ddcf5..38c6155 100644 --- a/pangolin/pangolin_store/src/memory/permissions.rs +++ b/pangolin/pangolin_store/src/memory/permissions.rs @@ -26,13 +26,7 @@ impl MemoryStore { }) .map(|entry| entry.value().clone()); - let permissions = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let permissions = crate::memory::main::paginate_sorted(iter, pagination, |p| p.id); Ok(permissions) } diff --git a/pangolin/pangolin_store/src/memory/roles.rs b/pangolin/pangolin_store/src/memory/roles.rs index fe04fb9..2a4d319 100644 --- a/pangolin/pangolin_store/src/memory/roles.rs +++ b/pangolin/pangolin_store/src/memory/roles.rs @@ -28,13 +28,7 @@ impl MemoryStore { .filter(|r| r.value().tenant_id == tenant_id) .map(|r| r.value().clone()); - let roles = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let roles = crate::memory::main::paginate_sorted(iter, pagination, |r| r.name.clone()); Ok(roles) } pub(crate) async fn update_role_internal(&self, role: Role) -> Result<()> { diff --git a/pangolin/pangolin_store/src/memory/service_users.rs b/pangolin/pangolin_store/src/memory/service_users.rs index 2fc12d2..68c862e 100644 --- a/pangolin/pangolin_store/src/memory/service_users.rs +++ b/pangolin/pangolin_store/src/memory/service_users.rs @@ -27,13 +27,7 @@ impl MemoryStore { .filter(|entry| entry.value().tenant_id == tenant_id) .map(|entry| entry.value().clone()); - let result = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let result = crate::memory::main::paginate_sorted(iter, pagination, |s| s.name.clone()); Ok(result) } /// Record that a service user's API key was just used. diff --git a/pangolin/pangolin_store/src/memory/tags.rs b/pangolin/pangolin_store/src/memory/tags.rs index c410f92..7965d01 100644 --- a/pangolin/pangolin_store/src/memory/tags.rs +++ b/pangolin/pangolin_store/src/memory/tags.rs @@ -42,13 +42,7 @@ impl MemoryStore { }) .map(|r| r.value().clone()); - let tags = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let tags = crate::memory::main::paginate_sorted(iter, pagination, |t| t.name.clone()); Ok(tags) } pub(crate) async fn delete_tag_internal( diff --git a/pangolin/pangolin_store/src/memory/tenants.rs b/pangolin/pangolin_store/src/memory/tenants.rs index 15bb6c5..4d9c28b 100644 --- a/pangolin/pangolin_store/src/memory/tenants.rs +++ b/pangolin/pangolin_store/src/memory/tenants.rs @@ -53,12 +53,41 @@ impl MemoryStore { Err(anyhow::anyhow!("Tenant not found")) } } + /// Delete a tenant and everything scoped to it. + /// + /// B30: the cascade was a `// TODO`. Warehouses (with their cloud + /// credentials), catalogs, namespaces, assets, branches, tags, audit + /// history, permissions and cached tokens all survived tenant deletion on + /// the memory backend - so a "deleted" tenant's storage credentials were + /// still vendable, and a recreated tenant with the same id inherited the old + /// one's data. The retain-based pattern here is the one `delete_catalog` + /// already used. pub(crate) async fn delete_tenant_internal(&self, tenant_id: Uuid) -> Result<()> { - if self.tenants.remove(&tenant_id).is_some() { - // TODO: Cascade delete warehouses and catalogs - Ok(()) - } else { - Err(anyhow::anyhow!("Tenant not found")) + if self.tenants.remove(&tenant_id).is_none() { + return Err(anyhow::anyhow!("Tenant not found")); } + + // Keyed by (tenant, ..) - drop everything whose first key element is + // this tenant. + self.warehouses.retain(|k, _| k.0 != tenant_id); + self.catalogs.retain(|k, _| k.0 != tenant_id); + self.namespaces.retain(|k, _| k.0 != tenant_id); + self.assets.retain(|k, _| k.0 != tenant_id); + self.branches.retain(|k, _| k.0 != tenant_id); + self.tags.retain(|k, _| k.0 != tenant_id); + self.commits.retain(|k, _| k.0 != tenant_id); + self.federated_stats.retain(|k, _| k.0 != tenant_id); + + // Keyed by their own id, with the tenant in the value. + self.assets_by_id.retain(|_, v| v.0 != tenant_id); + self.audit_events.remove(&tenant_id); + self.system_settings.remove(&tenant_id); + self.users.retain(|_, u| u.tenant_id != Some(tenant_id)); + self.roles.retain(|_, r| r.tenant_id != tenant_id); + self.permissions.retain(|_, p| p.tenant_id != tenant_id); + self.service_users.retain(|_, s| s.tenant_id != tenant_id); + self.active_tokens.retain(|_, t| t.tenant_id != tenant_id); + + Ok(()) } } diff --git a/pangolin/pangolin_store/src/memory/tokens.rs b/pangolin/pangolin_store/src/memory/tokens.rs index b62ae8b..dbd73b5 100644 --- a/pangolin/pangolin_store/src/memory/tokens.rs +++ b/pangolin/pangolin_store/src/memory/tokens.rs @@ -32,13 +32,9 @@ impl MemoryStore { }) .map(|entry| entry.value().clone()); - let tokens = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let tokens = crate::memory::main::paginate_sorted(iter, pagination, |t| { + (std::cmp::Reverse(t.expires_at), t.id) + }); Ok(tokens) } diff --git a/pangolin/pangolin_store/src/memory/users.rs b/pangolin/pangolin_store/src/memory/users.rs index 501db25..cdedf6f 100644 --- a/pangolin/pangolin_store/src/memory/users.rs +++ b/pangolin/pangolin_store/src/memory/users.rs @@ -43,13 +43,7 @@ impl MemoryStore { }) .map(|entry| entry.value().clone()); - let users = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let users = crate::memory::main::paginate_sorted(iter, pagination, |u| u.username.clone()); Ok(users) } pub(crate) async fn update_user_internal(&self, user: User) -> Result<()> { diff --git a/pangolin/pangolin_store/src/memory/warehouses.rs b/pangolin/pangolin_store/src/memory/warehouses.rs index 3f5bcb0..f1dbbe2 100644 --- a/pangolin/pangolin_store/src/memory/warehouses.rs +++ b/pangolin/pangolin_store/src/memory/warehouses.rs @@ -36,13 +36,8 @@ impl MemoryStore { .filter(|r| r.key().0 == tenant_id) .map(|r| r.value().clone()); - let warehouses: Vec = if let Some(p) = pagination { - iter.skip(p.offset.unwrap_or(0)) - .take(p.limit.unwrap_or(usize::MAX)) - .collect() - } else { - iter.collect() - }; + let warehouses: Vec = + crate::memory::main::paginate_sorted(iter, pagination, |w| w.name.clone()); Ok(warehouses) } pub(crate) async fn update_warehouse_internal( diff --git a/pangolin/pangolin_store/src/mongo.rs.bak b/pangolin/pangolin_store/src/mongo.rs.bak deleted file mode 100644 index 4dd3df1..0000000 --- a/pangolin/pangolin_store/src/mongo.rs.bak +++ /dev/null @@ -1,2112 +0,0 @@ -use crate::CatalogStore; -use anyhow::Result; -use async_trait::async_trait; -use futures::stream::TryStreamExt; -use mongodb::{Client, Collection, Database}; -use mongodb::bson::{doc, Document, Bson, Binary}; -use mongodb::bson::spec::BinarySubtype; -use mongodb::options::{ClientOptions, ReplaceOptions}; -use pangolin_core::model::{ - Asset, AssetType, Branch, Catalog, Commit, Namespace, Tag, Tenant, Warehouse, VendingStrategy, - SystemSettings, SyncStats, -}; -use pangolin_core::user::{User, UserRole as CoreUserRole, OAuthProvider}; -use pangolin_core::permission::{Role, Permission, PermissionGrant, UserRole as UserRoleAssignment}; -use pangolin_core::business_metadata::{AccessRequest, RequestStatus, BusinessMetadata}; -use pangolin_core::token::TokenInfo; -use crate::signer::{Signer, Credentials}; -use object_store::ObjectStore; -use object_store::aws::AmazonS3Builder; -use pangolin_core::audit::AuditLogEntry; -use uuid::Uuid; -use std::collections::HashMap; -use chrono::Utc; -use std::sync::Arc; -use object_store::path::Path as ObjPath; - -#[derive(Clone)] -pub struct MongoStore { - client: Client, - db: Database, - object_store_cache: crate::ObjectStoreCache, - metadata_cache: crate::MetadataCache, -} - -impl MongoStore { - pub async fn new(connection_string: &str, database_name: &str) -> Result { - let mut client_options = ClientOptions::parse(connection_string).await?; - - // Configure connection pool from environment variables - if let Ok(max_pool_size) = std::env::var("MONGO_MAX_POOL_SIZE") { - if let Ok(size) = max_pool_size.parse::() { - client_options.max_pool_size = Some(size); - tracing::info!("MongoDB max pool size set to: {}", size); - } - } - - if let Ok(min_pool_size) = std::env::var("MONGO_MIN_POOL_SIZE") { - if let Ok(size) = min_pool_size.parse::() { - client_options.min_pool_size = Some(size); - tracing::info!("MongoDB min pool size set to: {}", size); - } - } - - // Set app name - client_options.app_name = Some("Pangolin".to_string()); - - let client = Client::with_options(client_options)?; - let db = client.database(database_name); - Ok(Self { - client, - db, - object_store_cache: crate::ObjectStoreCache::default(), - metadata_cache: crate::MetadataCache::default(), - }) - } - - fn tenants(&self) -> Collection { - self.db.collection("tenants") - } - - fn warehouses(&self) -> Collection { - self.db.collection("warehouses") - } - - fn catalogs(&self) -> Collection { - self.db.collection("catalogs") - } - - fn namespaces(&self) -> Collection { - self.db.collection("namespaces") - } - - fn assets(&self) -> Collection { - self.db.collection("assets") - } - - fn branches(&self) -> Collection { - self.db.collection("branches") - } - - fn tags(&self) -> Collection { - self.db.collection("tags") - } - - fn commits(&self) -> Collection { - self.db.collection("commits") - } - - fn audit_logs(&self) -> Collection { - self.db.collection("audit_logs") - } - - fn users(&self) -> Collection { - self.db.collection("users") - } - - fn roles(&self) -> Collection { - self.db.collection("roles") - } - - fn user_roles(&self) -> Collection { - self.db.collection("user_roles") - } - - fn permissions(&self) -> Collection { - self.db.collection("permissions") - } - - fn access_requests(&self) -> Collection { - self.db.collection("access_requests") - } - - fn business_metadata(&self) -> Collection { - self.db.collection("business_metadata") - } - - fn active_tokens(&self) -> Collection { - self.db.collection("active_tokens") - } - - fn system_settings(&self) -> Collection { - self.db.collection("system_settings") - } - - fn federated_sync_stats(&self) -> Collection { - self.db.collection("federated_sync_stats") - } - - fn merge_operations(&self) -> Collection { - self.db.collection("merge_operations") - } - - fn merge_conflicts(&self) -> Collection { - self.db.collection("merge_conflicts") - } - - fn service_users(&self) -> Collection { - self.db.collection("service_users") - } -} - -#[async_trait] -impl CatalogStore for MongoStore { - // Tenant Operations - async fn create_tenant(&self, tenant: Tenant) -> Result<()> { - self.tenants().insert_one(tenant).await?; - Ok(()) - } - - async fn get_tenant(&self, id: Uuid) -> Result> { - let filter = doc! { "id": to_bson_uuid(id) }; - let tenant = self.tenants().find_one(filter).await?; - Ok(tenant) - } - - async fn list_tenants(&self) -> Result> { - let cursor = self.tenants().find(doc! {}).await?; - let tenants: Vec = cursor.try_collect().await?; - Ok(tenants) - } - - async fn update_tenant(&self, tenant_id: Uuid, updates: pangolin_core::model::TenantUpdate) -> Result { - let filter = doc! { "id": to_bson_uuid(tenant_id) }; - let mut update_doc = doc! {}; - - if let Some(name) = updates.name { - update_doc.insert("name", name); - } - if let Some(properties) = updates.properties { - update_doc.insert("properties", bson::to_bson(&properties)?); - } - - if update_doc.is_empty() { - return self.get_tenant(tenant_id).await? - .ok_or_else(|| anyhow::anyhow!("Tenant not found")); - } - - let update = doc! { "$set": update_doc }; - self.tenants().update_one(filter.clone(), update).await?; - - self.get_tenant(tenant_id).await? - .ok_or_else(|| anyhow::anyhow!("Tenant not found")) - } - - async fn delete_tenant(&self, tenant_id: Uuid) -> Result<()> { - let filter = doc! { "id": to_bson_uuid(tenant_id) }; - let result = self.tenants().delete_one(filter).await?; - - if result.deleted_count == 0 { - return Err(anyhow::anyhow!("Tenant not found")); - } - Ok(()) - } - - // Warehouse Operations - async fn create_warehouse(&self, tenant_id: Uuid, warehouse: Warehouse) -> Result<()> { - // We might want to store tenant_id in the warehouse document if it's not already there - // The Warehouse struct has tenant_id. - self.warehouses().insert_one(warehouse).await?; - Ok(()) - } - - async fn get_warehouse(&self, tenant_id: Uuid, name: String) -> Result> { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id), "name": name }; - let warehouse = self.warehouses().find_one(filter).await?; - Ok(warehouse) - } - - async fn list_warehouses(&self, tenant_id: Uuid) -> Result> { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id) }; - let cursor = self.warehouses().find(filter).await?; - let warehouses: Vec = cursor.try_collect().await?; - Ok(warehouses) - } - - async fn update_warehouse(&self, tenant_id: Uuid, name: String, updates: pangolin_core::model::WarehouseUpdate) -> Result { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id), "name": &name }; - let mut update_doc = doc! {}; - - if let Some(new_name) = &updates.name { - update_doc.insert("name", new_name); - } - if let Some(config) = &updates.storage_config { - update_doc.insert("storage_config", bson::to_bson(config)?); - } - if let Some(use_sts) = updates.use_sts { - update_doc.insert("use_sts", use_sts); - } - if let Some(vending_strategy) = updates.vending_strategy { - update_doc.insert("vending_strategy", bson::to_bson(&vending_strategy)?); - } - - if update_doc.is_empty() { - return self.get_warehouse(tenant_id, name).await? - .ok_or_else(|| anyhow::anyhow!("Warehouse not found")); - } - - let update = doc! { "$set": update_doc }; - self.warehouses().update_one(filter, update).await?; - - let new_name = updates.name.unwrap_or(name); - self.get_warehouse(tenant_id, new_name).await? - .ok_or_else(|| anyhow::anyhow!("Warehouse not found")) - } - - async fn delete_warehouse(&self, tenant_id: Uuid, name: String) -> Result<()> { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id), "name": &name }; - let result = self.warehouses().delete_one(filter).await?; - - if result.deleted_count == 0 { - return Err(anyhow::anyhow!("Warehouse '{}' not found", name)); - } - Ok(()) - } - - // Catalog Operations - async fn create_catalog(&self, tenant_id: Uuid, catalog: Catalog) -> Result<()> { - // Catalog struct doesn't have tenant_id, so we need to wrap it or add it? - // Wait, Catalog struct in model.rs: - // pub struct Catalog { pub name: String, pub warehouse_name: Option, pub storage_location: Option, pub properties: HashMap } - // It doesn't have tenant_id. - // In Postgres we added a column. In Mongo we can wrap it in a document or just add the field dynamically if we use Document. - // But we are using typed Collection. - // We should probably use a wrapper struct for storage or just use Document. - // Let's use Document for flexibility here since we need to add tenant_id context. - - let mut doc = doc! { - "id": to_bson_uuid(catalog.id), - "tenant_id": to_bson_uuid(tenant_id), - "name": &catalog.name, - "catalog_type": format!("{:?}", catalog.catalog_type), - "properties": mongodb::bson::to_bson(&catalog.properties)? - }; - - // Add optional fields - if let Some(ref warehouse_name) = catalog.warehouse_name { - doc.insert("warehouse_name", warehouse_name); - } - if let Some(ref storage_location) = catalog.storage_location { - doc.insert("storage_location", storage_location); - } - if let Some(ref federated_config) = catalog.federated_config { - doc.insert("federated_config", mongodb::bson::to_bson(federated_config)?); - } - - self.db.collection::("catalogs").insert_one(doc).await?; - Ok(()) - } - - async fn get_catalog(&self, tenant_id: Uuid, name: String) -> Result> { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id), "name": name }; - let doc = self.db.collection::("catalogs").find_one(filter).await?; - Ok(doc) - } - - async fn list_catalogs(&self, tenant_id: Uuid) -> Result> { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id) }; - let cursor = self.db.collection::("catalogs").find(filter).await?; - let catalogs: Vec = cursor.try_collect().await?; - Ok(catalogs) - } - - async fn update_catalog(&self, tenant_id: Uuid, name: String, updates: pangolin_core::model::CatalogUpdate) -> Result { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id), "name": &name }; - let mut update_doc = doc! {}; - - if let Some(warehouse_name) = updates.warehouse_name { - update_doc.insert("warehouse_name", warehouse_name); - } - if let Some(storage_location) = updates.storage_location { - update_doc.insert("storage_location", storage_location); - } - if let Some(properties) = updates.properties { - update_doc.insert("properties", bson::to_bson(&properties)?); - } - - if update_doc.is_empty() { - return self.get_catalog(tenant_id, name).await? - .ok_or_else(|| anyhow::anyhow!("Catalog not found")); - } - - let update = doc! { "$set": update_doc }; - self.db.collection::("catalogs").update_one(filter, update).await?; - - self.get_catalog(tenant_id, name).await? - .ok_or_else(|| anyhow::anyhow!("Catalog not found")) - } - - async fn delete_catalog(&self, tenant_id: Uuid, name: String) -> Result<()> { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id), "name": &name }; // For catalog - // For children, filter is slightly different (catalog_name property) or similar - let child_filter = doc! { "tenant_id": to_bson_uuid(tenant_id), "catalog_name": &name }; - - // 1. Tags - self.db.collection::("tags").delete_many(child_filter.clone()).await?; - // 2. Branches - self.db.collection::("branches").delete_many(child_filter.clone()).await?; - // 3. Assets - self.db.collection::("assets").delete_many(child_filter.clone()).await?; - // 4. Namespaces - self.db.collection::("namespaces").delete_many(child_filter.clone()).await?; - - // 5. Catalog - let result = self.db.collection::("catalogs").delete_one(filter).await?; - - if result.deleted_count == 0 { - return Err(anyhow::anyhow!("Catalog '{}' not found", name)); - } - Ok(()) - } - - // Namespace Operations - async fn create_namespace(&self, tenant_id: Uuid, catalog_name: &str, namespace: Namespace) -> Result<()> { - let doc = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "name": &namespace.name, // Vec -> Array - "properties": mongodb::bson::to_bson(&namespace.properties)? - }; - self.db.collection::("namespaces").insert_one(doc).await?; - Ok(()) - } - - async fn get_namespace(&self, tenant_id: Uuid, catalog_name: &str, namespace: Vec) -> Result> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "name": namespace - }; - let doc = self.db.collection::("namespaces").find_one(filter).await?; - Ok(doc) - } - - async fn list_namespaces(&self, tenant_id: Uuid, catalog_name: &str, parent: Option) -> Result> { - let mut filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name - }; - - if let Some(p) = parent { - // This is tricky. Namespace name is Vec. - // "parent" usually implies a hierarchy. - // If parent is "a.b", we want "a.b.c", "a.b.d". - // We can filter where "name" starts with the parent components. - // But `parent` argument is String (dot joined?). The trait says `parent: Option`. - // In Postgres we did LIKE 'parent%'. - // Here we need to match array prefix. - // Let's assume parent string is dot-separated or something. - // Actually, `Namespace` struct has `name: Vec`. - // If parent is provided, we should convert it to Vec and check if it's a prefix. - // But `parent` is just a string. - // Let's assume for now we just list all and filter in memory or implement prefix match if possible. - // MVP: List all for catalog. - } - - let cursor = self.db.collection::("namespaces").find(filter).await?; - let namespaces: Vec = cursor.try_collect().await?; - Ok(namespaces) - } - - async fn delete_namespace(&self, tenant_id: Uuid, catalog_name: &str, namespace: Vec) -> Result<()> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "name": namespace - }; - self.db.collection::("namespaces").delete_one(filter).await?; - Ok(()) - } - - async fn update_namespace_properties(&self, tenant_id: Uuid, catalog_name: &str, namespace: Vec, properties: HashMap) -> Result<()> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "name": namespace - }; - - // We need to merge properties. - // $set: { "properties.key": "value" } - let mut set_doc = doc! {}; - for (k, v) in properties { - set_doc.insert(format!("properties.{}", k), v); - } - - let update = doc! { "$set": set_doc }; - self.db.collection::("namespaces").update_one(filter, update).await?; - Ok(()) - } - - // Asset Operations - async fn create_asset(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec, asset: Asset) -> Result<()> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "branch": &branch_name, - "namespace": &namespace, - "name": &asset.name - }; - - let doc = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "branch": &branch_name, - "namespace": namespace, - "id": to_bson_uuid(asset.id), - "name": &asset.name, - "kind": format!("{:?}", asset.kind), - "location": &asset.location, - "properties": mongodb::bson::to_bson(&asset.properties)? - }; - - let options = ReplaceOptions::builder().upsert(true).build(); - self.db.collection::("assets").replace_one(filter, doc).with_options(options).await?; - Ok(()) - } - - async fn get_asset(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec, name: String) -> Result> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "branch": branch_name, - "namespace": namespace, - "name": name - }; - // Note: Branch support in filter if needed, but usually asset name is unique in namespace? - // Or is it versioned by branch? - // In Postgres we had `active_branch`. - // Let's stick to the filter. - - let doc = self.db.collection::("assets").find_one(filter).await?; - - if let Some(d) = doc { - // Manual deserialization because we stored flattened fields - let kind_str = d.get_str("kind")?; - let kind = match kind_str { - "IcebergTable" => AssetType::IcebergTable, - "View" => AssetType::View, - _ => AssetType::IcebergTable, - }; - - let properties: HashMap = mongodb::bson::from_bson(d.get("properties").unwrap().clone())?; - - let id_bson = d.get("id").ok_or(anyhow::anyhow!("Missing id"))?; - let id = from_bson_uuid(id_bson)?; - - Ok(Some(Asset { - id, - name: d.get_str("name")?.to_string(), - kind, - location: d.get_str("location").unwrap_or("").to_string(), - properties, - })) - } else { - Ok(None) - } - } - - async fn list_assets(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec) -> Result> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "branch": branch_name, - "namespace": namespace - }; - let cursor = self.db.collection::("assets").find(filter).await?; - let docs: Vec = cursor.try_collect().await?; - - let mut assets = Vec::new(); - for d in docs { - let kind_str = d.get_str("kind")?; - let kind = match kind_str { - "IcebergTable" => AssetType::IcebergTable, - "View" => AssetType::View, - _ => AssetType::IcebergTable, - }; - - let properties: HashMap = mongodb::bson::from_bson(d.get("properties").unwrap().clone())?; - let id = if let Ok(i) = d.get("id").ok_or(anyhow::anyhow!("Missing id")).and_then(|b| from_bson_uuid(b)) { - i - } else { - Uuid::new_v4() // Fallback if old data - }; - - assets.push(Asset { - id, - name: d.get_str("name")?.to_string(), - kind, - location: d.get_str("location").unwrap_or("").to_string(), - properties, - }); - } - Ok(assets) - } - - async fn get_asset_by_id(&self, tenant_id: Uuid, asset_id: Uuid) -> Result)>> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "id": to_bson_uuid(asset_id) - }; - let d = self.db.collection::("assets").find_one(filter).await?; - - if let Some(d) = d { - let catalog_name = d.get_str("catalog_name")?.to_string(); - let namespace = d.get_array("namespace")?.iter().map(|v| v.as_str().unwrap().to_string()).collect(); - let kind_str = d.get_str("kind")?; - let kind = match kind_str { - "IcebergTable" => AssetType::IcebergTable, - "View" => AssetType::View, - _ => AssetType::IcebergTable, - }; - - let properties: HashMap = mongodb::bson::from_bson(d.get("properties").unwrap().clone())?; - - let asset = Asset { - id: asset_id, - name: d.get_str("name")?.to_string(), - kind, - location: d.get_str("location").unwrap_or("").to_string(), - properties, - }; - - Ok(Some((asset, catalog_name, namespace))) - } else { - Ok(None) - } - } - - async fn delete_asset(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec, name: String) -> Result<()> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "branch": branch_name, - "namespace": namespace, - "name": name - }; - self.db.collection::("assets").delete_one(filter).await?; - Ok(()) - } - - async fn rename_asset(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, source_namespace: Vec, source_name: String, dest_namespace: Vec, dest_name: String) -> Result<()> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "branch": &branch_name, - "namespace": source_namespace, - "name": source_name - }; - - let update = doc! { - "$set": { - "namespace": dest_namespace, - "name": dest_name - } - }; - - self.db.collection::("assets").update_one(filter, update).await?; - Ok(()) - } - - async fn count_namespaces(&self, tenant_id: Uuid) -> Result { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id) }; - let count = self.namespaces().count_documents(filter).await?; - Ok(count as usize) - } - - async fn count_assets(&self, tenant_id: Uuid) -> Result { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id) }; - let count = self.assets().count_documents(filter).await?; - Ok(count as usize) - } - - // Branch Operations - async fn create_branch(&self, tenant_id: Uuid, catalog_name: &str, branch: Branch) -> Result<()> { - let doc = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "name": &branch.name, - "head_commit_id": branch.head_commit_id, - "branch_type": format!("{:?}", branch.branch_type), - "assets": &branch.assets - }; - self.db.collection::("branches").insert_one(doc).await?; - Ok(()) - } - - async fn get_branch(&self, tenant_id: Uuid, catalog_name: &str, name: String) -> Result> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "name": name - }; - let doc = self.db.collection::("branches").find_one(filter).await?; - - if let Some(d) = doc { - let type_str = d.get_str("branch_type")?; - let branch_type = match type_str { - "Ingest" => pangolin_core::model::BranchType::Ingest, - "Experimental" => pangolin_core::model::BranchType::Experimental, - _ => pangolin_core::model::BranchType::Experimental, - }; - - Ok(Some(Branch { - name: d.get_str("name")?.to_string(), - head_commit_id: mongodb::bson::from_bson(d.get("head_commit_id").unwrap().clone())?, - branch_type, - assets: mongodb::bson::from_bson(d.get("assets").unwrap().clone())?, - })) - } else { - Ok(None) - } - } - - async fn list_branches(&self, tenant_id: Uuid, catalog_name: &str) -> Result> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name - }; - let cursor = self.db.collection::("branches").find(filter).await?; - let docs: Vec = cursor.try_collect().await?; - - let mut branches = Vec::new(); - for d in docs { - let type_str = d.get_str("branch_type")?; - let branch_type = match type_str { - "Ingest" => pangolin_core::model::BranchType::Ingest, - "Experimental" => pangolin_core::model::BranchType::Experimental, - _ => pangolin_core::model::BranchType::Experimental, - }; - - branches.push(Branch { - name: d.get_str("name")?.to_string(), - head_commit_id: mongodb::bson::from_bson(d.get("head_commit_id").unwrap().clone())?, - branch_type, - assets: mongodb::bson::from_bson(d.get("assets").unwrap().clone())?, - }); - } - Ok(branches) - } - - async fn merge_branch(&self, tenant_id: Uuid, catalog_name: &str, target_branch: String, source_branch: String) -> Result<()> { - let source = self.get_branch(tenant_id, catalog_name, source_branch.clone()).await? - .ok_or_else(|| anyhow::anyhow!("Source branch not found"))?; - - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "name": target_branch - }; - - let update = doc! { - "$set": { - "head_commit_id": source.head_commit_id - } - }; - - self.db.collection::("branches").update_one(filter, update).await?; - Ok(()) - } - - // Token Management - async fn list_active_tokens(&self, _tenant_id: Uuid, user_id: Uuid) -> Result> { - let filter = doc! { "user_id": to_bson_uuid(user_id) }; - let cursor = self.active_tokens().find(filter).await?; - let docs: Vec = cursor.try_collect().await?; - - let mut tokens = Vec::new(); - for d in docs { - tokens.push(TokenInfo { - id: from_bson_uuid(d.get("token_id").ok_or(anyhow::anyhow!("Missing token_id"))?)?, - tenant_id: Uuid::default(), // Not stored in mongo active_tokens - user_id: from_bson_uuid(d.get("user_id").ok_or(anyhow::anyhow!("Missing user_id"))?)?, - username: "unknown".to_string(), // Would need join - token: d.get_str("token").ok().map(|s| s.to_string()), - expires_at: mongodb::bson::from_bson(d.get("expires_at").unwrap().clone())?, - created_at: Utc::now(), // Not stored - is_valid: true, - }); - } - Ok(tokens) - } - - async fn store_token(&self, token_info: TokenInfo) -> Result<()> { - let doc = doc! { - "token_id": to_bson_uuid(token_info.id), - "user_id": to_bson_uuid(token_info.user_id), - "token": token_info.token.unwrap_or_default(), - "expires_at": token_info.expires_at - }; - self.active_tokens().insert_one(doc).await?; - Ok(()) - } - - // System Settings - async fn get_system_settings(&self, tenant_id: Uuid) -> Result { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id) }; - let doc = self.system_settings().find_one(filter).await?; - - if let Some(d) = doc { - Ok(mongodb::bson::from_bson(d.get("settings").unwrap().clone())?) - } else { - Ok(SystemSettings { - allow_public_signup: None, - default_warehouse_bucket: None, - default_retention_days: None, - smtp_host: None, - smtp_port: None, - smtp_user: None, - smtp_password: None, - }) - } - } - - async fn update_system_settings(&self, tenant_id: Uuid, settings: SystemSettings) -> Result { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id) }; - let update = doc! { - "$set": { - "settings": mongodb::bson::to_bson(&settings)? - } - }; - - let options = mongodb::options::UpdateOptions::builder().upsert(true).build(); - self.system_settings().update_one(filter, update).with_options(options).await?; - Ok(settings) - } - - // Service User Methods - async fn create_service_user(&self, service_user: pangolin_core::user::ServiceUser) -> Result<()> { - let doc = mongodb::bson::to_document(&service_user)?; - self.service_users().insert_one(doc).await?; - Ok(()) - } - - async fn get_service_user(&self, id: Uuid) -> Result> { - let filter = doc! { "_id": id.to_string() }; - if let Some(doc) = self.service_users().find_one(filter).await? { - Ok(Some(mongodb::bson::from_document(doc)?)) - } else { - Ok(None) - } - } - - async fn get_service_user_by_api_key_hash(&self, api_key_hash: &str) -> Result> { - let filter = doc! { "api_key_hash": api_key_hash }; - if let Some(doc) = self.service_users().find_one(filter).await? { - Ok(Some(mongodb::bson::from_document(doc)?)) - } else { - Ok(None) - } - } - - async fn list_service_users(&self, tenant_id: Uuid) -> Result> { - let filter = doc! { "tenant_id": tenant_id.to_string() }; - let mut cursor = self.service_users().find(filter).await?; - let mut users = Vec::new(); - - while cursor.advance().await? { - users.push(mongodb::bson::from_document(cursor.deserialize_current()?)?) -; - } - - Ok(users) - } - - async fn update_service_user( - &self, - id: Uuid, - name: Option, - description: Option, - active: Option, - ) -> Result<()> { - let filter = doc! { "_id": id.to_string() }; - let mut update_doc = doc! {}; - - if let Some(n) = name { - update_doc.insert("name", n); - } - if let Some(d) = description { - update_doc.insert("description", d); - } - if let Some(a) = active { - update_doc.insert("active", a); - } - - if !update_doc.is_empty() { - let update = doc! { "$set": update_doc }; - self.service_users().update_one(filter, update).await?; - } - - Ok(()) - } - - async fn delete_service_user(&self, id: Uuid) -> Result<()> { - let filter = doc! { "_id": id.to_string() }; - self.service_users().delete_one(filter).await?; - Ok(()) - } - - async fn update_service_user_last_used(&self, id: Uuid, timestamp: chrono::DateTime) -> Result<()> { - let filter = doc! { "_id": id.to_string() }; - let update = doc! { "$set": { "last_used": timestamp.to_rfc3339() } }; - self.service_users().update_one(filter, update).await?; - Ok(()) - } - - // Merge Operation Methods - async fn create_merge_operation(&self, operation: pangolin_core::model::MergeOperation) -> Result<()> { - let doc = mongodb::bson::to_document(&operation)?; - self.merge_operations().insert_one(doc).await?; - Ok(()) - } - - async fn get_merge_operation(&self, operation_id: Uuid) -> Result> { - let filter = doc! { "_id": operation_id.to_string() }; - if let Some(doc) = self.merge_operations().find_one(filter).await? { - Ok(Some(mongodb::bson::from_document(doc)?)) - } else { - Ok(None) - } - } - - async fn list_merge_operations(&self, tenant_id: Uuid, catalog_name: &str) -> Result> { - let filter = doc! { - "tenant_id": tenant_id.to_string(), - "catalog_name": catalog_name - }; - let mut cursor = self.merge_operations().find(filter).await?; - let mut operations = Vec::new(); - - while cursor.advance().await? { - operations.push(mongodb::bson::from_document(cursor.deserialize_current()?)?); - } - - Ok(operations) - } - - async fn update_merge_operation_status(&self, operation_id: Uuid, status: pangolin_core::model::MergeStatus) -> Result<()> { - let filter = doc! { "_id": operation_id.to_string() }; - let status_str = format!("{:?}", status); - let update = doc! { "$set": { "status": status_str } }; - self.merge_operations().update_one(filter, update).await?; - Ok(()) - } - - async fn complete_merge_operation(&self, operation_id: Uuid, result_commit_id: Uuid) -> Result<()> { - let filter = doc! { "_id": operation_id.to_string() }; - let update = doc! { - "$set": { - "status": "Completed", - "result_commit_id": result_commit_id.to_string(), - "completed_at": chrono::Utc::now().to_rfc3339() - } - }; - self.merge_operations().update_one(filter, update).await?; - Ok(()) - } - - async fn abort_merge_operation(&self, operation_id: Uuid) -> Result<()> { - let filter = doc! { "_id": operation_id.to_string() }; - let update = doc! { - "$set": { - "status": "Aborted", - "completed_at": chrono::Utc::now().to_rfc3339() - } - }; - self.merge_operations().update_one(filter, update).await?; - Ok(()) - } - - // Merge Conflict Methods - async fn create_merge_conflict(&self, conflict: pangolin_core::model::MergeConflict) -> Result<()> { - let doc = mongodb::bson::to_document(&conflict)?; - self.merge_conflicts().insert_one(doc).await?; - Ok(()) - } - - async fn get_merge_conflict(&self, conflict_id: Uuid) -> Result> { - let filter = doc! { "_id": conflict_id.to_string() }; - if let Some(doc) = self.merge_conflicts().find_one(filter).await? { - Ok(Some(mongodb::bson::from_document(doc)?)) - } else { - Ok(None) - } - } - - async fn list_merge_conflicts(&self, operation_id: Uuid) -> Result> { - let filter = doc! { "merge_operation_id": operation_id.to_string() }; - let mut cursor = self.merge_conflicts().find(filter).await?; - let mut conflicts = Vec::new(); - - while cursor.advance().await? { - conflicts.push(mongodb::bson::from_document(cursor.deserialize_current()?)?); - } - - Ok(conflicts) - } - - async fn resolve_merge_conflict(&self, conflict_id: Uuid, resolution: pangolin_core::model::ConflictResolution) -> Result<()> { - let filter = doc! { "_id": conflict_id.to_string() }; - let resolution_doc = mongodb::bson::to_document(&resolution)?; - let update = doc! { "$set": { "resolution": resolution_doc } }; - self.merge_conflicts().update_one(filter, update).await?; - Ok(()) - } - - async fn add_conflict_to_operation(&self, operation_id: Uuid, conflict_id: Uuid) -> Result<()> { - let filter = doc! { "_id": operation_id.to_string() }; - let update = doc! { "$addToSet": { "conflicts": conflict_id.to_string() } }; - self.merge_operations().update_one(filter, update).await?; - Ok(()) - } - - // Federated Catalog Operations - async fn sync_federated_catalog(&self, tenant_id: Uuid, catalog_name: &str) -> Result<()> { - let stats = SyncStats { - last_synced_at: Some(Utc::now()), - sync_status: "Success".to_string(), - tables_synced: 0, - namespaces_synced: 0, - error_message: None, - }; - - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name - }; - let update = doc! { - "$set": { - "stats": mongodb::bson::to_bson(&stats)? - } - }; - - let options = mongodb::options::UpdateOptions::builder().upsert(true).build(); - self.federated_sync_stats().update_one(filter, update).with_options(options).await?; - Ok(()) - } - - async fn get_federated_catalog_stats(&self, tenant_id: Uuid, catalog_name: &str) -> Result { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name - }; - let doc = self.federated_sync_stats().find_one(filter).await?; - - if let Some(d) = doc { - Ok(mongodb::bson::from_bson(d.get("stats").unwrap().clone())?) - } else { - Ok(SyncStats { - last_synced_at: None, - sync_status: "Never Synced".to_string(), - tables_synced: 0, - namespaces_synced: 0, - error_message: None, - }) - } - } - - // Commit Operations - async fn create_commit(&self, tenant_id: Uuid, commit: Commit) -> Result<()> { - let mut doc = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "id": to_bson_uuid(commit.id), - "timestamp": commit.timestamp, - "author": &commit.author, - "message": &commit.message, - "operations": mongodb::bson::to_bson(&commit.operations)? - }; - if let Some(parent_id) = commit.parent_id { - doc.insert("parent_id", to_bson_uuid(parent_id)); - } else { - doc.insert("parent_id", Bson::Null); - } - self.db.collection::("commits").insert_one(doc).await?; - Ok(()) - } - - async fn get_commit(&self, tenant_id: Uuid, id: Uuid) -> Result> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "id": to_bson_uuid(id) - }; - let doc = self.db.collection::("commits").find_one(filter).await?; - - if let Some(d) = doc { - Ok(Some(Commit { - id: mongodb::bson::from_bson(d.get("id").unwrap().clone())?, - parent_id: mongodb::bson::from_bson(d.get("parent_id").unwrap().clone())?, - timestamp: d.get_i64("timestamp")?, - author: d.get_str("author")?.to_string(), - message: d.get_str("message")?.to_string(), - operations: mongodb::bson::from_bson(d.get("operations").unwrap().clone())?, - })) - } else { - Ok(None) - } - } - - // File Operations - - async fn read_file(&self, path: &str) -> Result> { - // Use metadata cache for metadata.json files - if path.ends_with("metadata.json") || path.ends_with(".metadata.json") { - return self.metadata_cache.get_or_fetch(path, || async { - self.read_file_uncached(path).await - }).await; - } - - // Non-metadata files bypass cache - self.read_file_uncached(path).await - } - - - async fn write_file(&self, path: &str, data: Vec) -> Result<()> { - // Invalidate metadata cache on write - if path.ends_with("metadata.json") || path.ends_with(".metadata.json") { - self.metadata_cache.invalidate(path).await; - } - - // Try to look up warehouse credentials first - if let Some(warehouse) = self.get_warehouse_for_location(path).await? { - if path.starts_with("s3://") || path.starts_with("az://") || path.starts_with("gs://") { - // Use cached object store - let cache_key = self.get_object_store_cache_key(&warehouse.storage_config, path); - let store = self.object_store_cache.get_or_insert(cache_key, || { - Arc::new(crate::object_store_factory::create_object_store(&warehouse.storage_config, path).unwrap()) - }); - - // Extract key relative to bucket - let key = if let Some(rest) = path.strip_prefix("s3://").or_else(|| path.strip_prefix("az://")).or_else(|| path.strip_prefix("gs://")) { - rest.split_once('/').map(|(_, k)| k).unwrap_or(rest) - } else { - path - }; - - store.put(&ObjPath::from(key), data.into()).await?; - return Ok(()); - } - } - - // Fallback to existing logic (Global Env Vars) - if let Some(rest) = path.strip_prefix("s3://") { - let (bucket, key) = rest.split_once('/').ok_or_else(|| anyhow::anyhow!("Invalid S3 path"))?; - - let mut builder = AmazonS3Builder::new() - .with_bucket_name(bucket) - .with_allow_http(true); - - if let Ok(endpoint) = std::env::var("S3_ENDPOINT") { - builder = builder.with_endpoint(endpoint); - } - if let Ok(key_id) = std::env::var("AWS_ACCESS_KEY_ID") { - builder = builder.with_access_key_id(key_id); - } - if let Ok(secret) = std::env::var("AWS_SECRET_ACCESS_KEY") { - builder = builder.with_secret_access_key(secret); - } - if let Ok(region) = std::env::var("AWS_REGION") { - builder = builder.with_region(region); - } - - let store = builder.build()?; - let location = ObjPath::from(key); - store.put(&location, data.into()).await?; - Ok(()) - } else { - Err(anyhow::anyhow!("Only s3:// paths are supported in Mongo store")) - } - } - - - // Tag Operations - async fn create_tag(&self, tenant_id: Uuid, catalog_name: &str, tag: Tag) -> Result<()> { - let doc = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "name": &tag.name, - "commit_id": to_bson_uuid(tag.commit_id) - }; - self.db.collection::("tags").insert_one(doc).await?; - Ok(()) - } - - async fn get_tag(&self, tenant_id: Uuid, catalog_name: &str, name: String) -> Result> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "name": name - }; - let doc = self.db.collection::("tags").find_one(filter).await?; - - if let Some(d) = doc { - Ok(Some(Tag { - name: d.get_str("name")?.to_string(), - commit_id: mongodb::bson::from_bson(d.get("commit_id").unwrap().clone())?, - })) - } else { - Ok(None) - } - } - - async fn list_tags(&self, tenant_id: Uuid, catalog_name: &str) -> Result> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name - }; - let cursor = self.db.collection::("tags").find(filter).await?; - let docs: Vec = cursor.try_collect().await?; - - let mut tags = Vec::new(); - for d in docs { - tags.push(Tag { - name: d.get_str("name")?.to_string(), - commit_id: mongodb::bson::from_bson(d.get("commit_id").unwrap().clone())?, - }); - } - Ok(tags) - } - - async fn delete_tag(&self, tenant_id: Uuid, catalog_name: &str, name: String) -> Result<()> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "name": name - }; - self.db.collection::("tags").delete_one(filter).await?; - Ok(()) - } - - // Audit Operations - async fn log_audit_event(&self, tenant_id: Uuid, event: AuditLogEntry) -> Result<()> { - let doc = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "id": to_bson_uuid(event.id), - "user_id": event.user_id.map(to_bson_uuid).unwrap_or(Bson::Null), - "username": &event.username, - "action": format!("{:?}", event.action), - "resource_type": format!("{:?}", event.resource_type), - "resource_id": event.resource_id.map(to_bson_uuid).unwrap_or(Bson::Null), - "resource_name": &event.resource_name, - "timestamp": mongodb::bson::DateTime::from_chrono(event.timestamp), - "ip_address": event.ip_address.as_ref().map(|s| s.as_str()).unwrap_or(""), - "user_agent": event.user_agent.as_ref().map(|s| s.as_str()).unwrap_or(""), - "result": format!("{:?}", event.result), - "error_message": event.error_message.as_ref().map(|s| s.as_str()).unwrap_or(""), - "metadata": mongodb::bson::to_bson(&event.metadata)? - }; - self.db.collection::("audit_logs").insert_one(doc).await?; - Ok(()) - } - - async fn list_audit_events(&self, tenant_id: Uuid, filter: Option) -> Result> { - let mut query = doc! { "tenant_id": to_bson_uuid(tenant_id) }; - - // Build filter conditions - if let Some(ref f) = filter { - if let Some(user_id) = f.user_id { - query.insert("user_id", to_bson_uuid(user_id)); - } - if let Some(ref action) = f.action { - query.insert("action", format!("{:?}", action)); - } - if let Some(ref resource_type) = f.resource_type { - query.insert("resource_type", format!("{:?}", resource_type)); - } - if let Some(resource_id) = f.resource_id { - query.insert("resource_id", to_bson_uuid(resource_id)); - } - if let Some(start_time) = f.start_time { - query.insert("timestamp", doc! { "$gte": mongodb::bson::DateTime::from_chrono(start_time) }); - } - if let Some(end_time) = f.end_time { - let existing = query.get_document_mut("timestamp").ok(); - if let Some(existing_doc) = existing { - existing_doc.insert("$lte", mongodb::bson::DateTime::from_chrono(end_time)); - } else { - query.insert("timestamp", doc! { "$lte": mongodb::bson::DateTime::from_chrono(end_time) }); - } - } - if let Some(ref result) = f.result { - query.insert("result", format!("{:?}", result)); - } - } - - // Build options with pagination - let limit = filter.as_ref().and_then(|f| f.limit).unwrap_or(100) as i64; - let skip = filter.as_ref().and_then(|f| f.offset).map(|o| o as u64); - - let mut options = mongodb::options::FindOptions::builder() - .sort(doc! { "timestamp": -1 }) - .limit(limit) - .build(); - - if let Some(skip_val) = skip { - options.skip = Some(skip_val); - } - - let cursor = self.db.collection::("audit_logs") - .find(query) - .with_options(options) - .await?; - let docs: Vec = cursor.try_collect().await?; - - let mut events = Vec::new(); - for d in docs { - // Parse action enum from string - let action_str = d.get_str("action")?; - let action = serde_json::from_str(&format!("\"{}\"" , action_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::AuditAction::CreateCatalog); - - // Parse resource_type enum from string - let resource_type_str = d.get_str("resource_type")?; - let resource_type = serde_json::from_str(&format!("\"{}\"", resource_type_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::ResourceType::Catalog); - - // Parse result enum from string - let result_str = d.get_str("result")?; - let result = serde_json::from_str(&format!("\"{}\"", result_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::AuditResult::Success); - - events.push(AuditLogEntry { - id: mongodb::bson::from_bson(d.get("id").unwrap().clone())?, - tenant_id, - user_id: d.get("user_id").and_then(|b| from_bson_uuid(b).ok()), - username: d.get_str("username")?.to_string(), - action, - resource_type, - resource_id: d.get("resource_id").and_then(|b| from_bson_uuid(b).ok()), - resource_name: d.get_str("resource_name")?.to_string(), - timestamp: d.get_datetime("timestamp")?.to_chrono(), - ip_address: d.get_str("ip_address").ok().filter(|s| !s.is_empty()).map(|s| s.to_string()), - user_agent: d.get_str("user_agent").ok().filter(|s| !s.is_empty()).map(|s| s.to_string()), - result, - error_message: d.get_str("error_message").ok().filter(|s| !s.is_empty()).map(|s| s.to_string()), - metadata: mongodb::bson::from_bson(d.get("metadata").unwrap().clone())?, - }); - } - Ok(events) - } - - async fn get_audit_event(&self, tenant_id: Uuid, event_id: Uuid) -> Result> { - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "id": to_bson_uuid(event_id) - }; - - let doc = self.db.collection::("audit_logs").find_one(filter).await?; - - if let Some(d) = doc { - let action_str = d.get_str("action")?; - let action = serde_json::from_str(&format!("\"{}\"", action_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::AuditAction::CreateCatalog); - - let resource_type_str = d.get_str("resource_type")?; - let resource_type = serde_json::from_str(&format!("\"{}\"", resource_type_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::ResourceType::Catalog); - - let result_str = d.get_str("result")?; - let result = serde_json::from_str(&format!("\"{}\"", result_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::AuditResult::Success); - - Ok(Some(AuditLogEntry { - id: mongodb::bson::from_bson(d.get("id").unwrap().clone())?, - tenant_id, - user_id: d.get("user_id").and_then(|b| from_bson_uuid(b).ok()), - username: d.get_str("username")?.to_string(), - action, - resource_type, - resource_id: d.get("resource_id").and_then(|b| from_bson_uuid(b).ok()), - resource_name: d.get_str("resource_name")?.to_string(), - timestamp: d.get_datetime("timestamp")?.to_chrono(), - ip_address: d.get_str("ip_address").ok().filter(|s| !s.is_empty()).map(|s| s.to_string()), - user_agent: d.get_str("user_agent").ok().filter(|s| !s.is_empty()).map(|s| s.to_string()), - result, - error_message: d.get_str("error_message").ok().filter(|s| !s.is_empty()).map(|s| s.to_string()), - metadata: mongodb::bson::from_bson(d.get("metadata").unwrap().clone())?, - })) - } else { - Ok(None) - } - } - - async fn count_audit_events(&self, tenant_id: Uuid, filter: Option) -> Result { - let mut query = doc! { "tenant_id": to_bson_uuid(tenant_id) }; - - // Build same filter conditions as list_audit_events - if let Some(ref f) = filter { - if let Some(user_id) = f.user_id { - query.insert("user_id", to_bson_uuid(user_id)); - } - if let Some(ref action) = f.action { - query.insert("action", format!("{:?}", action)); - } - if let Some(ref resource_type) = f.resource_type { - query.insert("resource_type", format!("{:?}", resource_type)); - } - if let Some(resource_id) = f.resource_id { - query.insert("resource_id", to_bson_uuid(resource_id)); - } - if let Some(start_time) = f.start_time { - query.insert("timestamp", doc! { "$gte": mongodb::bson::DateTime::from_chrono(start_time) }); - } - if let Some(end_time) = f.end_time { - let existing = query.get_document_mut("timestamp").ok(); - if let Some(existing_doc) = existing { - existing_doc.insert("$lte", mongodb::bson::DateTime::from_chrono(end_time)); - } else { - query.insert("timestamp", doc! { "$lte": mongodb::bson::DateTime::from_chrono(end_time) }); - } - } - if let Some(ref result) = f.result { - query.insert("result", format!("{:?}", result)); - } - } - - let count = self.db.collection::("audit_logs") - .count_documents(query) - .await? as usize; - - Ok(count) - } - - // User Operations - async fn create_user(&self, user: User) -> Result<()> { - self.users().insert_one(user).await?; - Ok(()) - } - - async fn get_user(&self, user_id: Uuid) -> Result> { - let filter = doc! { "id": to_bson_uuid(user_id) }; - let user = self.users().find_one(filter).await?; - Ok(user) - } - - async fn get_user_by_username(&self, username: &str) -> Result> { - let filter = doc! { "username": username }; - let user = self.users().find_one(filter).await?; - Ok(user) - } - - async fn list_users(&self, tenant_id: Option) -> Result> { - let filter = if let Some(tid) = tenant_id { - doc! { "tenant_id": to_bson_uuid(tid) } - } else { - doc! {} - }; - let cursor = self.users().find(filter).await?; - let users: Vec = cursor.try_collect().await?; - Ok(users) - } - - async fn update_user(&self, user: User) -> Result<()> { - let filter = doc! { "id": to_bson_uuid(user.id) }; - let mut doc = mongodb::bson::to_document(&user)?; - // Ensure UUIDs are stored as Binary Subtype 0 for consistency with filters - doc.insert("id", to_bson_uuid(user.id)); - if let Some(tid) = user.tenant_id { - doc.insert("tenant-id", to_bson_uuid(tid)); - } - let update = doc! { "$set": doc }; - self.db.collection::("users").update_one(filter, update).await?; - Ok(()) - } - - async fn delete_user(&self, user_id: Uuid) -> Result<()> { - let filter = doc! { "id": to_bson_uuid(user_id) }; - self.users().delete_one(filter).await?; - Ok(()) - } - - // Role Operations - async fn create_role(&self, role: Role) -> Result<()> { - self.roles().insert_one(role).await?; - Ok(()) - } - - async fn get_role(&self, role_id: Uuid) -> Result> { - let filter = doc! { "id": to_bson_uuid(role_id) }; - let role = self.roles().find_one(filter).await?; - Ok(role) - } - - async fn list_roles(&self, tenant_id: Uuid) -> Result> { - let filter = doc! { "tenant-id": to_bson_uuid(tenant_id) }; - let cursor = self.roles().find(filter).await?; - let roles: Vec = cursor.try_collect().await?; - Ok(roles) - } - - async fn delete_role(&self, role_id: Uuid) -> Result<()> { - let filter = doc! { "id": to_bson_uuid(role_id) }; - self.roles().delete_one(filter).await?; - Ok(()) - } - - async fn update_role(&self, role: Role) -> Result<()> { - let filter = doc! { "id": to_bson_uuid(role.id) }; - let mut doc = mongodb::bson::to_document(&role)?; - doc.insert("id", to_bson_uuid(role.id)); - doc.insert("tenant-id", to_bson_uuid(role.tenant_id)); - doc.insert("created-by", to_bson_uuid(role.created_by)); - - let update = doc! { "$set": doc }; - self.db.collection::("roles").update_one(filter, update).await?; - Ok(()) - } - - async fn assign_role(&self, user_role: UserRoleAssignment) -> Result<()> { - self.user_roles().insert_one(user_role).await?; - Ok(()) - } - - async fn revoke_role(&self, user_id: Uuid, role_id: Uuid) -> Result<()> { - let filter = doc! { - "user-id": to_bson_uuid(user_id), - "role-id": to_bson_uuid(role_id) - }; - self.user_roles().delete_one(filter).await?; - Ok(()) - } - - async fn get_user_roles(&self, user_id: Uuid) -> Result> { - let filter = doc! { "user-id": to_bson_uuid(user_id) }; - let cursor = self.user_roles().find(filter).await?; - let roles: Vec = cursor.try_collect().await?; - Ok(roles) - } - - // Permission Operations - async fn create_permission(&self, permission: Permission) -> Result<()> { - self.permissions().insert_one(permission).await?; - Ok(()) - } - - async fn revoke_permission(&self, permission_id: Uuid) -> Result<()> { - let filter = doc! { "id": to_bson_uuid(permission_id) }; - self.permissions().delete_one(filter).await?; - Ok(()) - } - - async fn list_user_permissions(&self, user_id: Uuid) -> Result> { - // 1. Fetch direct permissions - let filter = doc! { "user-id": to_bson_uuid(user_id) }; - let cursor = self.permissions().find(filter).await?; - let mut perms: Vec = cursor.try_collect().await?; - - // 2. Fetch role-based permissions - let user_roles = self.get_user_roles(user_id).await?; - for ur in user_roles { - if let Some(role) = self.get_role(ur.role_id).await? { - for grant in role.permissions { - perms.push(Permission { - id: Uuid::new_v4(), // Synthesized ID - user_id, - scope: grant.scope, - actions: grant.actions, - granted_by: role.created_by, - granted_at: role.created_at, - }); - } - } - } - - Ok(perms) - } - - async fn list_permissions(&self, tenant_id: Uuid) -> Result> { - // 1. Get all user IDs for the tenant - let user_filter = doc! { "tenant-id": to_bson_uuid(tenant_id) }; - let user_cursor = self.users().find(user_filter).await?; - let users: Vec = user_cursor.try_collect().await?; - let user_ids: Vec = users.iter().map(|u| to_bson_uuid(u.id)).collect(); - - if user_ids.is_empty() { - return Ok(vec![]); - } - - // 2. Get permissions for those users - let perm_filter = doc! { "user-id": { "$in": user_ids } }; - let perm_cursor = self.permissions().find(perm_filter).await?; - let perms: Vec = perm_cursor.try_collect().await?; - Ok(perms) - } - - // Maintenance Operations - async fn expire_snapshots(&self, _tenant_id: Uuid, _catalog_name: &str, _branch: Option, _namespace: Vec, _table: String, _retention_ms: i64) -> Result<()> { - Ok(()) - } - - // Business Metadata Operations - async fn upsert_business_metadata(&self, metadata: BusinessMetadata) -> Result<()> { - let filter = doc! { "asset-id": to_bson_uuid(metadata.asset_id) }; - let mut doc = mongodb::bson::to_document(&metadata)?; - doc.insert("asset-id", to_bson_uuid(metadata.asset_id)); - self.db.collection::("business_metadata") - .replace_one(filter, doc) - .upsert(true) - .await?; - Ok(()) - } - - async fn get_business_metadata(&self, asset_id: Uuid) -> Result> { - let filter = doc! { "asset-id": to_bson_uuid(asset_id) }; - let meta = self.business_metadata().find_one(filter).await?; - Ok(meta) - } - - async fn delete_business_metadata(&self, asset_id: Uuid) -> Result<()> { - let filter = doc! { "asset-id": to_bson_uuid(asset_id) }; - self.business_metadata().delete_one(filter).await?; - Ok(()) - } - - async fn search_assets(&self, tenant_id: Uuid, query: &str, tags: Option>) -> Result, String, Vec)>> { - let query_regex = mongodb::bson::Regex { - pattern: format!(".*{}.*", regex::escape(query)), - options: "i".to_string(), - }; - - let mut pipeline = vec![ - doc! { "$match": { "tenant_id": to_bson_uuid(tenant_id) } }, - doc! { - "$lookup": { - "from": "business_metadata", - "localField": "id", - "foreignField": "asset_id", - "as": "metadata" - } - }, - doc! { - "$unwind": { - "path": "$metadata", - "preserveNullAndEmptyArrays": true - } - }, - doc! { - "$match": { - "$or": [ - { "name": query_regex.clone() }, - { "metadata.description": query_regex } - ] - } - } - ]; - - if let Some(tag_list) = tags { - if !tag_list.is_empty() { - pipeline.push(doc! { - "$match": { - "metadata.tags": { "$all": tag_list } - } - }); - } - } - - let cursor = self.assets().aggregate(pipeline).await?; - let docs: Vec = cursor.try_collect().await?; - - let mut results = Vec::new(); - - for d in docs { - let metadata_doc = d.get_document("metadata").ok(); - - // Manual deserialization for Asset to ensure we get what we expect, - // though from_document works if struct matches. - // But we need catalog and namespace which are in the doc but not in the struct. - let asset: Asset = mongodb::bson::from_document(d.clone())?; - - let metadata = if let Some(md_doc) = metadata_doc { - if md_doc.is_empty() { None } else { Some(mongodb::bson::from_document(md_doc.clone())?) } - } else { None }; - - let catalog_name = d.get_str("catalog_name")?.to_string(); - let namespace_bson = d.get_array("namespace")?; - let namespace: Vec = namespace_bson.iter() - .map(|b| b.as_str().unwrap_or_default().to_string()) - .collect(); - - results.push((asset, metadata, catalog_name, namespace)); - } - - Ok(results) - } - - async fn search_catalogs(&self, tenant_id: Uuid, query: &str) -> Result> { - let query_regex = mongodb::bson::Regex { - pattern: format!(".*{}.*", regex::escape(query)), - options: "i".to_string(), - }; - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "name": query_regex - }; - let cursor = self.catalogs().find(filter).await?; - let catalogs: Vec = cursor.try_collect().await?; - Ok(catalogs) - } - - async fn search_namespaces(&self, tenant_id: Uuid, query: &str) -> Result> { - // Namespaces search with aggregation to output Docs - let query_regex = mongodb::bson::Regex { - pattern: format!(".*{}.*", regex::escape(query)), - options: "i".to_string(), - }; - - let pipeline = vec![ - doc! { "$match": { "tenant_id": to_bson_uuid(tenant_id) } } - ]; - - // self.namespaces() returns Collection. aggregate returns Cursor. - // BUT we need to call aggregate on collection. self.namespaces() is typed. - // We can call aggregate on typed collection but it returns Cursor. - let cursor = self.namespaces().aggregate(pipeline).await?; - let docs: Vec = cursor.try_collect().await?; - - let mut results = Vec::new(); - let query_lower = query.to_lowercase(); - - for d in docs { - let ns: Namespace = mongodb::bson::from_document(d.clone())?; - let catalog_name = d.get_str("catalog_name")?.to_string(); - - if ns.to_string().to_lowercase().contains(&query_lower) { - results.push((ns, catalog_name)); - } - } - Ok(results) - } - - async fn search_branches(&self, tenant_id: Uuid, query: &str) -> Result> { - // Use aggregate instead of find to get Document cursor easily and access catalog_name - let query_regex = mongodb::bson::Regex { - pattern: format!(".*{}.*", regex::escape(query)), - options: "i".to_string(), - }; - - // Explicitly filter by query in pipeline - let pipeline = vec![ - doc! { "$match": { "tenant_id": to_bson_uuid(tenant_id), "name": query_regex } } - ]; - - let cursor = self.branches().aggregate(pipeline).await?; - let docs: Vec = cursor.try_collect().await?; - - let mut results = Vec::new(); - for d in docs { - let branch: Branch = mongodb::bson::from_document(d.clone())?; - let catalog_name = d.get_str("catalog_name")?.to_string(); - results.push((branch, catalog_name)); - } - Ok(results) - } - - // Access Request Operations - async fn create_access_request(&self, request: AccessRequest) -> Result<()> { - self.access_requests().insert_one(request).await?; - Ok(()) - } - - async fn get_access_request(&self, id: Uuid) -> Result> { - let filter = doc! { "id": to_bson_uuid(id) }; - let req = self.access_requests().find_one(filter).await?; - Ok(req) - } - - async fn list_access_requests(&self, tenant_id: Uuid) -> Result> { - // AccessRequests stored with UserID/AssetID but not TenantID directly? - // Struct has: id, user_id, asset_id... - // User has tenant_id. - // To filter by tenant_id, we need a join (lookup) or we store tenant_id denormalized on AccessRequest? - // SQL implementation joins Users. - // Mongo: $lookup. - - // Creating aggregation pipeline: - let pipeline = vec![ - doc! { - "$lookup": { - "from": "users", - "localField": "user-id", - "foreignField": "id", - "as": "user" - } - }, - doc! { "$unwind": "$user" }, - doc! { "$match": { "user.tenant-id": to_bson_uuid(tenant_id) } }, - // Project back to AccessRequest root fields only? - // "replaceRoot"? Or simple map. - doc! { - "$project": { - "user": 0 // remove joined field to match struct - } - } - ]; - - let cursor = self.access_requests().aggregate(pipeline).await?; - // Cursor returns Documents, need to deserialize. - // aggregate returns Cursor. - let docs: Vec = cursor.try_collect().await?; - - let mut reqs = Vec::new(); - for d in docs { - reqs.push(mongodb::bson::from_document(d)?); - } - Ok(reqs) - } - - async fn update_access_request(&self, request: AccessRequest) -> Result<()> { - let filter = doc! { "id": to_bson_uuid(request.id) }; - self.access_requests().replace_one(filter, request).await?; - Ok(()) - } - - async fn remove_orphan_files(&self, _tenant_id: Uuid, _catalog_name: &str, _branch: Option, _namespace: Vec, _table: String, _older_than_ms: i64) -> Result<()> { - Ok(()) - } - - // Metadata IO - async fn get_metadata_location(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec, table: String) -> Result> { - let asset = self.get_asset(tenant_id, catalog_name, branch, namespace, table).await?; - if let Some(asset) = asset { - // First check if metadata_location is explicitly set in properties - if let Some(loc) = asset.properties.get("metadata_location") { - return Ok(Some(loc.clone())); - } - // Fall back to the asset's location field - return Ok(Some(asset.location)); - } - Ok(None) - } - - async fn update_metadata_location(&self, tenant_id: Uuid, catalog_name: &str, branch: Option, namespace: Vec, table: String, expected_location: Option, new_location: String) -> Result<()> { - let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let filter = doc! { - "tenant_id": to_bson_uuid(tenant_id), - "catalog_name": catalog_name, - "branch": branch_name, - "namespace": namespace, - "name": table - }; - - // CAS Logic - need to check both location field and properties.metadata_location - // because get_metadata_location falls back to location if metadata_location doesn't exist - let mut query = filter.clone(); - if let Some(expected) = expected_location { - // Match if either properties.metadata_location equals expected OR location equals expected (and metadata_location doesn't exist) - query.insert("$or", vec![ - doc! { "properties.metadata_location": &expected }, - doc! { "location": &expected, "properties.metadata_location": doc! { "$exists": false } } - ]); - } else { - // expected is None, meaning it shouldn't exist or should be null. - query.insert("properties.metadata_location", doc! { "$exists": false }); - } - - let update = doc! { - "$set": { - "properties.metadata_location": new_location - } - }; - - let result = self.db.collection::("assets").update_one(query, update).await?; - - if result.matched_count == 0 { - return Err(anyhow::anyhow!("CAS check failed: Metadata location mismatch or asset not found")); - } - - Ok(()) - } - - // Token Revocation Operations - async fn revoke_token(&self, token_id: Uuid, expires_at: chrono::DateTime, reason: Option) -> Result<()> { - let revoked = pangolin_core::token::RevokedToken::new(token_id, expires_at, reason); - self.db.collection("revoked_tokens").insert_one(revoked).await?; - Ok(()) - } - - async fn is_token_revoked(&self, token_id: Uuid) -> Result { - let filter = doc! { "token_id": to_bson_uuid(token_id) }; - let result = self.db.collection::("revoked_tokens") - .find_one(filter) - .await?; - Ok(result.is_some()) - } - - async fn cleanup_expired_tokens(&self) -> Result { - let now = chrono::Utc::now(); - let filter = doc! { "expires_at": { "$lt": now } }; - let result = self.db.collection::("revoked_tokens") - .delete_many(filter) - .await?; - Ok(result.deleted_count as usize) - } -} - - -#[async_trait] -impl Signer for MongoStore { - async fn get_table_credentials(&self, location: &str) -> Result { - // Attempt to extract bucket/container from location - let (_scheme, container) = if location.starts_with("s3://") { - ("s3", location[5..].split('/').next().unwrap_or("").to_string()) - } else if location.starts_with("az://") { - ("az", location[5..].split('/').next().unwrap_or("").to_string()) - } else if location.starts_with("gs://") { - ("gs", location[5..].split('/').next().unwrap_or("").to_string()) - } else if location.starts_with("abfs://") { - ("abfs", location[7..].split('/').next().unwrap_or("").split('@').next().unwrap_or("").to_string()) - } else { - ("unknown", String::new()) - }; - - // Find warehouse matching this container - let filter = doc! { - "$or": [ - { "storage_config.s3.bucket": &container }, - { "storage_config.azure.container": &container }, - { "storage_config.gcp.bucket": &container } - ] - }; - - let warehouse = self.warehouses().find_one(filter).await? - .ok_or_else(|| anyhow::anyhow!("No warehouse found for location: {}", location))?; - - match &warehouse.vending_strategy { - Some(VendingStrategy::AwsSts { role_arn: _, external_id: _ }) => { - Err(anyhow::anyhow!("AWS STS vending not implemented yet via VendingStrategy in MongoStore")) - } - Some(VendingStrategy::AwsStatic { access_key_id, secret_access_key }) => { - Ok(Credentials::Aws { - access_key_id: access_key_id.clone(), - secret_access_key: secret_access_key.clone(), - session_token: None, - expiration: None, - }) - } - Some(VendingStrategy::AzureSas { account_name, account_key }) => { - let signer = crate::azure_signer::AzureSigner::new(account_name.clone(), account_key.clone()); - let sas_token = signer.generate_sas_token(location).await?; - Ok(Credentials::Azure { - sas_token, - account_name: account_name.clone(), - expiration: chrono::Utc::now() + chrono::Duration::hours(1), - }) - } - Some(VendingStrategy::GcpDownscoped { service_account_email, private_key }) => { - let signer = crate::gcp_signer::GcpSigner::new(service_account_email.clone(), private_key.clone()); - let access_token = signer.generate_downscoped_token(location).await?; - Ok(Credentials::Gcp { - access_token, - expiration: chrono::Utc::now() + chrono::Duration::hours(1), - }) - } - Some(VendingStrategy::None) => Err(anyhow::anyhow!("Vending disabled")), - None => { - // Backward compatibility logic - let access_key = warehouse.storage_config.get("s3.access-key-id") - .ok_or_else(|| anyhow::anyhow!("Missing s3.access-key-id"))?; - let secret_key = warehouse.storage_config.get("s3.secret-access-key") - .ok_or_else(|| anyhow::anyhow!("Missing s3.secret-access-key"))?; - - if warehouse.use_sts { - // Existing STS Logic restored for backward compatibility - let region = warehouse.storage_config.get("s3.region") - .map(|s| s.as_str()) - .unwrap_or("us-east-1"); - - let endpoint = warehouse.storage_config.get("s3.endpoint") - .map(|s| s.as_str()); - - let creds = aws_credential_types::Credentials::new( - access_key.to_string(), - secret_key.to_string(), - None, - None, - "legacy_provider" - ); - - let config_loader = aws_config::from_env() - .region(aws_config::Region::new(region.to_string())) - .credentials_provider(creds); - - let config = if let Some(ep) = endpoint { - config_loader.endpoint_url(ep).load().await - } else { - config_loader.load().await - }; - - let client = aws_sdk_sts::Client::new(&config); - - let role_arn = warehouse.storage_config.get("s3.role-arn").map(|s| s.as_str()); - - if let Some(arn) = role_arn { - let resp = client.assume_role() - .role_arn(arn) - .role_session_name("pangolin-mongo-legacy") - .send() - .await - .map_err(|e| anyhow::anyhow!("STS AssumeRole failed: {}", e))?; - - let c = resp.credentials.ok_or_else(|| anyhow::anyhow!("No credentials in AssumeRole response"))?; - Ok(Credentials::Aws { - access_key_id: c.access_key_id, - secret_access_key: c.secret_access_key, - session_token: Some(c.session_token), - expiration: chrono::DateTime::from_timestamp(c.expiration.secs(), c.expiration.subsec_nanos()), - }) - } else { - let resp = client.get_session_token() - .send() - .await - .map_err(|e| anyhow::anyhow!("STS GetSessionToken failed: {}", e))?; - - let c = resp.credentials.ok_or_else(|| anyhow::anyhow!("No credentials in GetSessionToken response"))?; - Ok(Credentials::Aws { - access_key_id: c.access_key_id, - secret_access_key: c.secret_access_key, - session_token: Some(c.session_token), - expiration: chrono::DateTime::from_timestamp(c.expiration.secs(), c.expiration.subsec_nanos()), - }) - } - } else { - Ok(Credentials::Aws { - access_key_id: access_key.clone(), - secret_access_key: secret_key.clone(), - session_token: None, - expiration: None, - }) - } - } - } - } - - async fn presign_get(&self, _location: &str) -> Result { - Err(anyhow::anyhow!("MongoStore does not support presigning yet")) - } -} - -fn to_bson_uuid(id: Uuid) -> Bson { - Bson::Binary(Binary { - subtype: BinarySubtype::Generic, - bytes: id.as_bytes().to_vec(), - }) -} - -fn from_bson_uuid(bson: &Bson) -> Result { - match bson { - Bson::Binary(Binary { subtype: BinarySubtype::Generic, bytes }) => { - Ok(Uuid::from_slice(bytes)?) - }, - _ => Err(anyhow::anyhow!("Invalid UUID bson")), - } -} - -impl MongoStore { - pub async fn create_user(&self, user: User) -> Result<()> { - self.users().insert_one(user).await?; - Ok(()) - } - - pub async fn get_user(&self, id: Uuid) -> Result> { - let filter = doc! { "id": to_bson_uuid(id) }; - let user = self.users().find_one(filter).await?; - Ok(user) - } - - pub async fn list_users(&self, tenant_id: Option) -> Result> { - let filter = if let Some(tid) = tenant_id { - doc! { "tenant-id": to_bson_uuid(tid) } - } else { - doc! {} - }; - let cursor = self.users().find(filter).await?; - let users: Vec = cursor.try_collect().await?; - Ok(users) - } - - async fn get_warehouse_for_location(&self, location: &str) -> Result> { - let cursor = self.warehouses().find(doc! {}).await.map_err(|e| anyhow::anyhow!(e))?; - let warehouses: Vec = cursor.try_collect().await.map_err(|e| anyhow::anyhow!(e))?; - - for warehouse in warehouses { - let s3_match = warehouse.storage_config.get("s3.bucket").or_else(|| warehouse.storage_config.get("bucket")).map(|b| location.contains(b)).unwrap_or(false); - let azure_match = warehouse.storage_config.get("azure.container").map(|c| location.contains(c)).unwrap_or(false); - let gcp_match = warehouse.storage_config.get("gcp.bucket").map(|b| location.contains(b)).unwrap_or(false); - - if s3_match || azure_match || gcp_match { - return Ok(Some(warehouse)); - } - } - - Ok(None) - } - - fn get_object_store_cache_key(&self, config: &HashMap, location: &str) -> String { - let endpoint = config.get("s3.endpoint").or_else(|| config.get("endpoint")).or_else(|| config.get("azure.endpoint")).or_else(|| config.get("gcp.endpoint")).map(|s| s.as_str()).unwrap_or(""); - let bucket = config.get("s3.bucket").or_else(|| config.get("bucket")).or_else(|| config.get("azure.container")).or_else(|| config.get("gcp.bucket")).map(|s| s.as_str()).unwrap_or_else(|| { - location.strip_prefix("s3://").or_else(|| location.strip_prefix("az://")).or_else(|| location.strip_prefix("gs://")).and_then(|s| s.split('/').next()).unwrap_or("") - }); - let access_key = config.get("s3.access-key-id").or_else(|| config.get("access_key_id")).or_else(|| config.get("azure.account-name")).or_else(|| config.get("gcp.service-account-key")).map(|s| s.as_str()).unwrap_or(""); - let region = config.get("s3.region").or_else(|| config.get("region")).or_else(|| config.get("azure.region")).or_else(|| config.get("gcp.region")).map(|s| s.as_str()).unwrap_or(""); - crate::ObjectStoreCache::cache_key(endpoint, &bucket, access_key, region) - } - - async fn read_file_uncached(&self, path: &str) -> Result> { - // Try to look up warehouse credentials first - if let Some(warehouse) = self.get_warehouse_for_location(path).await? { - if path.starts_with("s3://") || path.starts_with("az://") || path.starts_with("gs://") { - // Use cached object store - let cache_key = self.get_object_store_cache_key(&warehouse.storage_config, path); - let store = self.object_store_cache.get_or_insert(cache_key, || { - Arc::new(crate::object_store_factory::create_object_store(&warehouse.storage_config, path).unwrap()) - }); - - // Extract key relative to bucket - let key = if let Some(rest) = path.strip_prefix("s3://").or_else(|| path.strip_prefix("az://")).or_else(|| path.strip_prefix("gs://")) { - rest.split_once('/').map(|(_, k)| k).unwrap_or(rest) - } else { - path - }; - - match store.get(&ObjPath::from(key)).await { - Ok(result) => return Ok(result.bytes().await?.to_vec()), - Err(e) => { - tracing::warn!("Failed to read from warehouse-configured store for {}, falling back to global env: {}", path, e); - } - } - } - } - - if let Some(rest) = path.strip_prefix("s3://") { - let (bucket, key) = rest.split_once('/').ok_or_else(|| anyhow::anyhow!("Invalid S3 path"))?; - - let mut builder = AmazonS3Builder::new() - .with_bucket_name(bucket) - .with_allow_http(true); - - if let Ok(endpoint) = std::env::var("S3_ENDPOINT") { - builder = builder.with_endpoint(endpoint); - } - if let Ok(key_id) = std::env::var("AWS_ACCESS_KEY_ID") { - builder = builder.with_access_key_id(key_id); - } - if let Ok(secret) = std::env::var("AWS_SECRET_ACCESS_KEY") { - builder = builder.with_secret_access_key(secret); - } - if let Ok(region) = std::env::var("AWS_REGION") { - builder = builder.with_region(region); - } - - let store = builder.build()?; - let location = ObjPath::from(key); - let result = store.get(&location).await?; - let bytes = result.bytes().await?; - Ok(bytes.to_vec()) - } else { - Err(anyhow::anyhow!("Only s3:// paths are supported in Mongo store")) - } - } -} diff --git a/pangolin/pangolin_store/src/mongo/assets.rs b/pangolin/pangolin_store/src/mongo/assets.rs index 9b0bfc1..aba0ab6 100644 --- a/pangolin/pangolin_store/src/mongo/assets.rs +++ b/pangolin/pangolin_store/src/mongo/assets.rs @@ -33,7 +33,7 @@ impl MongoStore { "namespace": namespace, "id": to_bson_uuid(asset.id), "name": &asset.name, - "kind": format!("{:?}", asset.kind), + "kind": asset.kind.as_stored_str(), "location": &asset.location, "properties": mongodb::bson::to_bson(&asset.properties)? }; @@ -72,11 +72,8 @@ impl MongoStore { if let Some(d) = doc { let kind_str = d.get_str("kind")?; - let kind = match kind_str { - "IcebergTable" => AssetType::IcebergTable, - "View" => AssetType::View, - _ => AssetType::IcebergTable, - }; + // B7: an unknown value used to fall through to `IcebergTable`. + let kind = AssetType::from_stored_str(kind_str).map_err(|e| anyhow::anyhow!(e))?; let properties: HashMap = mongodb::bson::from_bson(d.get("properties").unwrap().clone())?; @@ -128,11 +125,8 @@ impl MongoStore { let mut assets = Vec::new(); for d in docs { let kind_str = d.get_str("kind")?; - let kind = match kind_str { - "IcebergTable" => AssetType::IcebergTable, - "View" => AssetType::View, - _ => AssetType::IcebergTable, - }; + // B7: an unknown value used to fall through to `IcebergTable`. + let kind = AssetType::from_stored_str(kind_str).map_err(|e| anyhow::anyhow!(e))?; let properties: HashMap = mongodb::bson::from_bson(d.get("properties").unwrap().clone())?; @@ -180,11 +174,8 @@ impl MongoStore { .map(|v| v.as_str().unwrap().to_string()) .collect(); let kind_str = d.get_str("kind")?; - let kind = match kind_str { - "IcebergTable" => AssetType::IcebergTable, - "View" => AssetType::View, - _ => AssetType::IcebergTable, - }; + // B7: an unknown value used to fall through to `IcebergTable`. + let kind = AssetType::from_stored_str(kind_str).map_err(|e| anyhow::anyhow!(e))?; let properties: HashMap = mongodb::bson::from_bson(d.get("properties").unwrap().clone())?; diff --git a/pangolin/pangolin_store/src/mongo/audit.rs b/pangolin/pangolin_store/src/mongo/audit.rs index cd7fa0d..1aa33d1 100644 --- a/pangolin/pangolin_store/src/mongo/audit.rs +++ b/pangolin/pangolin_store/src/mongo/audit.rs @@ -6,6 +6,9 @@ use mongodb::bson::{doc, Bson, Document}; use pangolin_core::audit::{AuditLogEntry, AuditLogFilter}; use uuid::Uuid; +/// Default cap on a listing, matching the SQL backends' `LIMIT 100`. +const DEFAULT_AUDIT_LIMIT: usize = 100; + impl MongoStore { pub async fn log_audit_event(&self, entry: AuditLogEntry) -> Result<()> { let mut doc = mongodb::bson::to_document(&entry)?; @@ -31,8 +34,21 @@ impl MongoStore { Ok(()) } - pub async fn get_audit_event(&self, id: Uuid) -> Result> { - let filter = doc! { "id": to_bson_uuid(id) }; + /// Fetch one audit event, scoped to its tenant. + /// + /// B1: the filter was `{ "id": ... }` alone and the caller's `tenant_id` + /// was discarded, so any tenant holding an audit-event UUID could read + /// another tenant's audit record - username, IP, resource names, metadata. + /// Postgres and SQLite both scoped by tenant; only Mongo did not. + pub async fn get_audit_event( + &self, + tenant_id: Uuid, + id: Uuid, + ) -> Result> { + let filter = doc! { + "id": to_bson_uuid(id), + "tenant_id": to_bson_uuid(tenant_id), + }; let doc = self .db .collection::("audit_logs") @@ -60,11 +76,28 @@ impl MongoStore { tenant_id: Uuid, filter: Option, ) -> Result> { + // B23: this applied no sort, no limit and no offset while the SQL + // backends used `ORDER BY timestamp DESC LIMIT 100`. On a busy tenant + // Mongo streamed the entire audit collection into memory and returned + // it in storage order. + let (limit, offset) = filter + .as_ref() + .map(|f| { + ( + f.limit.unwrap_or(DEFAULT_AUDIT_LIMIT), + f.offset.unwrap_or(0), + ) + }) + .unwrap_or((DEFAULT_AUDIT_LIMIT, 0)); + let mongo_filter = self.build_audit_filter(tenant_id, filter)?; let cursor = self .db .collection::("audit_logs") .find(mongo_filter) + .sort(doc! { "timestamp": -1 }) + .skip(offset as u64) + .limit(limit as i64) .await?; let entries: Vec = cursor.try_collect().await?; Ok(entries) @@ -77,15 +110,29 @@ impl MongoStore { ) -> Result { let mut mongo_filter = doc! { "tenant_id": to_bson_uuid(tenant_id) }; if let Some(f) = filter { + // B23: these used `format!("{:?}", ..)` - the Debug spelling, + // `"CreateBranch"` - against documents serde wrote in snake_case + // (`"create_branch"`). The filters could never match, so an + // action- or resource-type-filtered listing always returned zero + // rows and `count_audit_events` always returned 0. Going through + // `bson::to_bson` uses the same serde naming as the write path. if let Some(rt) = f.resource_type { - mongo_filter.insert("resource_type", format!("{:?}", rt)); + mongo_filter.insert("resource_type", mongodb::bson::to_bson(&rt)?); } if let Some(ra) = f.action { - mongo_filter.insert("action", format!("{:?}", ra)); + mongo_filter.insert("action", mongodb::bson::to_bson(&ra)?); } if let Some(uid) = f.user_id { mongo_filter.insert("user_id", to_bson_uuid(uid)); } + // B23: `resource_id` and `result` were accepted by the filter type + // and then silently ignored. + if let Some(rid) = f.resource_id { + mongo_filter.insert("resource_id", to_bson_uuid(rid)); + } + if let Some(result) = f.result { + mongo_filter.insert("result", mongodb::bson::to_bson(&result)?); + } if let Some(from) = f.start_time { mongo_filter.insert("timestamp", doc! { "$gte": Bson::DateTime(from.into()) }); } diff --git a/pangolin/pangolin_store/src/mongo/branches.rs b/pangolin/pangolin_store/src/mongo/branches.rs index 2dd10ba..06633c9 100644 --- a/pangolin/pangolin_store/src/mongo/branches.rs +++ b/pangolin/pangolin_store/src/mongo/branches.rs @@ -1,4 +1,4 @@ -use super::main::to_bson_uuid; +use super::main::{read_optional_uuid, to_bson_uuid}; use super::MongoStore; use anyhow::Result; use futures::stream::TryStreamExt; @@ -17,7 +17,10 @@ impl MongoStore { "tenant_id": to_bson_uuid(tenant_id), "catalog_name": catalog_name, "name": &branch.name, - "head_commit_id": branch.head_commit_id, + "head_commit_id": branch + .head_commit_id + .map(to_bson_uuid) + .unwrap_or(mongodb::bson::Bson::Null), "branch_type": format!("{:?}", branch.branch_type), "assets": &branch.assets }; @@ -55,7 +58,7 @@ impl MongoStore { Ok(Some(Branch { name: d.get_str("name")?.to_string(), - head_commit_id: mongodb::bson::from_bson(d.get("head_commit_id").unwrap().clone())?, + head_commit_id: read_optional_uuid(&d, "head_commit_id")?, branch_type, assets: mongodb::bson::from_bson(d.get("assets").unwrap().clone())?, })) @@ -100,7 +103,7 @@ impl MongoStore { branches.push(Branch { name: d.get_str("name")?.to_string(), - head_commit_id: mongodb::bson::from_bson(d.get("head_commit_id").unwrap().clone())?, + head_commit_id: read_optional_uuid(&d, "head_commit_id")?, branch_type, assets: mongodb::bson::from_bson(d.get("assets").unwrap().clone())?, }); @@ -171,7 +174,10 @@ impl MongoStore { let update = doc! { "$set": { - "head_commit_id": source.head_commit_id + "head_commit_id": source + .head_commit_id + .map(to_bson_uuid) + .unwrap_or(mongodb::bson::Bson::Null) } }; diff --git a/pangolin/pangolin_store/src/mongo/business_metadata.rs b/pangolin/pangolin_store/src/mongo/business_metadata.rs index 30ba040..25e4280 100644 --- a/pangolin/pangolin_store/src/mongo/business_metadata.rs +++ b/pangolin/pangolin_store/src/mongo/business_metadata.rs @@ -1,4 +1,4 @@ -use super::main::{from_bson_uuid, to_bson_uuid}; +use super::main::{from_bson_uuid, read_optional_uuid, to_bson_uuid, with_binary_uuids}; use super::MongoStore; use anyhow::Result; use futures::stream::TryStreamExt; @@ -9,10 +9,25 @@ use std::collections::HashMap; use uuid::Uuid; impl MongoStore { + /// Insert or replace an asset's business metadata. + /// + /// Only `asset-id` used to be rewritten as Binary. The other three UUID + /// fields kept whatever `to_document` produced - a string - so the record + /// could be written and located but never *deserialized*: + /// `get_business_metadata` failed with `invalid type: string "...", + /// expected bytes` on the first field it reached. Writing metadata + /// therefore made an asset's metadata permanently unreadable. pub async fn upsert_business_metadata(&self, metadata: BusinessMetadata) -> Result<()> { let filter = doc! { "asset-id": to_bson_uuid(metadata.asset_id) }; - let mut doc = mongodb::bson::to_document(&metadata)?; - doc.insert("asset-id", to_bson_uuid(metadata.asset_id)); + let doc = with_binary_uuids( + mongodb::bson::to_document(&metadata)?, + &[ + ("id", metadata.id), + ("asset-id", metadata.asset_id), + ("created-by", metadata.created_by), + ("updated-by", metadata.updated_by), + ], + ); let options = mongodb::options::ReplaceOptions::builder() .upsert(true) @@ -211,9 +226,7 @@ impl MongoStore { results.push(( Branch { name: d.get_str("name")?.to_string(), - head_commit_id: mongodb::bson::from_bson( - d.get("head_commit_id").unwrap().clone(), - )?, + head_commit_id: read_optional_uuid(&d, "head_commit_id")?, branch_type, assets: mongodb::bson::from_bson(d.get("assets").unwrap().clone())?, }, diff --git a/pangolin/pangolin_store/src/mongo/catalogs.rs b/pangolin/pangolin_store/src/mongo/catalogs.rs index b294d4a..4886b5b 100644 --- a/pangolin/pangolin_store/src/mongo/catalogs.rs +++ b/pangolin/pangolin_store/src/mongo/catalogs.rs @@ -6,6 +6,40 @@ use mongodb::bson::{doc, Document}; use pangolin_core::model::{Catalog, CatalogUpdate}; use uuid::Uuid; +/// Why a transactional catalog delete stopped. +enum CatalogDeleteError { + /// The catalog does not exist. + NotFound, + /// The deployment has no transaction support - a standalone `mongod`. + /// + /// MongoDB reports this as `IllegalOperation` (code 20) with "Transaction + /// numbers are only allowed on a replica set member or mongos", and only + /// once the first operation inside the transaction reaches the server. + TransactionsUnsupported(mongodb::error::Error), + Other(anyhow::Error), +} + +impl CatalogDeleteError { + fn from_mongo(e: mongodb::error::Error) -> Self { + // MongoDB raises `IllegalOperation` for "Transaction numbers are only + // allowed on a replica set member or mongos". The driver replaces that + // server text with its own "does not support retryable writes" wording, + // so matching on the message is unreliable - the code is the stable + // signal. This classifier only ever runs on failures from inside a + // transaction attempt, where an `IllegalOperation` means the deployment + // cannot do transactions; the cost of a false positive is a non-atomic + // delete rather than a wrong result. + const ILLEGAL_OPERATION: i32 = 20; + + if let mongodb::error::ErrorKind::Command(command_error) = &*e.kind { + if command_error.code == ILLEGAL_OPERATION { + return Self::TransactionsUnsupported(e); + } + } + Self::Other(anyhow::anyhow!(e)) + } +} + impl MongoStore { pub async fn create_catalog(&self, tenant_id: Uuid, catalog: Catalog) -> Result<()> { let mut doc = doc! { @@ -128,43 +162,94 @@ impl MongoStore { }; if let Some(session) = session.as_mut() { + // `start_transaction` is a *local* call in the Rust driver: it + // allocates a transaction number and returns without contacting the + // server, so it cannot fail for "this deployment has no transaction + // support". The fallback below was therefore unreachable - the + // topology error surfaced on the first operation *inside* the + // transaction and propagated through `?`, failing the delete + // outright on any standalone `mongod`. Found by running the parity + // suite against a live standalone MongoDB. + // + // The transactional attempt now runs in full and a topology error + // anywhere within it degrades to the sequential path, which is what + // the original comment promised. if let Err(e) = session.start_transaction().await { tracing::warn!( error = %e, - "this MongoDB deployment does not support transactions (a replica set is \ - required); deleting a catalog will not be atomic" + "could not begin a MongoDB transaction; deleting a catalog will not be atomic" ); return self .delete_catalog_unsafe(filter, child_filter, &name) .await; } - for collection in ["tags", "branches", "assets", "namespaces"] { - self.db - .collection::(collection) - .delete_many(child_filter.clone()) - .session(&mut *session) - .await?; + match self + .delete_catalog_in_transaction(session, &filter, &child_filter) + .await + { + Ok(()) => return Ok(()), + Err(CatalogDeleteError::NotFound) => { + let _ = session.abort_transaction().await; + return Err(anyhow::anyhow!("Catalog '{}' not found", name)); + } + Err(CatalogDeleteError::TransactionsUnsupported(e)) => { + tracing::warn!( + error = %e, + "this MongoDB deployment does not support transactions (a replica set \ + is required); deleting a catalog will not be atomic" + ); + let _ = session.abort_transaction().await; + // Falls through to the sequential path below. + } + Err(CatalogDeleteError::Other(e)) => { + let _ = session.abort_transaction().await; + return Err(e); + } } + } + + self.delete_catalog_unsafe(filter, child_filter, &name) + .await + } - let result = self - .db - .collection::("catalogs") - .delete_one(filter) + /// The transactional half of [`Self::delete_catalog`]. + /// + /// Split out so a topology error can be told apart from a genuine failure + /// and from "no such catalog" - the caller handles each differently. + async fn delete_catalog_in_transaction( + &self, + session: &mut mongodb::ClientSession, + filter: &Document, + child_filter: &Document, + ) -> std::result::Result<(), CatalogDeleteError> { + for collection in ["tags", "branches", "assets", "namespaces"] { + self.db + .collection::(collection) + .delete_many(child_filter.clone()) .session(&mut *session) - .await?; + .await + .map_err(CatalogDeleteError::from_mongo)?; + } - if result.deleted_count == 0 { - let _ = session.abort_transaction().await; - return Err(anyhow::anyhow!("Catalog '{}' not found", name)); - } + let result = self + .db + .collection::("catalogs") + .delete_one(filter.clone()) + .session(&mut *session) + .await + .map_err(CatalogDeleteError::from_mongo)?; - session.commit_transaction().await?; - return Ok(()); + if result.deleted_count == 0 { + return Err(CatalogDeleteError::NotFound); } - self.delete_catalog_unsafe(filter, child_filter, &name) + session + .commit_transaction() .await + .map_err(CatalogDeleteError::from_mongo)?; + + Ok(()) } /// Non-atomic fallback for deployments without transaction support. @@ -174,6 +259,29 @@ impl MongoStore { child_filter: Document, name: &str, ) -> Result<()> { + // Existence is checked *first*. This path used to delete every matching + // tag, branch, asset and namespace and only then discover the catalog + // did not exist - returning "not found" to a caller who had every reason + // to believe nothing had happened. That is B21, which was fixed for + // SQLite during the roadmap work; the same shape survived here because + // this branch only runs on a deployment without transactions, and + // nothing had ever run it. + // + // Without a transaction the cascade below is still not atomic - a + // failure partway through leaves a partial delete. Checking first at + // least means a *no-op* call destroys nothing, which is the case an + // operator is most likely to hit by typo. + let exists = self + .db + .collection::("catalogs") + .find_one(filter.clone()) + .await? + .is_some(); + + if !exists { + return Err(anyhow::anyhow!("Catalog '{}' not found", name)); + } + for collection in ["tags", "branches", "assets", "namespaces"] { self.db .collection::(collection) @@ -181,15 +289,11 @@ impl MongoStore { .await?; } - let result = self - .db + self.db .collection::("catalogs") .delete_one(filter) .await?; - if result.deleted_count == 0 { - return Err(anyhow::anyhow!("Catalog '{}' not found", name)); - } Ok(()) } } diff --git a/pangolin/pangolin_store/src/mongo/main.rs b/pangolin/pangolin_store/src/mongo/main.rs index a9a2011..1615491 100644 --- a/pangolin/pangolin_store/src/mongo/main.rs +++ b/pangolin/pangolin_store/src/mongo/main.rs @@ -197,12 +197,32 @@ impl MongoStore { .get_asset(tenant_id, catalog_name, branch, namespace, table) .await? { - Ok(asset.properties.get("metadata_location").cloned()) + // Found by running the parity suite against a live MongoDB: this + // read only `properties["metadata_location"]` and had no fallback to + // the asset's own `location`, which is what memory, SQLite and + // Postgres all fall back to. A table created with a location but no + // explicit metadata-location property therefore reported *no* + // metadata location on Mongo alone - so `load_table` could not find + // its metadata and every commit's compare-and-swap was working from + // a different notion of "current" than the read path. + Ok(current_metadata_location(&asset)) } else { Ok(None) } } + /// Publish a new metadata location, but only if the current one still + /// matches `expected_location`. + /// + /// B5: `expected_location` was ignored (`_expected_location`) and the update + /// was an unconditional `$set`. Memory, Postgres and SQLite all enforce the + /// compare-and-swap; on Mongo two concurrent Iceberg commits both + /// "succeeded" and one snapshot was silently lost - the exact failure class + /// the 0.6.0 work fixed at the API layer, still wide open one layer down. + /// + /// Folding the expectation into the *filter* keeps this a single-document + /// atomic update, so it works on a standalone `mongod` with no multi-document + /// transaction required. pub async fn update_metadata_location( &self, tenant_id: Uuid, @@ -210,10 +230,10 @@ impl MongoStore { branch: Option, namespace: Vec, table: String, - _expected_location: Option, + expected_location: Option, new_location: String, ) -> Result<()> { - let filter = doc! { + let mut filter = doc! { "tenant_id": to_bson_uuid(tenant_id), "catalog_name": catalog_name, "branch": branch.unwrap_or_else(|| "main".to_string()), @@ -221,20 +241,106 @@ impl MongoStore { "name": table }; + // The expectation has to be expressed against the *same* notion of + // "current location" that `get_metadata_location` returns - property + // first, then the asset's own `location`. Keeping it inside the filter + // rather than reading-then-comparing preserves the single-document + // atomicity that makes this a real CAS on a standalone mongod. + match &expected_location { + Some(expected) => { + filter.insert( + "$or", + vec![ + doc! { "properties.metadata_location": expected.clone() }, + doc! { "$and": vec![ + doc! { "properties.metadata_location": { "$exists": false } }, + doc! { "location": expected.clone() }, + ]}, + ], + ); + } + // `None` means "there must not be one yet" - the create-path CAS. + None => { + filter.insert( + "$and", + vec![ + doc! { "properties.metadata_location": { "$exists": false } }, + doc! { "$or": vec![ + doc! { "location": { "$exists": false } }, + doc! { "location": "" }, + ]}, + ], + ); + } + } + let update = doc! { "$set": { - "properties.metadata_location": new_location + "properties.metadata_location": &new_location, + "location": &new_location, } }; - self.db + let result = self + .db .collection::("assets") .update_one(filter, update) .await?; + + if result.matched_count == 0 { + return Err(anyhow::anyhow!( + "CAS failure: metadata location did not match {:?}", + expected_location + )); + } Ok(()) } } +/// The metadata location a reader would see for `asset`. +/// +/// Property first, then the asset's own `location` when it is non-empty - the +/// same resolution memory, SQLite and Postgres use. Defined once so the read +/// path and the compare-and-swap cannot drift apart again. +fn current_metadata_location(asset: &pangolin_core::model::Asset) -> Option { + asset + .properties + .get("metadata_location") + .cloned() + .or_else(|| { + if asset.location.is_empty() { + None + } else { + Some(asset.location.clone()) + } + }) +} + +/// Rewrite the named keys of a serde-produced document as BSON Binary UUIDs. +/// +/// `bson::to_document` writes a `Uuid` as a *string*, but the driver's +/// deserializer expects Binary - so a document written through serde alone +/// cannot be read back into a struct with `Uuid` fields ("invalid type: string, +/// expected bytes"), and a filter built with [`to_bson_uuid`] never matches it. +/// +/// That asymmetry is the single cause of the Mongo RBAC failures: role +/// assignments were written by serde and queried as Binary, so `get_user_roles` +/// always returned empty and every role-derived permission silently vanished. +/// The audit-log and token-revocation paths (B1, B2) were the same bug in two +/// other collections. +/// +/// The keys given are the *serialized* names - kebab-case for these types - so +/// the document stays deserializable into its struct. +pub(crate) fn with_binary_uuids( + mut doc: mongodb::bson::Document, + fields: &[(&str, Uuid)], +) -> mongodb::bson::Document { + for (key, value) in fields { + doc.insert(*key, to_bson_uuid(*value)); + } + doc +} + pub(crate) fn to_bson_uuid(id: Uuid) -> Bson { Bson::Binary(Binary { subtype: BinarySubtype::Generic, @@ -242,12 +348,44 @@ pub(crate) fn to_bson_uuid(id: Uuid) -> Bson { }) } +/// Decode a UUID that may have been written by any of the three routes. +/// +/// There are three, and they disagree: +/// +/// 1. [`to_bson_uuid`] - `Binary` with the *generic* subtype; +/// 2. `doc! { "k": some_uuid }` - `Binary` with the *UUID* subtype, via bson's +/// `From for Bson`; +/// 3. `bson::to_document` - a plain `String`. +/// +/// Writes should use `to_bson_uuid` so new data is uniform, but reads have to +/// accept all three: documents written by the other two are already in +/// deployed databases. Being strict here is what made a branch with a head +/// commit unreadable - `create_branch` used route 2 and the reader accepted +/// only route 1. pub(crate) fn from_bson_uuid(bson: &Bson) -> Result { match bson { Bson::Binary(Binary { - subtype: BinarySubtype::Generic, + subtype: BinarySubtype::Generic | BinarySubtype::Uuid, bytes, }) => Ok(Uuid::from_slice(bytes)?), + Bson::String(s) => Ok(Uuid::parse_str(s)?), _ => Err(anyhow::anyhow!("Invalid UUID bson")), } } + +/// Decode an optional UUID field. +/// +/// A missing key and an explicit `null` both mean "absent"; anything else has +/// to decode, because silently returning `None` for a value that is present +/// but unreadable would turn a corrupt record into a plausible-looking one. +/// +/// `bson::from_bson::>` cannot be used for this: handed a +/// `Bson::Binary` it reports `invalid type: map, expected a UUID string`, +/// because the deserializer presents binary data as the extended-JSON map +/// `{"$binary": ...}` while `Uuid`'s `Deserialize` wants a string. +pub(crate) fn read_optional_uuid(doc: &mongodb::bson::Document, key: &str) -> Result> { + match doc.get(key) { + None | Some(Bson::Null) => Ok(None), + Some(value) => from_bson_uuid(value).map(Some), + } +} diff --git a/pangolin/pangolin_store/src/mongo/mod.rs b/pangolin/pangolin_store/src/mongo/mod.rs index 891b7d2..7268555 100644 --- a/pangolin/pangolin_store/src/mongo/mod.rs +++ b/pangolin/pangolin_store/src/mongo/mod.rs @@ -154,6 +154,17 @@ impl CatalogStore for MongoStore { .await } + async fn replace_namespace_properties( + &self, + tenant_id: Uuid, + catalog_name: &str, + namespace: Vec, + properties: HashMap, + ) -> Result<()> { + self.replace_namespace_properties(tenant_id, catalog_name, namespace, properties) + .await + } + // Asset Operations async fn create_asset( &self, @@ -460,10 +471,12 @@ impl CatalogStore for MongoStore { } async fn get_audit_event( &self, - _tenant_id: Uuid, + tenant_id: Uuid, id: Uuid, ) -> Result> { - self.get_audit_event(id).await + // B1: `tenant_id` used to be discarded here (`_tenant_id`), which is + // where the cross-tenant audit read came from. + self.get_audit_event(tenant_id, id).await } async fn count_audit_events( &self, @@ -607,6 +620,14 @@ impl CatalogStore for MongoStore { async fn write_file(&self, path: &str, data: Vec) -> Result<()> { self.write_file(path, data).await } + async fn delete_file(&self, path: &str) -> Result<()> { + self.metadata_cache.invalidate(path).await; + let storage_config = self + .get_warehouse_for_location(path) + .await? + .map(|w| w.storage_config); + crate::file_delete::delete_location(storage_config.as_ref(), path).await + } // Access Requests async fn create_access_request(&self, request: AccessRequest) -> Result<()> { diff --git a/pangolin/pangolin_store/src/mongo/namespaces.rs b/pangolin/pangolin_store/src/mongo/namespaces.rs index 6c7d7c7..9cca7aa 100644 --- a/pangolin/pangolin_store/src/mongo/namespaces.rs +++ b/pangolin/pangolin_store/src/mongo/namespaces.rs @@ -120,6 +120,34 @@ impl MongoStore { Ok(()) } + /// Replace a namespace's properties wholesale; see the SQLite twin for why + /// a merge-only method could not implement Iceberg property removals (B16h). + pub async fn replace_namespace_properties( + &self, + tenant_id: Uuid, + catalog_name: &str, + namespace: Vec, + properties: HashMap, + ) -> Result<()> { + let filter = doc! { + "tenant_id": to_bson_uuid(tenant_id), + "catalog_name": catalog_name, + "name": namespace + }; + + let props = bson::to_bson(&properties)?; + let update = doc! { "$set": { "properties": props } }; + let result = self + .db + .collection::("namespaces") + .update_one(filter, update) + .await?; + if result.matched_count == 0 { + return Err(anyhow::anyhow!("Namespace not found")); + } + Ok(()) + } + pub async fn count_namespaces(&self, tenant_id: Uuid) -> Result { let filter = doc! { "tenant_id": to_bson_uuid(tenant_id) }; let count = self.namespaces().count_documents(filter).await?; diff --git a/pangolin/pangolin_store/src/mongo/permissions.rs b/pangolin/pangolin_store/src/mongo/permissions.rs index 9411cbe..dc6cdd5 100644 --- a/pangolin/pangolin_store/src/mongo/permissions.rs +++ b/pangolin/pangolin_store/src/mongo/permissions.rs @@ -1,4 +1,4 @@ -use super::main::to_bson_uuid; +use super::main::{to_bson_uuid, with_binary_uuids}; use super::MongoStore; use anyhow::Result; use futures::stream::TryStreamExt; @@ -9,11 +9,21 @@ use uuid::Uuid; impl MongoStore { pub async fn grant_permission(&self, permission: Permission) -> Result<()> { - let mut doc = mongodb::bson::to_document(&permission)?; - doc.insert("id", to_bson_uuid(permission.id)); - doc.insert("user_id", to_bson_uuid(permission.user_id)); - doc.insert("tenant_id", to_bson_uuid(permission.tenant_id)); - doc.insert("granted_by", to_bson_uuid(permission.granted_by)); + // The overrides used *snake_case* names while `Permission` serializes + // as kebab-case, so this added a second set of fields rather than + // replacing the first: the document carried both `user-id` (a string) + // and `user_id` (Binary). Filters matched the Binary copy, but + // deserializing the record back into `Permission` read the string and + // failed with "invalid type: string, expected bytes". + let doc = with_binary_uuids( + mongodb::bson::to_document(&permission)?, + &[ + ("id", permission.id), + ("user-id", permission.user_id), + ("tenant-id", permission.tenant_id), + ("granted-by", permission.granted_by), + ], + ); self.db .collection::("permissions") @@ -37,7 +47,7 @@ impl MongoStore { pagination: Option, ) -> Result> { // 1. Fetch direct permissions - let filter = doc! { "user_id": to_bson_uuid(user_id) }; + let filter = doc! { "user-id": to_bson_uuid(user_id) }; let cursor = self .db .collection::("permissions") @@ -98,7 +108,7 @@ impl MongoStore { } // 2. Get permissions for those users - let perm_filter = doc! { "user_id": { "$in": user_ids } }; + let perm_filter = doc! { "user-id": { "$in": user_ids } }; let collection = self.db.collection::("permissions"); let mut find = collection.find(perm_filter); diff --git a/pangolin/pangolin_store/src/mongo/roles.rs b/pangolin/pangolin_store/src/mongo/roles.rs index 9ca1062..d24c40b 100644 --- a/pangolin/pangolin_store/src/mongo/roles.rs +++ b/pangolin/pangolin_store/src/mongo/roles.rs @@ -1,4 +1,4 @@ -use super::main::to_bson_uuid; +use super::main::{to_bson_uuid, with_binary_uuids}; use super::MongoStore; use anyhow::Result; use futures::stream::TryStreamExt; @@ -8,7 +8,22 @@ use uuid::Uuid; impl MongoStore { pub async fn create_role(&self, role: Role) -> Result<()> { - self.roles().insert_one(role).await?; + // Written through an explicit document so the UUID fields are Binary. + // `insert_one(role)` went through serde, which writes them as strings - + // and then `get_role`'s Binary filter could never match, so a role could + // be created and never found again. + let doc = with_binary_uuids( + mongodb::bson::to_document(&role)?, + &[ + ("id", role.id), + ("tenant-id", role.tenant_id), + ("created-by", role.created_by), + ], + ); + self.db + .collection::("roles") + .insert_one(doc) + .await?; Ok(()) } @@ -61,13 +76,38 @@ impl MongoStore { pub async fn update_role(&self, role: Role) -> Result<()> { let filter = doc! { "id": to_bson_uuid(role.id) }; - self.roles().replace_one(filter, role).await?; + // Same reason as `create_role`: a serde replacement would put the string + // form back and make the role unfindable again. + let doc = with_binary_uuids( + mongodb::bson::to_document(&role)?, + &[ + ("id", role.id), + ("tenant-id", role.tenant_id), + ("created-by", role.created_by), + ], + ); + self.db + .collection::("roles") + .replace_one(filter, doc) + .await?; Ok(()) } // Role Assignment pub async fn assign_role(&self, assignment: UserRole) -> Result<()> { - let doc = mongodb::bson::to_document(&assignment)?; + // The assignment's UUIDs are written as Binary under their serialized + // (kebab-case) names, matching both `get_user_roles`' filter and the + // deserializer. Previously serde wrote strings, so `get_user_roles` + // returned nothing and every role-derived permission was lost - a user + // with an admin role was authorized as if they had none. + let doc = with_binary_uuids( + mongodb::bson::to_document(&assignment)?, + &[ + ("user-id", assignment.user_id), + ("role-id", assignment.role_id), + ("assigned-by", assignment.assigned_by), + ], + ); self.db .collection::("user_roles") .insert_one(doc) @@ -76,9 +116,10 @@ impl MongoStore { } pub async fn remove_role(&self, user_id: Uuid, role_id: Uuid) -> Result<()> { + // Kebab-case: these are the serialized field names. let filter = doc! { - "user_id": to_bson_uuid(user_id), - "role_id": to_bson_uuid(role_id) + "user-id": to_bson_uuid(user_id), + "role-id": to_bson_uuid(role_id) }; self.db .collection::("user_roles") @@ -88,7 +129,7 @@ impl MongoStore { } pub async fn get_user_roles(&self, user_id: Uuid) -> Result> { - let filter = doc! { "user_id": to_bson_uuid(user_id) }; + let filter = doc! { "user-id": to_bson_uuid(user_id) }; let cursor = self .db .collection::("user_roles") diff --git a/pangolin/pangolin_store/src/mongo/service_users.rs b/pangolin/pangolin_store/src/mongo/service_users.rs index 6709172..868cf4e 100644 --- a/pangolin/pangolin_store/src/mongo/service_users.rs +++ b/pangolin/pangolin_store/src/mongo/service_users.rs @@ -1,4 +1,34 @@ -use super::main::to_bson_uuid; +//! Service users (API-key principals) for the MongoDB backend. +//! +//! Every method here was broken, in three overlapping ways, and none of it was +//! visible from reading a single function: +//! +//! * **The lookup key did not exist.** `create_service_user` inserted the serde +//! document as-is, so Mongo generated an `ObjectId` for `_id`. The four +//! by-id methods then filtered on `{"_id": id.to_string()}`, which matched +//! nothing, ever. Fetching, updating, deleting and touching a service user +//! were all no-ops - `update_service_user` reported "Service user not found" +//! for a user that had just been created. +//! * **The field names were wrong.** `ServiceUser` is `rename_all = +//! "kebab-case"`, so the stored fields are `tenant-id` and `api-key-hash`. +//! The filters used `tenant_id` and `api_key_hash`, so listing returned +//! nothing and - the one that matters - **API-key authentication could never +//! resolve a service user**. It fails closed, so this was a total outage of +//! service-user auth on Mongo rather than a bypass. +//! * **The UUIDs were the wrong type.** The usual asymmetry: `to_document` +//! writes a `Uuid` as a string, while the deserializer and `to_bson_uuid` +//! filters want BSON Binary. +//! +//! Found by the entity round-trip suite, which asserts that every UUID-bearing +//! collection can be written and read back. +//! +//! Note that the `chrono` fields go the *other* way: bson's serializer reports +//! itself human-readable, so `to_document` writes them as RFC3339 strings and +//! that is what `from_document` reads back. `update_service_user_last_used` +//! must therefore write a string too - a `Bson::DateTime` there would be +//! written happily and then break every subsequent read. + +use super::main::{to_bson_uuid, with_binary_uuids}; use super::MongoStore; use anyhow::Result; use futures::stream::TryStreamExt; @@ -8,13 +38,20 @@ use uuid::Uuid; impl MongoStore { pub async fn create_service_user(&self, service_user: ServiceUser) -> Result<()> { - let doc = mongodb::bson::to_document(&service_user)?; + let doc = with_binary_uuids( + mongodb::bson::to_document(&service_user)?, + &[ + ("id", service_user.id), + ("tenant-id", service_user.tenant_id), + ("created-by", service_user.created_by), + ], + ); self.service_users().insert_one(doc).await?; Ok(()) } pub async fn get_service_user(&self, id: Uuid) -> Result> { - let filter = doc! { "_id": id.to_string() }; + let filter = doc! { "id": to_bson_uuid(id) }; if let Some(doc) = self.service_users().find_one(filter).await? { Ok(Some(mongodb::bson::from_document(doc)?)) } else { @@ -22,11 +59,15 @@ impl MongoStore { } } + /// Resolve an API key to its principal. + /// + /// This is the authentication path. The hash is stored verbatim, so an + /// exact match is right; only the field name was wrong. pub async fn get_service_user_by_api_key_hash( &self, api_key_hash: &str, ) -> Result> { - let filter = doc! { "api_key_hash": api_key_hash }; + let filter = doc! { "api-key-hash": api_key_hash }; if let Some(doc) = self.service_users().find_one(filter).await? { Ok(Some(mongodb::bson::from_document(doc)?)) } else { @@ -39,7 +80,7 @@ impl MongoStore { tenant_id: Uuid, pagination: Option, ) -> Result> { - let filter = doc! { "tenant_id": to_bson_uuid(tenant_id) }; + let filter = doc! { "tenant-id": to_bson_uuid(tenant_id) }; let collection = self.service_users(); let mut find = collection.find(filter); @@ -69,7 +110,7 @@ impl MongoStore { role: Option, active: Option, ) -> Result { - let filter = doc! { "_id": id.to_string() }; + let filter = doc! { "id": to_bson_uuid(id) }; let mut update_doc = doc! {}; if let Some(n) = name { @@ -79,7 +120,10 @@ impl MongoStore { update_doc.insert("description", d); } if let Some(r) = role { - update_doc.insert("role", format!("{:?}", r)); + // `format!("{:?}", r)` wrote the Rust variant name (`TenantAdmin`). + // `UserRole` is kebab-case, so the record then failed to + // deserialize: changing a service user's role made it unreadable. + update_doc.insert("role", mongodb::bson::to_bson(&r)?); } if let Some(a) = active { update_doc.insert("active", a); @@ -96,14 +140,19 @@ impl MongoStore { } pub async fn delete_service_user(&self, id: Uuid) -> Result<()> { - let filter = doc! { "_id": id.to_string() }; + let filter = doc! { "id": to_bson_uuid(id) }; self.service_users().delete_one(filter).await?; Ok(()) } + /// Stamp the last time this key was used. + /// + /// The field is `last-used`; `last_used_at` was a field no reader ever + /// looked at, so the timestamp shown for every service user stayed at its + /// creation value forever. See the module note on why this is a string. pub async fn update_service_user_last_used(&self, id: Uuid) -> Result<()> { - let filter = doc! { "_id": id.to_string() }; - let update = doc! { "$set": { "last_used_at": chrono::Utc::now() } }; + let filter = doc! { "id": to_bson_uuid(id) }; + let update = doc! { "$set": { "last-used": chrono::Utc::now().to_rfc3339() } }; self.service_users().update_one(filter, update).await?; Ok(()) } diff --git a/pangolin/pangolin_store/src/mongo/tokens.rs b/pangolin/pangolin_store/src/mongo/tokens.rs index 2b92da7..fa5aaf1 100644 --- a/pangolin/pangolin_store/src/mongo/tokens.rs +++ b/pangolin/pangolin_store/src/mongo/tokens.rs @@ -2,10 +2,36 @@ use super::main::{from_bson_uuid, to_bson_uuid}; use super::MongoStore; use anyhow::Result; use futures::stream::TryStreamExt; -use mongodb::bson::{doc, Document}; +use mongodb::bson::{doc, Bson, Document}; use pangolin_core::token::TokenInfo; use uuid::Uuid; +/// Read a timestamp that may have been written either way. +/// +/// `store_token` builds its document with `doc!`, and `doc!` converts a +/// `chrono::DateTime` into a `Bson::DateTime`. Feeding that back through +/// `bson::from_bson::>` does not work: the deserializer presents +/// a `Bson::DateTime` as the map `{"$date": ...}`, while chrono's `Deserialize` +/// only accepts an RFC3339 string. So `list_active_tokens` failed outright with +/// `invalid type: map, expected an RFC 3339 formatted date and time string` - +/// **listing active tokens was broken for any token that existed**, and the +/// `created_at` arm swallowed the same error and substituted `now()`, so had +/// the first one not aborted the call, every token would have reported the +/// listing time as its creation time. +/// +/// Both encodings are accepted because documents written before this fix are +/// still in the collection, and a token record is not worth a migration - it +/// expires on its own. +fn read_datetime(doc: &Document, key: &str) -> Option> { + match doc.get(key)? { + Bson::DateTime(dt) => Some(dt.to_chrono()), + Bson::String(s) => chrono::DateTime::parse_from_rfc3339(s) + .ok() + .map(|dt| dt.with_timezone(&chrono::Utc)), + _ => None, + } +} + impl MongoStore { pub async fn list_active_tokens( &self, @@ -49,13 +75,9 @@ impl MongoStore { )?, username: d.get_str("username").unwrap_or("unknown").to_string(), token: d.get_str("token").ok().map(|s| s.to_string()), - expires_at: mongodb::bson::from_bson(d.get("expires_at").unwrap().clone())?, - created_at: mongodb::bson::from_bson( - d.get("created_at") - .unwrap_or(&mongodb::bson::Bson::Null) - .clone(), - ) - .unwrap_or_else(|_| chrono::Utc::now()), + expires_at: read_datetime(&d, "expires_at") + .ok_or(anyhow::anyhow!("Missing or unreadable expires_at"))?, + created_at: read_datetime(&d, "created_at").unwrap_or_else(chrono::Utc::now), is_valid: true, }); } @@ -76,16 +98,37 @@ impl MongoStore { Ok(()) } + /// Record a revocation. + /// + /// B2: this used to `insert_one(revoked)` through serde, which wrote the + /// token id as a *string* under the field name `id` (the struct's field), + /// while [`Self::is_token_revoked`] queried `token_id` as a BSON Binary + /// UUID. Neither the field name nor the type matched, so the lookup could + /// never find a revocation and the check returned `false` for every token: + /// on Mongo, revocation - including logout - was a silent no-op and revoked + /// JWTs stayed valid until they expired naturally. + /// + /// Both sides now go through an explicit `doc!` using the same encoding as + /// [`Self::store_token`]. pub async fn revoke_token( &self, token_id: Uuid, expires_at: chrono::DateTime, reason: Option, ) -> Result<()> { - let revoked = pangolin_core::token::RevokedToken::new(token_id, expires_at, reason); + let doc = doc! { + "token_id": to_bson_uuid(token_id), + "expires_at": Bson::DateTime(expires_at.into()), + "reason": reason.map(Bson::String).unwrap_or(Bson::Null), + }; + // Upsert so revoking twice is idempotent rather than accumulating rows. self.db - .collection("revoked_tokens") - .insert_one(revoked) + .collection::("revoked_tokens") + .update_one( + doc! { "token_id": to_bson_uuid(token_id) }, + doc! { "$set": doc }, + ) + .upsert(true) .await?; Ok(()) } @@ -94,18 +137,24 @@ impl MongoStore { let filter = doc! { "token_id": to_bson_uuid(token_id) }; let result = self .db - .collection::("revoked_tokens") + .collection::("revoked_tokens") .find_one(filter) .await?; Ok(result.is_some()) } + /// Drop revocation records whose tokens have expired anyway. + /// + /// B2 (second half): the comparison was `$lt` against a BSON DateTime while + /// serde had written `expires_at` as an RFC3339 *string*, so this deleted + /// nothing and the collection grew without bound. With `revoke_token` + /// writing a real `Bson::DateTime`, the comparison is now type-consistent. pub async fn cleanup_expired_tokens(&self) -> Result { let now = chrono::Utc::now(); - let filter = doc! { "expires_at": { "$lt": now } }; + let filter = doc! { "expires_at": { "$lt": Bson::DateTime(now.into()) } }; let result = self .db - .collection::("revoked_tokens") + .collection::("revoked_tokens") .delete_many(filter) .await?; Ok(result.deleted_count as usize) diff --git a/pangolin/pangolin_store/src/postgres/assets.rs b/pangolin/pangolin_store/src/postgres/assets.rs index 5e29a96..5ed4120 100644 --- a/pangolin/pangolin_store/src/postgres/assets.rs +++ b/pangolin/pangolin_store/src/postgres/assets.rs @@ -22,7 +22,7 @@ impl PostgresStore { .bind(&branch_name) .bind(&namespace) .bind(&asset.name) - .bind(format!("{:?}", asset.kind)) + .bind(asset.kind.as_stored_str()) .bind(asset.properties.get("metadata_location").unwrap_or(&asset.location)) .bind(serde_json::to_value(&asset.properties)?) .execute(&self.pool) @@ -50,11 +50,9 @@ impl PostgresStore { if let Some(row) = row { let asset_type_str: String = row.get("asset_type"); - let kind = match asset_type_str.as_str() { - "IcebergTable" => pangolin_core::model::AssetType::IcebergTable, - "View" => pangolin_core::model::AssetType::View, - _ => pangolin_core::model::AssetType::IcebergTable, - }; + // B7: an unknown value used to fall through to `IcebergTable`. + let kind = pangolin_core::model::AssetType::from_stored_str(&asset_type_str) + .map_err(|e| anyhow::anyhow!(e))?; let a = Asset { id: row.get("id"), @@ -86,11 +84,9 @@ impl PostgresStore { let catalog_name: String = row.get("catalog_name"); let namespace_path: Vec = row.get("namespace_path"); let asset_type_str: String = row.get("asset_type"); - let kind = match asset_type_str.as_str() { - "IcebergTable" => pangolin_core::model::AssetType::IcebergTable, - "View" => pangolin_core::model::AssetType::View, - _ => pangolin_core::model::AssetType::IcebergTable, - }; + // B7: an unknown value used to fall through to `IcebergTable`. + let kind = pangolin_core::model::AssetType::from_stored_str(&asset_type_str) + .map_err(|e| anyhow::anyhow!(e))?; let asset = Asset { id: row.get("id"), @@ -137,11 +133,9 @@ impl PostgresStore { let mut assets = Vec::new(); for row in rows { let asset_type_str: String = row.get("asset_type"); - let kind = match asset_type_str.as_str() { - "IcebergTable" => pangolin_core::model::AssetType::IcebergTable, - "View" => pangolin_core::model::AssetType::View, - _ => pangolin_core::model::AssetType::IcebergTable, - }; + // B7: an unknown value used to fall through to `IcebergTable`. + let kind = pangolin_core::model::AssetType::from_stored_str(&asset_type_str) + .map_err(|e| anyhow::anyhow!(e))?; let a = Asset { id: row.get("id"), diff --git a/pangolin/pangolin_store/src/postgres/branches.rs b/pangolin/pangolin_store/src/postgres/branches.rs index 26177d9..7571fc2 100644 --- a/pangolin/pangolin_store/src/postgres/branches.rs +++ b/pangolin/pangolin_store/src/postgres/branches.rs @@ -69,7 +69,7 @@ impl PostgresStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT name, head_commit_id, branch_type, assets FROM branches WHERE tenant_id = $1 AND catalog_name = $2 LIMIT $3 OFFSET $4") + let rows = sqlx::query("SELECT name, head_commit_id, branch_type, assets FROM branches WHERE tenant_id = $1 AND catalog_name = $2 ORDER BY name LIMIT $3 OFFSET $4") .bind(tenant_id) .bind(catalog_name) .bind(limit) diff --git a/pangolin/pangolin_store/src/postgres/business_metadata.rs b/pangolin/pangolin_store/src/postgres/business_metadata.rs new file mode 100644 index 0000000..72dcb6d --- /dev/null +++ b/pangolin/pangolin_store/src/postgres/business_metadata.rs @@ -0,0 +1,95 @@ +//! Business metadata for the Postgres backend. +//! +//! These three methods did not exist. `PostgresStore` inherited the trait's +//! "Operation not supported by this store" defaults, while `search_assets` +//! joined a `business_metadata` table that no migration created - so on +//! Postgres, business metadata could not be written *and* asset search failed +//! with a hard SQL error rather than returning nothing. +//! +//! Found by the cross-backend parity suite on its first run against a live +//! Postgres. The suite asserts tag filtering works identically on all four +//! backends, which is not something a per-backend test can check. + +use super::PostgresStore; +use anyhow::Result; +use pangolin_core::business_metadata::BusinessMetadata; +use sqlx::Row; +use uuid::Uuid; + +impl PostgresStore { + /// Insert or replace an asset's business metadata. + /// + /// Keyed on `asset_id` rather than `id`: an asset has at most one metadata + /// record, and callers construct a fresh `BusinessMetadata` (with a new + /// `id`) when updating. Conflicting on `id` would insert a duplicate row + /// per update and break the `asset_id` unique constraint. + pub async fn upsert_business_metadata(&self, metadata: BusinessMetadata) -> Result<()> { + sqlx::query( + "INSERT INTO business_metadata ( + id, asset_id, description, tags, properties, + discoverable, created_by, created_at, updated_by, updated_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (asset_id) DO UPDATE SET + description = EXCLUDED.description, + tags = EXCLUDED.tags, + properties = EXCLUDED.properties, + discoverable = EXCLUDED.discoverable, + updated_by = EXCLUDED.updated_by, + updated_at = EXCLUDED.updated_at", + ) + .bind(metadata.id) + .bind(metadata.asset_id) + .bind(&metadata.description) + .bind(serde_json::to_value(&metadata.tags)?) + .bind(serde_json::to_value(&metadata.properties)?) + .bind(metadata.discoverable) + .bind(metadata.created_by) + .bind(metadata.created_at) + .bind(metadata.updated_by) + .bind(metadata.updated_at) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn get_business_metadata(&self, asset_id: Uuid) -> Result> { + let row = sqlx::query( + "SELECT id, asset_id, description, tags, properties, + discoverable, created_by, created_at, updated_by, updated_at + FROM business_metadata + WHERE asset_id = $1", + ) + .bind(asset_id) + .fetch_optional(&self.pool) + .await?; + + let Some(row) = row else { + return Ok(None); + }; + + Ok(Some(BusinessMetadata { + id: row.get("id"), + asset_id: row.get("asset_id"), + description: row.get("description"), + tags: serde_json::from_value(row.get("tags")).unwrap_or_default(), + properties: serde_json::from_value(row.get("properties")).unwrap_or_default(), + discoverable: row.get("discoverable"), + created_by: row.get("created_by"), + created_at: row.get("created_at"), + updated_by: row.get("updated_by"), + updated_at: row.get("updated_at"), + })) + } + + /// Delete an asset's entire metadata record. + /// + /// Deleting something that is not there is not an error: callers use this + /// to ensure absence. + pub async fn delete_business_metadata(&self, asset_id: Uuid) -> Result<()> { + sqlx::query("DELETE FROM business_metadata WHERE asset_id = $1") + .bind(asset_id) + .execute(&self.pool) + .await?; + Ok(()) + } +} diff --git a/pangolin/pangolin_store/src/postgres/catalogs.rs b/pangolin/pangolin_store/src/postgres/catalogs.rs index fd73273..6978579 100644 --- a/pangolin/pangolin_store/src/postgres/catalogs.rs +++ b/pangolin/pangolin_store/src/postgres/catalogs.rs @@ -62,7 +62,18 @@ impl PostgresStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT id, name, warehouse_name, storage_location, properties FROM catalogs WHERE tenant_id = $1 LIMIT $2 OFFSET $3") + // B24: the SELECT omitted `catalog_type` and `federated_config`, and the + // loop below hardcoded `Local` / `None`. Every federated catalog looked + // Local in a Postgres listing, so anything branching on `catalog_type` + // over a listing - including the federated-forwarding decision - took + // the wrong path. SQLite and Mongo returned the real values; only + // Postgres invented them. `get_catalog` two functions up decodes both + // correctly, which is the shape mirrored here. + // + // B27: `ORDER BY name` so two pages cover the set exactly once. Without + // it Postgres may return rows in any order between queries, so + // `LIMIT/OFFSET` paging could repeat or skip catalogs. + let rows = sqlx::query("SELECT id, name, catalog_type, warehouse_name, storage_location, federated_config, properties FROM catalogs WHERE tenant_id = $1 ORDER BY name LIMIT $2 OFFSET $3") .bind(tenant_id) .bind(limit) .bind(offset) @@ -71,13 +82,19 @@ impl PostgresStore { let mut catalogs = Vec::new(); for row in rows { + let catalog_type_str: String = row.get("catalog_type"); + let catalog_type = match catalog_type_str.as_str() { + "Federated" => pangolin_core::model::CatalogType::Federated, + _ => pangolin_core::model::CatalogType::Local, + }; + catalogs.push(Catalog { id: row.get("id"), name: row.get("name"), - catalog_type: pangolin_core::model::CatalogType::Local, + catalog_type, warehouse_name: row.get("warehouse_name"), storage_location: row.get("storage_location"), - federated_config: None, + federated_config: serde_json::from_value(row.get("federated_config")).ok(), properties: serde_json::from_value(row.get("properties")).unwrap_or_default(), }); } diff --git a/pangolin/pangolin_store/src/postgres/main.rs b/pangolin/pangolin_store/src/postgres/main.rs index cf34019..8191a76 100644 --- a/pangolin/pangolin_store/src/postgres/main.rs +++ b/pangolin/pangolin_store/src/postgres/main.rs @@ -256,6 +256,17 @@ impl CatalogStore for PostgresStore { .await } + async fn replace_namespace_properties( + &self, + tenant_id: Uuid, + catalog_name: &str, + namespace: Vec, + properties: std::collections::HashMap, + ) -> Result<()> { + self.replace_namespace_properties(tenant_id, catalog_name, namespace, properties) + .await + } + // Asset Operations async fn create_asset( &self, @@ -414,7 +425,7 @@ impl CatalogStore for PostgresStore { .unwrap_or(0); let rows = if let Some(uid) = user_id { - sqlx::query("SELECT token_id, user_id, tenant_id, token, expires_at FROM active_tokens WHERE tenant_id = $1 AND user_id = $2 AND expires_at > $3 LIMIT $4 OFFSET $5") + sqlx::query("SELECT token_id, user_id, tenant_id, token, expires_at FROM active_tokens WHERE tenant_id = $1 AND user_id = $2 AND expires_at > $3 ORDER BY expires_at DESC, token_id LIMIT $4 OFFSET $5") .bind(tenant_id) .bind(uid) .bind(Utc::now()) @@ -423,7 +434,7 @@ impl CatalogStore for PostgresStore { .fetch_all(&self.pool) .await? } else { - sqlx::query("SELECT token_id, user_id, tenant_id, token, expires_at FROM active_tokens WHERE tenant_id = $1 AND expires_at > $2 LIMIT $3 OFFSET $4") + sqlx::query("SELECT token_id, user_id, tenant_id, token, expires_at FROM active_tokens WHERE tenant_id = $1 AND expires_at > $2 ORDER BY expires_at DESC, token_id LIMIT $3 OFFSET $4") .bind(tenant_id) .bind(Utc::now()) .bind(limit) @@ -1153,6 +1164,38 @@ impl CatalogStore for PostgresStore { } } + async fn delete_file(&self, path: &str) -> Result<()> { + self.metadata_cache.invalidate(path).await; + let storage_config = self + .get_warehouse_for_location(path) + .await? + .map(|w| w.storage_config); + crate::file_delete::delete_location(storage_config.as_ref(), path).await + } + + // Business Metadata Operations + // + // These were absent, so `PostgresStore` fell through to the trait defaults + // ("Operation not supported by this store") while `search_assets` joined a + // table no migration created. + async fn upsert_business_metadata( + &self, + metadata: pangolin_core::business_metadata::BusinessMetadata, + ) -> Result<()> { + self.upsert_business_metadata(metadata).await + } + + async fn get_business_metadata( + &self, + asset_id: Uuid, + ) -> Result> { + self.get_business_metadata(asset_id).await + } + + async fn delete_business_metadata(&self, asset_id: Uuid) -> Result<()> { + self.delete_business_metadata(asset_id).await + } + // Tag Operations async fn create_tag(&self, tenant_id: Uuid, catalog_name: &str, tag: Tag) -> Result<()> { self.create_tag(tenant_id, catalog_name, tag).await @@ -1325,10 +1368,11 @@ impl CatalogStore for PostgresStore { m.created_at as meta_created_at, m.updated_by as meta_updated_by, m.updated_at as meta_updated_at FROM assets a LEFT JOIN business_metadata m ON a.id = m.asset_id - WHERE a.tenant_id = $1 AND (a.name ILIKE $2 OR m.description ILIKE $2)" + WHERE a.tenant_id = $1 AND (a.name ILIKE $2 ESCAPE '\\' OR m.description ILIKE $2 ESCAPE '\\')" ); - let query_pattern = format!("%{}%", query); + // B28: unescaped `%`/`_` in the term were LIKE wildcards. + let query_pattern = crate::search::contains_pattern(query); let mut param_index = 3; if let Some(ref tag_list) = tags { @@ -1388,8 +1432,13 @@ impl CatalogStore for PostgresStore { }; let catalog_name: String = row.get("catalog_name"); - let namespace_path: String = row.get("namespace_path"); - let namespace: Vec = namespace_path.split('\x1F').map(String::from).collect(); + // B4: this decoded a `TEXT[]` column as `String`. `sqlx::Row::get` + // *panics* on a decode failure, so any search with at least one hit + // panicked the request - the failure only stayed hidden because a + // search with no results never reached this line. The correct + // decode is already used elsewhere in this file and in + // `postgres/assets.rs`. + let namespace: Vec = row.get("namespace_path"); results.push((asset, metadata, catalog_name, namespace)); } @@ -1398,8 +1447,9 @@ impl CatalogStore for PostgresStore { } async fn search_catalogs(&self, tenant_id: Uuid, query: &str) -> Result> { - let query_pattern = format!("%{}%", query); - let rows = sqlx::query("SELECT id, name, catalog_type, warehouse_name, storage_location, federated_config, properties FROM catalogs WHERE tenant_id = $1 AND name ILIKE $2") + // B28: unescaped `%`/`_` in the term were LIKE wildcards. + let query_pattern = crate::search::contains_pattern(query); + let rows = sqlx::query("SELECT id, name, catalog_type, warehouse_name, storage_location, federated_config, properties FROM catalogs WHERE tenant_id = $1 AND name ILIKE $2 ESCAPE '\\' ORDER BY name") .bind(tenant_id) .bind(&query_pattern) .fetch_all(&self.pool) @@ -1434,9 +1484,10 @@ impl CatalogStore for PostgresStore { tenant_id: Uuid, query: &str, ) -> Result> { - let query_pattern = format!("%{}%", query); + // B28: unescaped `%`/`_` in the term were LIKE wildcards. + let query_pattern = crate::search::contains_pattern(query); // Postgres stores namespace_path as TEXT[] - let rows = sqlx::query("SELECT catalog_name, namespace_path, properties FROM namespaces WHERE tenant_id = $1 AND array_to_string(namespace_path, '.') ILIKE $2") + let rows = sqlx::query("SELECT catalog_name, namespace_path, properties FROM namespaces WHERE tenant_id = $1 AND array_to_string(namespace_path, '.') ILIKE $2 ESCAPE '\\' ORDER BY catalog_name, namespace_path") .bind(tenant_id) .bind(&query_pattern) .fetch_all(&self.pool) @@ -1456,8 +1507,9 @@ impl CatalogStore for PostgresStore { } async fn search_branches(&self, tenant_id: Uuid, query: &str) -> Result> { - let query_pattern = format!("%{}%", query); - let rows = sqlx::query("SELECT catalog_name, name, head_commit_id, branch_type, assets FROM branches WHERE tenant_id = $1 AND name ILIKE $2") + // B28: unescaped `%`/`_` in the term were LIKE wildcards. + let query_pattern = crate::search::contains_pattern(query); + let rows = sqlx::query("SELECT catalog_name, name, head_commit_id, branch_type, assets FROM branches WHERE tenant_id = $1 AND name ILIKE $2 ESCAPE '\\' ORDER BY catalog_name, name") .bind(tenant_id) .bind(&query_pattern) .fetch_all(&self.pool) diff --git a/pangolin/pangolin_store/src/postgres/mod.rs b/pangolin/pangolin_store/src/postgres/mod.rs index 923f662..e25aab6 100644 --- a/pangolin/pangolin_store/src/postgres/mod.rs +++ b/pangolin/pangolin_store/src/postgres/mod.rs @@ -2,6 +2,7 @@ pub mod access_requests; pub mod assets; pub mod audit; pub mod branches; +pub mod business_metadata; pub mod catalogs; pub mod commits; pub mod main; diff --git a/pangolin/pangolin_store/src/postgres/namespaces.rs b/pangolin/pangolin_store/src/postgres/namespaces.rs index 22283ea..9fa5cf1 100644 --- a/pangolin/pangolin_store/src/postgres/namespaces.rs +++ b/pangolin/pangolin_store/src/postgres/namespaces.rs @@ -60,7 +60,7 @@ impl PostgresStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT namespace_path, properties FROM namespaces WHERE tenant_id = $1 AND catalog_name = $2 LIMIT $3 OFFSET $4") + let rows = sqlx::query("SELECT namespace_path, properties FROM namespaces WHERE tenant_id = $1 AND catalog_name = $2 ORDER BY namespace_path LIMIT $3 OFFSET $4") .bind(tenant_id) .bind(catalog_name) .bind(limit) @@ -116,6 +116,28 @@ impl PostgresStore { Ok(()) } + /// Replace a namespace's properties wholesale; see the SQLite twin for why + /// a merge-only method could not implement Iceberg property removals (B16h). + pub async fn replace_namespace_properties( + &self, + tenant_id: Uuid, + catalog_name: &str, + namespace: Vec, + properties: std::collections::HashMap, + ) -> Result<()> { + let result = sqlx::query("UPDATE namespaces SET properties = $1 WHERE tenant_id = $2 AND catalog_name = $3 AND namespace_path = $4") + .bind(serde_json::to_value(&properties)?) + .bind(tenant_id) + .bind(catalog_name) + .bind(&namespace) + .execute(&self.pool) + .await?; + if result.rows_affected() == 0 { + return Err(anyhow::anyhow!("Namespace not found")); + } + Ok(()) + } + pub async fn count_namespaces(&self, tenant_id: Uuid) -> Result { let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM namespaces WHERE tenant_id = $1") .bind(tenant_id) diff --git a/pangolin/pangolin_store/src/postgres/permissions.rs b/pangolin/pangolin_store/src/postgres/permissions.rs index f77f87a..7899486 100644 --- a/pangolin/pangolin_store/src/postgres/permissions.rs +++ b/pangolin/pangolin_store/src/postgres/permissions.rs @@ -113,7 +113,7 @@ impl PostgresStore { "SELECT id, user_id, tenant_id, scope, actions, granted_by, granted_at FROM permissions WHERE tenant_id = $1 - LIMIT $2 OFFSET $3", + ORDER BY id LIMIT $2 OFFSET $3", ) .bind(tenant_id) .bind(limit) diff --git a/pangolin/pangolin_store/src/postgres/roles.rs b/pangolin/pangolin_store/src/postgres/roles.rs index 8727f13..b8025eb 100644 --- a/pangolin/pangolin_store/src/postgres/roles.rs +++ b/pangolin/pangolin_store/src/postgres/roles.rs @@ -57,7 +57,7 @@ impl PostgresStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT id, tenant_id, name, description, permissions, created_by, created_at, updated_at FROM roles WHERE tenant_id = $1 LIMIT $2 OFFSET $3") + let rows = sqlx::query("SELECT id, tenant_id, name, description, permissions, created_by, created_at, updated_at FROM roles WHERE tenant_id = $1 ORDER BY name LIMIT $2 OFFSET $3") .bind(tenant_id) .bind(limit) .bind(offset) diff --git a/pangolin/pangolin_store/src/postgres/tags.rs b/pangolin/pangolin_store/src/postgres/tags.rs index 63a4db2..521591d 100644 --- a/pangolin/pangolin_store/src/postgres/tags.rs +++ b/pangolin/pangolin_store/src/postgres/tags.rs @@ -55,7 +55,7 @@ impl PostgresStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT name, commit_id FROM tags WHERE tenant_id = $1 AND catalog_name = $2 LIMIT $3 OFFSET $4") + let rows = sqlx::query("SELECT name, commit_id FROM tags WHERE tenant_id = $1 AND catalog_name = $2 ORDER BY name LIMIT $3 OFFSET $4") .bind(tenant_id) .bind(catalog_name) .bind(limit) diff --git a/pangolin/pangolin_store/src/postgres/tenants.rs b/pangolin/pangolin_store/src/postgres/tenants.rs index df2a7a7..7179cdc 100644 --- a/pangolin/pangolin_store/src/postgres/tenants.rs +++ b/pangolin/pangolin_store/src/postgres/tenants.rs @@ -45,11 +45,13 @@ impl PostgresStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT id, name, properties FROM tenants LIMIT $1 OFFSET $2") - .bind(limit) - .bind(offset) - .fetch_all(&self.pool) - .await?; + let rows = sqlx::query( + "SELECT id, name, properties FROM tenants ORDER BY name LIMIT $1 OFFSET $2", + ) + .bind(limit) + .bind(offset) + .fetch_all(&self.pool) + .await?; let mut tenants = Vec::new(); for row in rows { diff --git a/pangolin/pangolin_store/src/postgres/users.rs b/pangolin/pangolin_store/src/postgres/users.rs index f52b27a..c5beb32 100644 --- a/pangolin/pangolin_store/src/postgres/users.rs +++ b/pangolin/pangolin_store/src/postgres/users.rs @@ -98,14 +98,14 @@ impl PostgresStore { .unwrap_or(0); let rows = if let Some(tid) = tenant_id { - sqlx::query("SELECT id, username, email, password_hash, oauth_provider, oauth_subject, tenant_id, role, active, created_at, updated_at, last_login FROM users WHERE tenant_id = $1 LIMIT $2 OFFSET $3") + sqlx::query("SELECT id, username, email, password_hash, oauth_provider, oauth_subject, tenant_id, role, active, created_at, updated_at, last_login FROM users WHERE tenant_id = $1 ORDER BY username LIMIT $2 OFFSET $3") .bind(tid) .bind(limit) .bind(offset) .fetch_all(&self.pool) .await? } else { - sqlx::query("SELECT id, username, email, password_hash, oauth_provider, oauth_subject, tenant_id, role, active, created_at, updated_at, last_login FROM users LIMIT $1 OFFSET $2") + sqlx::query("SELECT id, username, email, password_hash, oauth_provider, oauth_subject, tenant_id, role, active, created_at, updated_at, last_login FROM users ORDER BY username LIMIT $1 OFFSET $2") .bind(limit) .bind(offset) .fetch_all(&self.pool) diff --git a/pangolin/pangolin_store/src/postgres/warehouses.rs b/pangolin/pangolin_store/src/postgres/warehouses.rs index 61c72de..769b7a9 100644 --- a/pangolin/pangolin_store/src/postgres/warehouses.rs +++ b/pangolin/pangolin_store/src/postgres/warehouses.rs @@ -53,7 +53,7 @@ impl PostgresStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT id, name, tenant_id, use_sts, storage_config, vending_strategy FROM warehouses WHERE tenant_id = $1 LIMIT $2 OFFSET $3") + let rows = sqlx::query("SELECT id, name, tenant_id, use_sts, storage_config, vending_strategy FROM warehouses WHERE tenant_id = $1 ORDER BY name LIMIT $2 OFFSET $3") .bind(tenant_id) .bind(limit) .bind(offset) diff --git a/pangolin/pangolin_store/src/search.rs b/pangolin/pangolin_store/src/search.rs new file mode 100644 index 0000000..7e271c8 --- /dev/null +++ b/pangolin/pangolin_store/src/search.rs @@ -0,0 +1,94 @@ +//! Shared search semantics for the four backends. +//! +//! B28: search behaved four different ways. +//! +//! * **Wildcards.** Postgres and SQLite built their patterns with +//! `format!("%{}%", query)` and no escaping, so a query containing `%` or `_` +//! was interpreted as a LIKE wildcard: searching for `100%` matched +//! everything, and `a_b` matched `axb`. Mongo escaped correctly with +//! `regex::escape`, and the memory backend used a literal `contains`. Four +//! backends, four answers to the same query. +//! * **Tag filters.** Memory and SQLite matched *any* requested tag; Postgres +//! (`@>`) and Mongo (`$all`) required *all* of them. And an empty tag list +//! returned zero results on memory but everything on the others. +//! +//! This module is the single definition of both, so a backend can only diverge +//! by not calling it. + +/// The escape character used with `LIKE ... ESCAPE`. +pub const LIKE_ESCAPE_CHAR: char = '\\'; + +/// Escape LIKE metacharacters in a user-supplied search term. +/// +/// Must be paired with `ESCAPE '\'` in the SQL, which the helpers below embed. +pub fn escape_like(query: &str) -> String { + let mut escaped = String::with_capacity(query.len()); + for ch in query.chars() { + if ch == '%' || ch == '_' || ch == LIKE_ESCAPE_CHAR { + escaped.push(LIKE_ESCAPE_CHAR); + } + escaped.push(ch); + } + escaped +} + +/// Build a `%term%` LIKE pattern with metacharacters escaped. +pub fn contains_pattern(query: &str) -> String { + format!("%{}%", escape_like(query)) +} + +/// The `ESCAPE` clause every `LIKE`/`ILIKE` using [`contains_pattern`] needs. +pub const ESCAPE_CLAUSE: &str = " ESCAPE '\\'"; + +/// Does `tags` satisfy a tag filter? +/// +/// **The chosen semantic is ALL-match**: a result qualifies only if it carries +/// every requested tag. That is what Postgres's `@>` and Mongo's `$all` already +/// did, so aligning on it keeps the two SQL/document backends unchanged and +/// moves memory and SQLite - and it is the semantic faceted filtering wants, +/// where each added tag narrows the result set. +/// +/// An **empty or absent** filter means "no tag constraint", matching everything. +/// Previously an empty list returned nothing on the memory backend and +/// everything elsewhere. +pub fn tags_match(tags: &[String], required: Option<&[String]>) -> bool { + match required { + None => true, + Some(required) => required.iter().all(|want| tags.contains(want)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn like_metacharacters_are_escaped() { + assert_eq!(escape_like("100%"), "100\\%"); + assert_eq!(escape_like("a_b"), "a\\_b"); + assert_eq!(escape_like("back\\slash"), "back\\\\slash"); + assert_eq!(escape_like("plain"), "plain"); + } + + #[test] + fn contains_pattern_wraps_the_escaped_term() { + assert_eq!(contains_pattern("50%"), "%50\\%%"); + } + + #[test] + fn tag_filter_is_all_match() { + let tags = vec!["pii".to_string(), "finance".to_string()]; + + assert!(tags_match(&tags, None), "no filter matches everything"); + assert!(tags_match(&tags, Some(&[])), "an empty filter is no filter"); + assert!(tags_match(&tags, Some(&["pii".to_string()]))); + assert!(tags_match( + &tags, + Some(&["pii".to_string(), "finance".to_string()]) + )); + assert!( + !tags_match(&tags, Some(&["pii".to_string(), "hr".to_string()])), + "every requested tag must be present" + ); + } +} diff --git a/pangolin/pangolin_store/src/sqlite/access_requests.rs b/pangolin/pangolin_store/src/sqlite/access_requests.rs index 5cc50f4..b4874dd 100644 --- a/pangolin/pangolin_store/src/sqlite/access_requests.rs +++ b/pangolin/pangolin_store/src/sqlite/access_requests.rs @@ -52,13 +52,14 @@ impl SqliteStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = - sqlx::query("SELECT * FROM access_requests WHERE tenant_id = ? LIMIT ? OFFSET ?") - .bind(tenant_id.to_string()) - .bind(limit) - .bind(offset) - .fetch_all(&self.pool) - .await?; + let rows = sqlx::query( + "SELECT * FROM access_requests WHERE tenant_id = ? ORDER BY id LIMIT ? OFFSET ?", + ) + .bind(tenant_id.to_string()) + .bind(limit) + .bind(offset) + .fetch_all(&self.pool) + .await?; let mut requests = Vec::new(); for row in rows { diff --git a/pangolin/pangolin_store/src/sqlite/assets.rs b/pangolin/pangolin_store/src/sqlite/assets.rs index 9ec47d0..b4dad5f 100644 --- a/pangolin/pangolin_store/src/sqlite/assets.rs +++ b/pangolin/pangolin_store/src/sqlite/assets.rs @@ -24,7 +24,7 @@ impl SqliteStore { .bind(&namespace_path) .bind(&asset.name) .bind(&branch_name) - .bind(format!("{:?}", asset.kind)) + .bind(asset.kind.as_stored_str()) .bind(asset.properties.get("metadata_location").unwrap_or(&asset.location)) .bind(serde_json::to_string(&asset.properties)?) .execute(&self.pool) @@ -139,11 +139,9 @@ impl SqliteStore { if let Some(row) = row { let asset_type_str: String = row.get("asset_type"); - let kind = match asset_type_str.as_str() { - "IcebergTable" => AssetType::IcebergTable, - "View" => AssetType::View, - _ => AssetType::IcebergTable, - }; + // B7: an unknown value used to fall through to `IcebergTable`. + let kind = + AssetType::from_stored_str(&asset_type_str).map_err(|e| anyhow::anyhow!(e))?; Ok(Some(Asset { id: Uuid::parse_str(&row.get::("id"))?, @@ -178,11 +176,9 @@ impl SqliteStore { serde_json::from_str(&namespace_json).unwrap_or_default(); let asset_type_str: String = row.get("asset_type"); - let kind = match asset_type_str.as_str() { - "IcebergTable" => AssetType::IcebergTable, - "View" => AssetType::View, - _ => AssetType::IcebergTable, - }; + // B7: an unknown value used to fall through to `IcebergTable`. + let kind = + AssetType::from_stored_str(&asset_type_str).map_err(|e| anyhow::anyhow!(e))?; let asset = Asset { id: Uuid::parse_str(&row.get::("id"))?, @@ -218,7 +214,7 @@ impl SqliteStore { let namespace_path = serde_json::to_string(&namespace)?; let branch_name = branch.unwrap_or_else(|| "main".to_string()); - let rows = sqlx::query("SELECT id, name, asset_type, metadata_location, properties FROM assets WHERE tenant_id = ? AND catalog_name = ? AND namespace_path = ? AND branch_name = ? LIMIT ? OFFSET ?") + let rows = sqlx::query("SELECT id, name, asset_type, metadata_location, properties FROM assets WHERE tenant_id = ? AND catalog_name = ? AND namespace_path = ? AND branch_name = ? ORDER BY name LIMIT ? OFFSET ?") .bind(tenant_id.to_string()) .bind(catalog_name) .bind(&namespace_path) @@ -231,11 +227,9 @@ impl SqliteStore { let mut assets = Vec::new(); for row in rows { let asset_type_str: String = row.get("asset_type"); - let kind = match asset_type_str.as_str() { - "IcebergTable" => AssetType::IcebergTable, - "View" => AssetType::View, - _ => AssetType::IcebergTable, - }; + // B7: an unknown value used to fall through to `IcebergTable`. + let kind = + AssetType::from_stored_str(&asset_type_str).map_err(|e| anyhow::anyhow!(e))?; assets.push(Asset { id: Uuid::parse_str(&row.get::("id"))?, @@ -319,16 +313,23 @@ impl SqliteStore { &self, tenant_id: Uuid, catalog_name: &str, - _branch: Option, + branch: Option, namespace: Vec, table: String, ) -> Result> { + // B19: `branch` was discarded (`_branch`) and the query matched rows + // from *every* branch, with `fetch_optional` returning an arbitrary one. + // Reading a table on `dev` could hand back `main`'s metadata pointer - + // silently, and differently depending on row order. Postgres and Mongo + // both scope by branch. + let branch_name = branch.unwrap_or_else(|| "main".to_string()); let namespace_path = serde_json::to_string(&namespace)?; - let row = sqlx::query("SELECT metadata_location, properties FROM assets WHERE tenant_id = ? AND catalog_name = ? AND namespace_path = ? AND name = ?") + let row = sqlx::query("SELECT metadata_location, properties FROM assets WHERE tenant_id = ? AND catalog_name = ? AND namespace_path = ? AND name = ? AND branch_name = ?") .bind(tenant_id.to_string()) .bind(catalog_name) .bind(&namespace_path) .bind(&table) + .bind(&branch_name) .fetch_optional(&self.pool) .await?; @@ -388,9 +389,15 @@ impl SqliteStore { )); } - props.insert("metadata_location".to_string(), new_location); + props.insert("metadata_location".to_string(), new_location.clone()); - let update_result = sqlx::query("UPDATE assets SET properties = ? WHERE tenant_id = ? AND catalog_name = ? AND namespace_path = ? AND name = ? AND branch_name = ?") + // B20: only `properties` was updated, leaving the `metadata_location` + // *column* stale - and reads populate `Asset.location` from that + // column. On SQLite an asset's `location` was therefore frozen at + // creation time no matter how many Iceberg commits followed. + // Postgres updates both. + let update_result = sqlx::query("UPDATE assets SET metadata_location = ?, properties = ? WHERE tenant_id = ? AND catalog_name = ? AND namespace_path = ? AND name = ? AND branch_name = ?") + .bind(&new_location) .bind(serde_json::to_string(&props)?) .bind(tenant_id.to_string()) .bind(catalog_name) diff --git a/pangolin/pangolin_store/src/sqlite/audit_logs.rs b/pangolin/pangolin_store/src/sqlite/audit_logs.rs index 9e6ae43..3cd4254 100644 --- a/pangolin/pangolin_store/src/sqlite/audit_logs.rs +++ b/pangolin/pangolin_store/src/sqlite/audit_logs.rs @@ -19,14 +19,16 @@ impl SqliteStore { .bind(tenant_id.to_string()) .bind(event.user_id.map(|u| u.to_string())) .bind(&event.username) - .bind(format!("{:?}", event.action)) - .bind(format!("{:?}", event.resource_type)) + .bind(pangolin_core::audit::audit_enum_to_stored(&event.action)) + .bind(pangolin_core::audit::audit_enum_to_stored( + &event.resource_type, + )) .bind(event.resource_id.map(|u| u.to_string())) .bind(&event.resource_name) .bind(event.timestamp.timestamp_millis()) .bind(event.ip_address.as_deref().unwrap_or("")) .bind(event.user_agent.as_deref().unwrap_or("")) - .bind(format!("{:?}", event.result)) + .bind(pangolin_core::audit::audit_enum_to_stored(&event.result)) .bind(event.error_message.as_deref().unwrap_or("")) .bind(serde_json::to_string(&event.metadata)?) .execute(&self.pool) @@ -103,18 +105,22 @@ impl SqliteStore { let ts_millis: i64 = row.get("timestamp"); // Parse enums from strings + // B22: these used to lowercase the stored Debug spelling and + // deserialize against snake_case, which never matched, then swallow + // the failure with `.unwrap_or(CreateCatalog)`. Errors now + // propagate: a corrupt audit row is a loud failure, not a + // plausible-looking lie about what happened. let action_str: String = row.get("action"); - let action = serde_json::from_str(&format!("\"{}\"", action_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::AuditAction::CreateCatalog); + let action = pangolin_core::audit::audit_enum_from_stored(&action_str) + .map_err(|e| anyhow::anyhow!(e))?; let resource_type_str: String = row.get("resource_type"); - let resource_type = - serde_json::from_str(&format!("\"{}\"", resource_type_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::ResourceType::Catalog); + let resource_type = pangolin_core::audit::audit_enum_from_stored(&resource_type_str) + .map_err(|e| anyhow::anyhow!(e))?; let result_str: String = row.get("result"); - let result = serde_json::from_str(&format!("\"{}\"", result_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::AuditResult::Success); + let result = pangolin_core::audit::audit_enum_from_stored(&result_str) + .map_err(|e| anyhow::anyhow!(e))?; events.push(AuditLogEntry { id: Uuid::parse_str(&row.get::("id"))?, @@ -183,18 +189,22 @@ impl SqliteStore { if let Some(row) = row { let ts_millis: i64 = row.get("timestamp"); + // B22: these used to lowercase the stored Debug spelling and + // deserialize against snake_case, which never matched, then swallow + // the failure with `.unwrap_or(CreateCatalog)`. Errors now + // propagate: a corrupt audit row is a loud failure, not a + // plausible-looking lie about what happened. let action_str: String = row.get("action"); - let action = serde_json::from_str(&format!("\"{}\"", action_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::AuditAction::CreateCatalog); + let action = pangolin_core::audit::audit_enum_from_stored(&action_str) + .map_err(|e| anyhow::anyhow!(e))?; let resource_type_str: String = row.get("resource_type"); - let resource_type = - serde_json::from_str(&format!("\"{}\"", resource_type_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::ResourceType::Catalog); + let resource_type = pangolin_core::audit::audit_enum_from_stored(&resource_type_str) + .map_err(|e| anyhow::anyhow!(e))?; let result_str: String = row.get("result"); - let result = serde_json::from_str(&format!("\"{}\"", result_str.to_lowercase())) - .unwrap_or(pangolin_core::audit::AuditResult::Success); + let result = pangolin_core::audit::audit_enum_from_stored(&result_str) + .map_err(|e| anyhow::anyhow!(e))?; Ok(Some(AuditLogEntry { id: Uuid::parse_str(&row.get::("id"))?, diff --git a/pangolin/pangolin_store/src/sqlite/branches.rs b/pangolin/pangolin_store/src/sqlite/branches.rs index bcd6e6c..4dd8d0c 100644 --- a/pangolin/pangolin_store/src/sqlite/branches.rs +++ b/pangolin/pangolin_store/src/sqlite/branches.rs @@ -71,7 +71,7 @@ impl SqliteStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT name, head_commit_id, branch_type, assets FROM branches WHERE tenant_id = ? AND catalog_name = ? LIMIT ? OFFSET ?") + let rows = sqlx::query("SELECT name, head_commit_id, branch_type, assets FROM branches WHERE tenant_id = ? AND catalog_name = ? ORDER BY name LIMIT ? OFFSET ?") .bind(tenant_id.to_string()) .bind(catalog_name) .bind(limit) @@ -100,19 +100,32 @@ impl SqliteStore { Ok(branches) } + /// Delete a branch and the assets that live on it. + /// + /// B3: two defects, both fatal together. The asset cleanup referenced a + /// column named `branch`, but the schema column is `branch_name` + /// (`sql/sqlite_schema.sql:71`), so the statement failed with "no such + /// column". And because the branch delete had already been committed as its + /// own statement, the branch was gone while its assets survived - orphaned + /// permanently, with no branch to reach them through - and the caller got an + /// error suggesting nothing had happened. Postgres was fixed for exactly + /// this and wraps both statements in a transaction; SQLite was never + /// patched. pub async fn delete_branch( &self, tenant_id: Uuid, catalog_name: &str, name: String, ) -> Result<()> { + let mut tx = self.pool.begin().await?; + let result = sqlx::query( "DELETE FROM branches WHERE tenant_id = ? AND catalog_name = ? AND name = ?", ) .bind(tenant_id.to_string()) .bind(catalog_name) .bind(&name) - .execute(&self.pool) + .execute(&mut *tx) .await?; if result.rows_affected() == 0 { @@ -120,13 +133,16 @@ impl SqliteStore { } // Also delete assets associated with this branch - sqlx::query("DELETE FROM assets WHERE tenant_id = ? AND catalog_name = ? AND branch = ?") - .bind(tenant_id.to_string()) - .bind(catalog_name) - .bind(&name) - .execute(&self.pool) - .await?; + sqlx::query( + "DELETE FROM assets WHERE tenant_id = ? AND catalog_name = ? AND branch_name = ?", + ) + .bind(tenant_id.to_string()) + .bind(catalog_name) + .bind(&name) + .execute(&mut *tx) + .await?; + tx.commit().await?; Ok(()) } diff --git a/pangolin/pangolin_store/src/sqlite/business_metadata.rs b/pangolin/pangolin_store/src/sqlite/business_metadata.rs index 9b8c90d..19c87e0 100644 --- a/pangolin/pangolin_store/src/sqlite/business_metadata.rs +++ b/pangolin/pangolin_store/src/sqlite/business_metadata.rs @@ -81,21 +81,28 @@ impl SqliteStore { m.created_at as meta_created_at, m.updated_by as meta_updated_by, m.updated_at as meta_updated_at FROM assets a LEFT JOIN business_metadata m ON a.id = m.asset_id - WHERE a.tenant_id = ? AND (a.name LIKE ? OR m.description LIKE ?)" + WHERE a.tenant_id = ? AND (a.name LIKE ? ESCAPE '\\' OR m.description LIKE ? ESCAPE '\\')" ); - let query_pattern = format!("%{}%", query); + let query_pattern = crate::search::contains_pattern(query); + // B28: this was an ANY-match (`EXISTS ... value IN (...)`) while + // Postgres (`@>`) and Mongo (`$all`) required *all* the requested tags. + // The chosen semantic - see `crate::search::tags_match` - is ALL-match, + // so counting distinct matches against the requested count is what makes + // SQLite agree with the other three. if let Some(ref tag_list) = tags { if !tag_list.is_empty() { - sql.push_str(" AND EXISTS (SELECT 1 FROM json_each(m.tags) WHERE value IN ("); + sql.push_str( + " AND (SELECT COUNT(DISTINCT value) FROM json_each(m.tags) WHERE value IN (", + ); for (i, _) in tag_list.iter().enumerate() { if i > 0 { sql.push_str(", "); } sql.push('?'); } - sql.push_str("))"); + sql.push_str(&format!(")) = {}", tag_list.len())); } } @@ -117,11 +124,9 @@ impl SqliteStore { for row in rows { let asset_type_str: String = row.get("asset_type"); - let kind = match asset_type_str.as_str() { - "IcebergTable" => AssetType::IcebergTable, - "View" => AssetType::View, - _ => AssetType::IcebergTable, - }; + // B7: an unknown value used to fall through to `IcebergTable`. + let kind = + AssetType::from_stored_str(&asset_type_str).map_err(|e| anyhow::anyhow!(e))?; let asset = Asset { id: Uuid::parse_str(row.get("id"))?, @@ -153,8 +158,14 @@ impl SqliteStore { }; let catalog_name: String = row.get("catalog_name"); + // B4 (SQLite sibling): `namespace_path` is stored as a JSON array + // (`serde_json::to_string(&namespace)`), so splitting it on 0x1F + // yielded a single element containing raw JSON - a search result's + // namespace came back as `["[\"a\",\"b\"]"]` rather than + // `["a", "b"]`. No panic here, just a silently wrong namespace on + // every search hit. let namespace_path: String = row.get("namespace_path"); - let namespace: Vec = namespace_path.split('\x1F').map(String::from).collect(); + let namespace: Vec = serde_json::from_str(&namespace_path).unwrap_or_default(); results.push((asset, metadata, catalog_name, namespace)); } @@ -163,8 +174,8 @@ impl SqliteStore { } pub async fn search_catalogs(&self, tenant_id: Uuid, query: &str) -> Result> { - let query_pattern = format!("%{}%", query); - let rows = sqlx::query("SELECT id, name, catalog_type, warehouse_name, storage_location, federated_config, properties FROM catalogs WHERE tenant_id = ? AND name LIKE ?") + let query_pattern = crate::search::contains_pattern(query); + let rows = sqlx::query("SELECT id, name, catalog_type, warehouse_name, storage_location, federated_config, properties FROM catalogs WHERE tenant_id = ? AND name LIKE ? ESCAPE '\\' ORDER BY name") .bind(tenant_id.to_string()) .bind(&query_pattern) .fetch_all(&self.pool) @@ -197,8 +208,8 @@ impl SqliteStore { tenant_id: Uuid, query: &str, ) -> Result> { - let query_pattern = format!("%{}%", query); - let rows = sqlx::query("SELECT catalog_name, namespace_path, properties FROM namespaces WHERE tenant_id = ? AND namespace_path LIKE ?") + let query_pattern = crate::search::contains_pattern(query); + let rows = sqlx::query("SELECT catalog_name, namespace_path, properties FROM namespaces WHERE tenant_id = ? AND namespace_path LIKE ? ESCAPE '\\' ORDER BY catalog_name, namespace_path") .bind(tenant_id.to_string()) .bind(&query_pattern) .fetch_all(&self.pool) @@ -228,8 +239,8 @@ impl SqliteStore { tenant_id: Uuid, query: &str, ) -> Result> { - let query_pattern = format!("%{}%", query); - let rows = sqlx::query("SELECT catalog_name, name, head_commit_id, branch_type, assets FROM branches WHERE tenant_id = ? AND name LIKE ?") + let query_pattern = crate::search::contains_pattern(query); + let rows = sqlx::query("SELECT catalog_name, name, head_commit_id, branch_type, assets FROM branches WHERE tenant_id = ? AND name LIKE ? ESCAPE '\\' ORDER BY catalog_name, name") .bind(tenant_id.to_string()) .bind(&query_pattern) .fetch_all(&self.pool) diff --git a/pangolin/pangolin_store/src/sqlite/catalogs.rs b/pangolin/pangolin_store/src/sqlite/catalogs.rs index b3d4c5e..3c05307 100644 --- a/pangolin/pangolin_store/src/sqlite/catalogs.rs +++ b/pangolin/pangolin_store/src/sqlite/catalogs.rs @@ -64,7 +64,7 @@ impl SqliteStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT id, name, catalog_type, warehouse_name, storage_location, federated_config, properties FROM catalogs WHERE tenant_id = ? LIMIT ? OFFSET ?") + let rows = sqlx::query("SELECT id, name, catalog_type, warehouse_name, storage_location, federated_config, properties FROM catalogs WHERE tenant_id = ? ORDER BY name LIMIT ? OFFSET ?") .bind(tenant_id.to_string()) .bind(limit) .bind(offset) @@ -143,48 +143,50 @@ impl SqliteStore { .ok_or_else(|| anyhow::anyhow!("Catalog not found")) } + /// Delete a catalog and cascade to its children. + /// + /// B21: this used to run five sequential deletes with no transaction, and + /// the "does the catalog exist?" check was the *last* statement. So + /// `delete_catalog(tenant, "nonexistent")` cheerfully deleted every tag, + /// branch, asset and namespace whose `catalog_name` matched - and only then + /// returned "not found", leaving the caller believing nothing had happened. + /// Any failure part-way through left the same wreckage. Postgres wraps the + /// identical cascade in a transaction; this now matches it, and checks + /// existence first. pub async fn delete_catalog(&self, tenant_id: Uuid, name: String) -> Result<()> { let tid = tenant_id.to_string(); - // Delete cascading children manually (no FK constraints on catalog_name) - // 1. Tags - sqlx::query("DELETE FROM tags WHERE tenant_id = ? AND catalog_name = ?") - .bind(&tid) - .bind(&name) - .execute(&self.pool) - .await?; - - // 2. Branches - sqlx::query("DELETE FROM branches WHERE tenant_id = ? AND catalog_name = ?") - .bind(&tid) - .bind(&name) - .execute(&self.pool) - .await?; - - // 3. Assets - sqlx::query("DELETE FROM assets WHERE tenant_id = ? AND catalog_name = ?") - .bind(&tid) - .bind(&name) - .execute(&self.pool) - .await?; + let mut tx = self.pool.begin().await?; + + // Existence first: nothing is destroyed on behalf of a catalog that is + // not there. + let exists: Option<(i64,)> = + sqlx::query_as("SELECT 1 FROM catalogs WHERE tenant_id = ? AND name = ?") + .bind(&tid) + .bind(&name) + .fetch_optional(&mut *tx) + .await?; + if exists.is_none() { + return Err(anyhow::anyhow!("Catalog '{}' not found", name)); + } - // 4. Namespaces - sqlx::query("DELETE FROM namespaces WHERE tenant_id = ? AND catalog_name = ?") - .bind(&tid) - .bind(&name) - .execute(&self.pool) - .await?; + // Delete cascading children manually (no FK constraints on catalog_name) + for table in ["tags", "branches", "assets", "namespaces"] { + let sql = format!("DELETE FROM {table} WHERE tenant_id = ? AND catalog_name = ?"); + sqlx::query(&sql) + .bind(&tid) + .bind(&name) + .execute(&mut *tx) + .await?; + } - // 5. Catalog - let result = sqlx::query("DELETE FROM catalogs WHERE tenant_id = ? AND name = ?") + sqlx::query("DELETE FROM catalogs WHERE tenant_id = ? AND name = ?") .bind(&tid) .bind(&name) - .execute(&self.pool) + .execute(&mut *tx) .await?; - if result.rows_affected() == 0 { - return Err(anyhow::anyhow!("Catalog '{}' not found", name)); - } + tx.commit().await?; Ok(()) } } diff --git a/pangolin/pangolin_store/src/sqlite/main.rs b/pangolin/pangolin_store/src/sqlite/main.rs index edf6375..1527208 100644 --- a/pangolin/pangolin_store/src/sqlite/main.rs +++ b/pangolin/pangolin_store/src/sqlite/main.rs @@ -9,12 +9,15 @@ use pangolin_core::model::{SyncStats, SystemSettings}; use pangolin_core::permission::{Permission, Role, UserRole as UserRoleAssignment}; use pangolin_core::token::TokenInfo; use pangolin_core::user::{OAuthProvider, User, UserRole}; -use sqlx::sqlite::{SqlitePool, SqlitePoolOptions}; +use sqlx::sqlite::{SqliteConnectOptions, SqlitePool, SqlitePoolOptions}; use sqlx::Row; use uuid::Uuid; /// Version of `sql/sqlite_schema.sql` recorded after a successful migration. -pub const SQLITE_SCHEMA_VERSION: i64 = 1; +/// Bumped to 2: the `audit_logs` table was recreated with the columns the code +/// actually writes. Databases created before this carry the old (actor, +/// resource, details) shape, on which every audit write failed. +pub const SQLITE_SCHEMA_VERSION: i64 = 2; #[derive(Clone)] pub struct SqliteStore { @@ -49,14 +52,20 @@ impl SqliteStore { .and_then(|v| v.parse::().ok()) .unwrap_or(5); + // B18: `PRAGMA foreign_keys` is *per connection*. Running it once via + // `execute(&pool)` configured exactly one arbitrary connection out of + // the pool, so `ON DELETE CASCADE` fired or did not fire depending on + // which connection happened to serve a request - nondeterministically, + // and differently between runs. Setting it through the connect options + // means every connection in the pool is configured, including ones + // created later to grow the pool. + let connect_options = database_url + .parse::()? + .foreign_keys(true); + let pool = SqlitePoolOptions::new() .max_connections(max_connections) - .connect(database_url) - .await?; - - // Enable foreign keys - sqlx::query("PRAGMA foreign_keys = ON") - .execute(&pool) + .connect_with(connect_options) .await?; Ok(Self { @@ -84,6 +93,13 @@ impl SqliteStore { .execute(&self.pool) .await?; + // Structural changes have to run *before* the schema file, because + // `CREATE TABLE IF NOT EXISTS` cannot alter a table that already exists + // - it silently does nothing, which is precisely how bumping + // SQLITE_SCHEMA_VERSION on its own left every upgraded database still + // broken while fresh ones looked fine. + self.migrate_audit_logs_to_v2().await?; + let schema = include_str!("../../sql/sqlite_schema.sql"); self.apply_schema(schema).await?; @@ -103,6 +119,88 @@ impl SqliteStore { Ok(()) } + /// Replace a pre-v2 `audit_logs` table with the current shape. + /// + /// The table was declared with the original `(actor, resource, details)` + /// columns while `sqlite/audit_logs.rs` inserted the full `AuditLogEntry`. + /// The two have been divergent since the SQLite backend was split into + /// modules, so on SQLite *every* audit write has failed with "table + /// audit_logs has no column named user_id" and the backend has never + /// recorded an audit trail. + /// + /// Keyed off table introspection rather than the recorded schema version: + /// some databases predate the version table entirely, and a version number + /// only says what a previous run *claimed*. Asking the table what columns it + /// has is the fact. That also makes this safe to run repeatedly. + /// + /// Any rows present are moved to `audit_logs_pre_v2` rather than discarded + /// or force-fitted. The old shape has no `resource_type`, which is + /// `NOT NULL` and parses as an enum, so there is no honest value to invent + /// for it - and a fabricated entry in an audit log is worse than an absent + /// one. In practice the table is empty, because nothing could ever write + /// to it. + async fn migrate_audit_logs_to_v2(&self) -> Result<()> { + let columns = self.table_columns("audit_logs").await?; + + // No table yet: a fresh database. The schema file creates it. + if columns.is_empty() { + return Ok(()); + } + + // Already current. + if columns.iter().any(|c| c == "user_id") { + return Ok(()); + } + + let row_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM audit_logs") + .fetch_one(&self.pool) + .await + .unwrap_or(0); + + tracing::warn!( + rows = row_count, + "migrating the pre-v2 sqlite audit_logs table; the old table is kept \ + as audit_logs_pre_v2" + ); + + let mut tx = self.pool.begin().await?; + + // A stale backup from an interrupted previous attempt would make the + // rename fail, so clear the way first. + sqlx::query("DROP TABLE IF EXISTS audit_logs_pre_v2") + .execute(&mut *tx) + .await?; + sqlx::query("ALTER TABLE audit_logs RENAME TO audit_logs_pre_v2") + .execute(&mut *tx) + .await?; + + // The index followed the table through the rename and would collide + // with the one the schema file recreates. + sqlx::query("DROP INDEX IF EXISTS idx_audit_logs_tenant_ts") + .execute(&mut *tx) + .await?; + + tx.commit().await?; + Ok(()) + } + + /// Column names of `table`, or an empty vec if it does not exist. + /// + /// Exposed so the migration tests can assert on the *actual* shape of a + /// database rather than on what the recorded version claims - the two + /// disagreeing is exactly the bug the v2 migration exists to fix. + pub async fn table_columns(&self, table: &str) -> Result> { + // `table` is never caller-supplied in a production path, and PRAGMA does + // not accept a bind parameter for a table name. + let rows = sqlx::query(&format!("PRAGMA table_info({table})")) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| r.get::("name")) + .collect()) + } + /// The schema version currently recorded in the database, if any. pub async fn schema_version(&self) -> Result> { let row = sqlx::query("SELECT MAX(version) as v FROM _pangolin_schema_version") @@ -114,9 +212,16 @@ impl SqliteStore { } pub async fn apply_schema(&self, schema_sql: &str) -> Result<()> { + // Schema creation runs on a single dedicated connection so the + // foreign-key toggle below is scoped to *this* work rather than + // leaking onto whichever pooled connection happened to serve it + // (B18) - the old code could leave `OFF` stuck on a connection the + // matching `ON` never touched. + let mut conn = self.pool.acquire().await?; + // Disable foreign keys during schema creation sqlx::query("PRAGMA foreign_keys = OFF") - .execute(&self.pool) + .execute(&mut *conn) .await?; // Parse statements @@ -151,12 +256,13 @@ impl SqliteStore { } for statement in statements { - sqlx::query(&statement).execute(&self.pool).await?; + sqlx::query(&statement).execute(&mut *conn).await?; } - // Re-enable foreign keys + // Re-enable foreign keys on this connection before returning it to the + // pool. sqlx::query("PRAGMA foreign_keys = ON") - .execute(&self.pool) + .execute(&mut *conn) .await?; Ok(()) } @@ -327,6 +433,17 @@ impl CatalogStore for SqliteStore { .await } + async fn replace_namespace_properties( + &self, + tenant_id: Uuid, + catalog_name: &str, + namespace: Vec, + properties: std::collections::HashMap, + ) -> Result<()> { + self.replace_namespace_properties(tenant_id, catalog_name, namespace, properties) + .await + } + async fn create_asset( &self, tenant_id: Uuid, @@ -533,6 +650,14 @@ impl CatalogStore for SqliteStore { async fn write_file(&self, path: &str, data: Vec) -> Result<()> { self.write_file(path, data).await } + async fn delete_file(&self, path: &str) -> Result<()> { + self.metadata_cache.invalidate(path).await; + let storage_config = self + .get_warehouse_for_location(path) + .await? + .map(|w| w.storage_config); + crate::file_delete::delete_location(storage_config.as_ref(), path).await + } // Phase 3 & 4 (Access Control, Audit, Settings, etc) async fn create_user(&self, user: User) -> Result<()> { diff --git a/pangolin/pangolin_store/src/sqlite/merge_operations.rs b/pangolin/pangolin_store/src/sqlite/merge_operations.rs index 57b6bc3..92c6933 100644 --- a/pangolin/pangolin_store/src/sqlite/merge_operations.rs +++ b/pangolin/pangolin_store/src/sqlite/merge_operations.rs @@ -97,7 +97,7 @@ impl SqliteStore { let rows = sqlx::query( "SELECT id, tenant_id, catalog_name, source_branch, target_branch, base_commit_id, status, initiated_by, initiated_at, result_commit_id, completed_at - FROM merge_operations WHERE tenant_id = ? AND catalog_name = ? LIMIT ? OFFSET ?" + FROM merge_operations WHERE tenant_id = ? AND catalog_name = ? ORDER BY id LIMIT ? OFFSET ?" ) .bind(tenant_id.to_string()) .bind(catalog_name) @@ -259,7 +259,7 @@ impl SqliteStore { let rows = sqlx::query( "SELECT id, operation_id, conflict_type, asset_id, description, resolution, created_at - FROM merge_conflicts WHERE operation_id = ? LIMIT ? OFFSET ?", + FROM merge_conflicts WHERE operation_id = ? ORDER BY id LIMIT ? OFFSET ?", ) .bind(operation_id.to_string()) .bind(limit) diff --git a/pangolin/pangolin_store/src/sqlite/mod.rs b/pangolin/pangolin_store/src/sqlite/mod.rs index 68435fa..04f916d 100644 --- a/pangolin/pangolin_store/src/sqlite/mod.rs +++ b/pangolin/pangolin_store/src/sqlite/mod.rs @@ -23,5 +23,5 @@ pub mod users; pub mod warehouses; // Re-export SqliteStore -pub use main::SqliteStore; +pub use main::{SqliteStore, SQLITE_SCHEMA_VERSION}; pub mod business_metadata; diff --git a/pangolin/pangolin_store/src/sqlite/namespaces.rs b/pangolin/pangolin_store/src/sqlite/namespaces.rs index d9f9d09..c70742c 100644 --- a/pangolin/pangolin_store/src/sqlite/namespaces.rs +++ b/pangolin/pangolin_store/src/sqlite/namespaces.rs @@ -63,7 +63,7 @@ impl SqliteStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT namespace_path, properties FROM namespaces WHERE tenant_id = ? AND catalog_name = ? LIMIT ? OFFSET ?") + let rows = sqlx::query("SELECT namespace_path, properties FROM namespaces WHERE tenant_id = ? AND catalog_name = ? ORDER BY namespace_path LIMIT ? OFFSET ?") .bind(tenant_id.to_string()) .bind(catalog_name) .bind(limit) @@ -120,4 +120,32 @@ impl SqliteStore { } Ok(()) } + + /// Replace a namespace's properties wholesale. + /// + /// `update_namespace_properties` merges, which cannot express a *removal*. + /// The Iceberg `updateProperties` endpoint takes both `updates` and + /// `removals`, and the handler used to ignore removals entirely while + /// reporting success (B16h); it now computes the resulting map and writes it + /// through here. + pub async fn replace_namespace_properties( + &self, + tenant_id: Uuid, + catalog_name: &str, + namespace: Vec, + properties: HashMap, + ) -> Result<()> { + let namespace_path = serde_json::to_string(&namespace)?; + let result = sqlx::query("UPDATE namespaces SET properties = ? WHERE tenant_id = ? AND catalog_name = ? AND namespace_path = ?") + .bind(serde_json::to_string(&properties)?) + .bind(tenant_id.to_string()) + .bind(catalog_name) + .bind(&namespace_path) + .execute(&self.pool) + .await?; + if result.rows_affected() == 0 { + return Err(anyhow::anyhow!("Namespace not found")); + } + Ok(()) + } } diff --git a/pangolin/pangolin_store/src/sqlite/permissions.rs b/pangolin/pangolin_store/src/sqlite/permissions.rs index a9a7d0c..a7d70b3 100644 --- a/pangolin/pangolin_store/src/sqlite/permissions.rs +++ b/pangolin/pangolin_store/src/sqlite/permissions.rs @@ -114,7 +114,7 @@ impl SqliteStore { "SELECT id, user_id, tenant_id, scope, actions, granted_by, granted_at FROM permissions WHERE tenant_id = ? - LIMIT ? OFFSET ?", + ORDER BY id LIMIT ? OFFSET ?", ) .bind(tenant_id.to_string()) .bind(limit) diff --git a/pangolin/pangolin_store/src/sqlite/roles.rs b/pangolin/pangolin_store/src/sqlite/roles.rs index 868eb0f..376abad 100644 --- a/pangolin/pangolin_store/src/sqlite/roles.rs +++ b/pangolin/pangolin_store/src/sqlite/roles.rs @@ -58,7 +58,7 @@ impl SqliteStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT id, tenant_id, name, description, permissions, created_by, created_at, updated_at FROM roles WHERE tenant_id = ? LIMIT ? OFFSET ?") + let rows = sqlx::query("SELECT id, tenant_id, name, description, permissions, created_by, created_at, updated_at FROM roles WHERE tenant_id = ? ORDER BY name LIMIT ? OFFSET ?") .bind(tenant_id.to_string()) .bind(limit) .bind(offset) diff --git a/pangolin/pangolin_store/src/sqlite/service_users.rs b/pangolin/pangolin_store/src/sqlite/service_users.rs index d92f5ad..b745500 100644 --- a/pangolin/pangolin_store/src/sqlite/service_users.rs +++ b/pangolin/pangolin_store/src/sqlite/service_users.rs @@ -128,7 +128,7 @@ impl SqliteStore { let rows = sqlx::query( "SELECT id, name, description, tenant_id, api_key_hash, role, created_at, created_by, last_used, expires_at, active - FROM service_users WHERE tenant_id = ? LIMIT ? OFFSET ?" + FROM service_users WHERE tenant_id = ? ORDER BY name LIMIT ? OFFSET ?" ) .bind(tenant_id.to_string()) .bind(limit) diff --git a/pangolin/pangolin_store/src/sqlite/tags.rs b/pangolin/pangolin_store/src/sqlite/tags.rs index 346081d..906376b 100644 --- a/pangolin/pangolin_store/src/sqlite/tags.rs +++ b/pangolin/pangolin_store/src/sqlite/tags.rs @@ -55,7 +55,7 @@ impl SqliteStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT name, commit_id FROM tags WHERE tenant_id = ? AND catalog_name = ? LIMIT ? OFFSET ?") + let rows = sqlx::query("SELECT name, commit_id FROM tags WHERE tenant_id = ? AND catalog_name = ? ORDER BY name LIMIT ? OFFSET ?") .bind(tenant_id.to_string()) .bind(catalog_name) .bind(limit) diff --git a/pangolin/pangolin_store/src/sqlite/tenants.rs b/pangolin/pangolin_store/src/sqlite/tenants.rs index 0188c91..83995de 100644 --- a/pangolin/pangolin_store/src/sqlite/tenants.rs +++ b/pangolin/pangolin_store/src/sqlite/tenants.rs @@ -44,11 +44,12 @@ impl SqliteStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT id, name, properties FROM tenants LIMIT ? OFFSET ?") - .bind(limit) - .bind(offset) - .fetch_all(&self.pool) - .await?; + let rows = + sqlx::query("SELECT id, name, properties FROM tenants ORDER BY name LIMIT ? OFFSET ?") + .bind(limit) + .bind(offset) + .fetch_all(&self.pool) + .await?; let mut tenants = Vec::new(); for row in rows { diff --git a/pangolin/pangolin_store/src/sqlite/tokens.rs b/pangolin/pangolin_store/src/sqlite/tokens.rs index f091e98..67e2ffd 100644 --- a/pangolin/pangolin_store/src/sqlite/tokens.rs +++ b/pangolin/pangolin_store/src/sqlite/tokens.rs @@ -7,6 +7,56 @@ use sqlx::Row; use uuid::Uuid; impl SqliteStore { + /// Record a token revocation. + /// + /// Found by the cross-backend parity suite: `SqliteStore` had **no** + /// inherent `revoke_token`, `is_token_revoked` or `cleanup_expired_tokens`, + /// so the trait implementations in `sqlite/main.rs` - written as + /// `self.revoke_token(..)` in the style of every other delegation in that + /// file - resolved to the *trait* method and called themselves. On SQLite, + /// revoking a token (i.e. logging out) recursed until the thread's stack was + /// exhausted and aborted the process: an unauthenticated-adjacent remote + /// crash, not merely a missing feature. + /// + /// The `revoked_tokens` table has existed in the schema since A-27; only the + /// code to use it was missing. + pub async fn revoke_token( + &self, + token_id: Uuid, + expires_at: chrono::DateTime, + reason: Option, + ) -> Result<()> { + sqlx::query( + "INSERT INTO revoked_tokens (token_id, expires_at, reason) VALUES (?, ?, ?) + ON CONFLICT(token_id) DO UPDATE SET + expires_at = excluded.expires_at, + reason = excluded.reason", + ) + .bind(token_id.to_string()) + .bind(expires_at.timestamp_millis()) + .bind(reason) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn is_token_revoked(&self, token_id: Uuid) -> Result { + let row: Option<(i64,)> = sqlx::query_as("SELECT 1 FROM revoked_tokens WHERE token_id = ?") + .bind(token_id.to_string()) + .fetch_optional(&self.pool) + .await?; + Ok(row.is_some()) + } + + /// Drop revocation records whose tokens have expired anyway. + pub async fn cleanup_expired_tokens(&self) -> Result { + let result = sqlx::query("DELETE FROM revoked_tokens WHERE expires_at < ?") + .bind(Utc::now().timestamp_millis()) + .execute(&self.pool) + .await?; + Ok(result.rows_affected() as usize) + } + pub async fn store_token(&self, token_info: TokenInfo) -> Result<()> { sqlx::query("INSERT INTO active_tokens (token_id, user_id, tenant_id, token, expires_at, created_at) VALUES (?, ?, ?, ?, ?, ?)") .bind(token_info.id.to_string()) @@ -76,7 +126,7 @@ impl SqliteStore { .unwrap_or(0); let rows = if let Some(uid) = user_id { - sqlx::query("SELECT token_id, user_id, tenant_id, token, expires_at FROM active_tokens WHERE tenant_id = ? AND user_id = ? AND expires_at > ? LIMIT ? OFFSET ?") + sqlx::query("SELECT token_id, user_id, tenant_id, token, expires_at FROM active_tokens WHERE tenant_id = ? AND user_id = ? AND expires_at > ? ORDER BY expires_at DESC, token_id LIMIT ? OFFSET ?") .bind(tenant_id.to_string()) .bind(uid.to_string()) .bind(Utc::now().timestamp()) @@ -85,7 +135,7 @@ impl SqliteStore { .fetch_all(&self.pool) .await? } else { - sqlx::query("SELECT token_id, user_id, tenant_id, token, expires_at FROM active_tokens WHERE tenant_id = ? AND expires_at > ? LIMIT ? OFFSET ?") + sqlx::query("SELECT token_id, user_id, tenant_id, token, expires_at FROM active_tokens WHERE tenant_id = ? AND expires_at > ? ORDER BY expires_at DESC, token_id LIMIT ? OFFSET ?") .bind(tenant_id.to_string()) .bind(Utc::now().timestamp()) .bind(limit) diff --git a/pangolin/pangolin_store/src/sqlite/users.rs b/pangolin/pangolin_store/src/sqlite/users.rs index 8164d41..013fd3e 100644 --- a/pangolin/pangolin_store/src/sqlite/users.rs +++ b/pangolin/pangolin_store/src/sqlite/users.rs @@ -56,14 +56,14 @@ impl SqliteStore { .unwrap_or(0); let rows = if let Some(tid) = tenant_id { - sqlx::query("SELECT id, username, email, password_hash, oauth_provider, oauth_subject, tenant_id, role, created_at, updated_at, last_login, active FROM users WHERE tenant_id = ? LIMIT ? OFFSET ?") + sqlx::query("SELECT id, username, email, password_hash, oauth_provider, oauth_subject, tenant_id, role, created_at, updated_at, last_login, active FROM users WHERE tenant_id = ? ORDER BY username LIMIT ? OFFSET ?") .bind(tid.to_string()) .bind(limit) .bind(offset) .fetch_all(&self.pool) .await? } else { - sqlx::query("SELECT id, username, email, password_hash, oauth_provider, oauth_subject, tenant_id, role, created_at, updated_at, last_login, active FROM users LIMIT ? OFFSET ?") + sqlx::query("SELECT id, username, email, password_hash, oauth_provider, oauth_subject, tenant_id, role, created_at, updated_at, last_login, active FROM users ORDER BY username LIMIT ? OFFSET ?") .bind(limit) .bind(offset) .fetch_all(&self.pool) diff --git a/pangolin/pangolin_store/src/sqlite/warehouses.rs b/pangolin/pangolin_store/src/sqlite/warehouses.rs index 78aa470..78f7ff6 100644 --- a/pangolin/pangolin_store/src/sqlite/warehouses.rs +++ b/pangolin/pangolin_store/src/sqlite/warehouses.rs @@ -55,7 +55,7 @@ impl SqliteStore { .map(|p| p.offset.unwrap_or(0) as i64) .unwrap_or(0); - let rows = sqlx::query("SELECT id, name, use_sts, storage_config, vending_strategy FROM warehouses WHERE tenant_id = ? LIMIT ? OFFSET ?") + let rows = sqlx::query("SELECT id, name, use_sts, storage_config, vending_strategy FROM warehouses WHERE tenant_id = ? ORDER BY name LIMIT ? OFFSET ?") .bind(tenant_id.to_string()) .bind(limit) .bind(offset) diff --git a/pangolin/pangolin_store/src/tests/mod.rs b/pangolin/pangolin_store/src/tests/mod.rs index bceed31..f06b9a3 100644 --- a/pangolin/pangolin_store/src/tests/mod.rs +++ b/pangolin/pangolin_store/src/tests/mod.rs @@ -273,6 +273,8 @@ pub async fn test_dashboard_stats_consistency(store: &S) { pub mod audit_tests; #[cfg(test)] pub mod multi_cloud; +/// Cross-backend parity suite (roadmap improvement #1). +pub mod parity; #[cfg(test)] pub mod postgres_merge_tests; #[cfg(test)] diff --git a/pangolin/pangolin_store/src/tests/parity.rs b/pangolin/pangolin_store/src/tests/parity.rs new file mode 100644 index 0000000..d08c6e3 --- /dev/null +++ b/pangolin/pangolin_store/src/tests/parity.rs @@ -0,0 +1,808 @@ +//! Cross-backend parity suite. +//! +//! Roadmap improvement #1. Nearly half the storage-layer findings in the +//! August audit (B1-B7, B17-B30) were one backend silently diverging from the +//! other three: a tenant filter dropped on Mongo, a CAS skipped in memory, an +//! enum round-tripping to the wrong variant on all the SQL backends, pagination +//! that repeats or skips rows, four different answers to the same search. +//! +//! Each of those is invisible to a per-backend test, because per-backend tests +//! assert what that backend does. What catches them is asserting that all four +//! backends do the *same* thing. Every function here runs against whichever +//! `CatalogStore` it is handed, and `tests/store_integration.rs` runs the whole +//! set against memory, SQLite, Postgres and Mongo. +//! +//! Each assertion names the finding it locks down, so a regression points +//! straight at what it broke. + +use crate::CatalogStore; +use pangolin_core::business_metadata::BusinessMetadata; +use pangolin_core::model::*; +use std::collections::HashMap; +use uuid::Uuid; + +/// Build the tenant -> warehouse -> catalog -> namespace chain the SQL backends' +/// foreign keys require, and return the tenant id. +async fn seed_hierarchy(store: &S, catalog: &str, namespace: &[String]) -> Uuid { + let tenant_id = Uuid::new_v4(); + + store + .create_tenant(Tenant { + id: tenant_id, + name: format!("parity_{tenant_id}"), + properties: HashMap::new(), + }) + .await + .expect("create tenant"); + + let _ = store + .create_warehouse( + tenant_id, + Warehouse { + id: Uuid::new_v4(), + name: "wh".to_string(), + tenant_id, + storage_config: HashMap::new(), + use_sts: false, + vending_strategy: None, + }, + ) + .await; + + store + .create_catalog( + tenant_id, + Catalog { + id: Uuid::new_v4(), + name: catalog.to_string(), + catalog_type: CatalogType::Local, + warehouse_name: Some("wh".to_string()), + storage_location: None, + federated_config: None, + properties: HashMap::new(), + }, + ) + .await + .expect("create catalog"); + + store + .create_namespace( + tenant_id, + catalog, + Namespace { + name: namespace.to_vec(), + properties: HashMap::new(), + }, + ) + .await + .expect("create namespace"); + + tenant_id +} + +fn asset(name: &str, kind: AssetType) -> Asset { + Asset { + id: Uuid::new_v4(), + name: name.to_string(), + kind, + location: format!("s3://bucket/{name}"), + properties: HashMap::new(), + } +} + +/// **B7.** Every `AssetType` variant must survive a write/read round trip. +/// +/// All three persistent backends stored the Debug spelling and parsed only +/// `IcebergTable`/`View`, defaulting the other 15 variants to `IcebergTable` - +/// so a `DeltaTable` came back as an Iceberg table with no error anywhere. +pub async fn asset_types_round_trip(store: &S) { + let catalog = "types"; + let namespace = vec!["ns".to_string()]; + let tenant_id = seed_hierarchy(store, catalog, &namespace).await; + + for kind in AssetType::all() { + let name = format!("asset_{}", kind.as_stored_str().to_lowercase()); + let written = asset(&name, kind.clone()); + + store + .create_asset(tenant_id, catalog, None, namespace.clone(), written.clone()) + .await + .expect("create asset"); + + let read = store + .get_asset(tenant_id, catalog, None, namespace.clone(), name.clone()) + .await + .expect("get asset") + .unwrap_or_else(|| panic!("asset {name} vanished")); + + assert_eq!( + read.kind, kind, + "asset type {kind:?} did not round-trip (B7); it came back as {:?}", + read.kind + ); + } +} + +/// **B27.** Two pages must cover the set exactly once. +/// +/// Every paginated query outside three Postgres call sites ran `LIMIT/OFFSET` +/// with no `ORDER BY`, and the memory backend paged over DashMap iteration +/// order. Both can repeat or skip a row between pages, which is invisible until +/// a client silently misses data. +pub async fn pagination_covers_the_set_exactly_once(store: &S) { + let catalog = "paging"; + let namespace = vec!["ns".to_string()]; + let tenant_id = seed_hierarchy(store, catalog, &namespace).await; + + const TOTAL: usize = 7; + const PAGE: usize = 3; + + for i in 0..TOTAL { + store + .create_asset( + tenant_id, + catalog, + None, + namespace.clone(), + // Zero-padded so lexical and numeric order agree, and the + // assertion is about paging rather than about collation. + asset(&format!("t{i:03}"), AssetType::IcebergTable), + ) + .await + .expect("create asset"); + } + + let mut seen: Vec = Vec::new(); + let mut offset = 0; + loop { + let page = store + .list_assets( + tenant_id, + catalog, + None, + namespace.clone(), + Some(crate::PaginationParams { + limit: Some(PAGE), + offset: Some(offset), + }), + ) + .await + .expect("list assets"); + + if page.is_empty() { + break; + } + seen.extend(page.iter().map(|a| a.name.clone())); + offset += PAGE; + + // Guard against a backend that ignores offset and loops forever. + assert!(offset <= TOTAL * 4, "pagination did not terminate"); + } + + let mut unique = seen.clone(); + unique.sort(); + unique.dedup(); + + assert_eq!( + unique.len(), + seen.len(), + "paging returned a duplicate row (B27): {seen:?}" + ); + assert_eq!( + unique.len(), + TOTAL, + "paging skipped a row (B27): saw {} of {TOTAL}", + unique.len() + ); +} + +/// **B27.** Repeating the same listing must return the same order. +pub async fn listing_order_is_stable(store: &S) { + let catalog = "stable"; + let namespace = vec!["ns".to_string()]; + let tenant_id = seed_hierarchy(store, catalog, &namespace).await; + + for name in ["delta", "alpha", "charlie", "bravo"] { + store + .create_asset( + tenant_id, + catalog, + None, + namespace.clone(), + asset(name, AssetType::IcebergTable), + ) + .await + .expect("create asset"); + } + + let first: Vec = store + .list_assets(tenant_id, catalog, None, namespace.clone(), None) + .await + .expect("list assets") + .into_iter() + .map(|a| a.name) + .collect(); + + for _ in 0..3 { + let again: Vec = store + .list_assets(tenant_id, catalog, None, namespace.clone(), None) + .await + .expect("list assets") + .into_iter() + .map(|a| a.name) + .collect(); + assert_eq!(again, first, "listing order is not stable (B27)"); + } +} + +/// **B2 / B0j.** A revoked token must read back as revoked. +/// +/// On Mongo the revocation write and the revocation *check* used different +/// field names and different types, so the check could never match: revocation +/// - including logout - was a silent no-op. +pub async fn revocation_round_trips(store: &S) { + let token_id = Uuid::new_v4(); + let expires_at = chrono::Utc::now() + chrono::Duration::hours(1); + + assert!( + !store + .is_token_revoked(token_id) + .await + .expect("check revocation"), + "a fresh token must not be revoked" + ); + + store + .revoke_token(token_id, expires_at, Some("parity test".to_string())) + .await + .expect("revoke token"); + + assert!( + store + .is_token_revoked(token_id) + .await + .expect("check revocation"), + "a revoked token must read back as revoked (B2)" + ); +} + +/// **B1.** An audit event must not be readable from another tenant. +pub async fn audit_events_are_tenant_scoped(store: &S) { + let owner = Uuid::new_v4(); + let stranger = Uuid::new_v4(); + + for tenant in [owner, stranger] { + store + .create_tenant(Tenant { + id: tenant, + name: format!("audit_{tenant}"), + properties: HashMap::new(), + }) + .await + .expect("create tenant"); + } + + let entry = pangolin_core::audit::AuditLogEntry::success( + owner, + Some(Uuid::new_v4()), + "owner".to_string(), + pangolin_core::audit::AuditAction::CreateCatalog, + pangolin_core::audit::ResourceType::Catalog, + Some(Uuid::new_v4()), + "secret_catalog".to_string(), + ); + let event_id = entry.id; + + store + .log_audit_event(owner, entry) + .await + .expect("log audit event"); + + assert!( + store + .get_audit_event(owner, event_id) + .await + .expect("get audit event") + .is_some(), + "the owning tenant must be able to read its own audit event" + ); + + assert!( + store + .get_audit_event(stranger, event_id) + .await + .expect("get audit event") + .is_none(), + "an audit event must not be readable across tenants (B1)" + ); +} + +/// **B22 / B23.** Audit actions must round-trip, not collapse to a default. +/// +/// SQLite persisted the Debug spelling and parsed snake_case, then swallowed the +/// mismatch with `unwrap_or(CreateCatalog)`, so nearly every multi-word action +/// was misattributed. Mongo's filters had the mirror-image problem and always +/// matched zero rows. +pub async fn audit_actions_round_trip(store: &S) { + use pangolin_core::audit::{AuditAction, AuditLogEntry, ResourceType}; + + let tenant_id = Uuid::new_v4(); + store + .create_tenant(Tenant { + id: tenant_id, + name: format!("audit_actions_{tenant_id}"), + properties: HashMap::new(), + }) + .await + .expect("create tenant"); + + // Deliberately multi-word: the single-word ones happened to survive. + let actions = [ + AuditAction::CreateBranch, + AuditAction::DeleteNamespace, + AuditAction::CommitTable, + ]; + + for action in &actions { + store + .log_audit_event( + tenant_id, + AuditLogEntry::success( + tenant_id, + Some(Uuid::new_v4()), + "auditor".to_string(), + action.clone(), + ResourceType::Table, + Some(Uuid::new_v4()), + format!("{action:?}_target"), + ), + ) + .await + .expect("log audit event"); + } + + let events = store + .list_audit_events(tenant_id, None) + .await + .expect("list audit events"); + + for action in &actions { + assert!( + events.iter().any(|e| e.action == *action), + "audit action {action:?} did not round-trip (B22); \ + the listing held {:?}", + events.iter().map(|e| &e.action).collect::>() + ); + } +} + +/// **B28.** A search term containing LIKE metacharacters must be literal. +pub async fn search_treats_wildcards_literally(store: &S) { + let catalog = "search"; + let namespace = vec!["ns".to_string()]; + let tenant_id = seed_hierarchy(store, catalog, &namespace).await; + + for name in ["margin_100pct", "unrelated_table"] { + store + .create_asset( + tenant_id, + catalog, + None, + namespace.clone(), + asset(name, AssetType::IcebergTable), + ) + .await + .expect("create asset"); + } + + // `%` is a LIKE wildcard. Unescaped, this matched everything. + let hits = store + .search_assets(tenant_id, "%", None) + .await + .expect("search assets"); + + assert!( + hits.is_empty(), + "a literal '%' matched {} assets (B28); the term was treated as a wildcard", + hits.len() + ); + + // A genuine substring still matches. + let hits = store + .search_assets(tenant_id, "margin", None) + .await + .expect("search assets"); + assert_eq!( + hits.len(), + 1, + "an ordinary substring search should still find its asset" + ); +} + +/// **B28.** Tag filtering is ALL-match, and an empty filter is no filter. +pub async fn tag_filter_semantics_agree(store: &S) { + let catalog = "tags"; + let namespace = vec!["ns".to_string()]; + let tenant_id = seed_hierarchy(store, catalog, &namespace).await; + + let both = asset("tagged_both", AssetType::IcebergTable); + let one = asset("tagged_one", AssetType::IcebergTable); + + for a in [&both, &one] { + store + .create_asset(tenant_id, catalog, None, namespace.clone(), a.clone()) + .await + .expect("create asset"); + } + + let mut meta_both = BusinessMetadata::new(both.id, Uuid::new_v4()); + meta_both.tags = vec!["pii".to_string(), "finance".to_string()]; + store + .upsert_business_metadata(meta_both) + .await + .expect("upsert metadata"); + + let mut meta_one = BusinessMetadata::new(one.id, Uuid::new_v4()); + meta_one.tags = vec!["pii".to_string()]; + store + .upsert_business_metadata(meta_one) + .await + .expect("upsert metadata"); + + // One tag: both assets carry it. + let hits = store + .search_assets(tenant_id, "tagged", Some(vec!["pii".to_string()])) + .await + .expect("search assets"); + assert_eq!(hits.len(), 2, "single-tag filter should match both assets"); + + // Two tags: ALL-match, so only the asset carrying both. + let hits = store + .search_assets( + tenant_id, + "tagged", + Some(vec!["pii".to_string(), "finance".to_string()]), + ) + .await + .expect("search assets"); + assert_eq!( + hits.len(), + 1, + "the tag filter must be ALL-match (B28): every requested tag has to be present" + ); + + // An empty filter is not a filter. + let hits = store + .search_assets(tenant_id, "tagged", Some(vec![])) + .await + .expect("search assets"); + assert_eq!( + hits.len(), + 2, + "an empty tag list must mean 'no tag filter' (B28)" + ); +} + +/// **B26 / B5.** The compare-and-swap must actually compare. +/// +/// Mongo ignored `expected_location` entirely, so two concurrent commits both +/// "succeeded" and one snapshot was lost; memory skipped the check whenever the +/// expectation was `None`. +pub async fn metadata_cas_rejects_a_stale_writer(store: &S) { + let catalog = "cas"; + let namespace = vec!["ns".to_string()]; + let tenant_id = seed_hierarchy(store, catalog, &namespace).await; + + let table = "committed".to_string(); + let v1 = "s3://bucket/cas/v1.json".to_string(); + let mut a = asset(&table, AssetType::IcebergTable); + a.location = v1.clone(); + a.properties + .insert("metadata_location".to_string(), v1.clone()); + + store + .create_asset(tenant_id, catalog, None, namespace.clone(), a) + .await + .expect("create asset"); + + // Writer A wins. + let v2 = "s3://bucket/cas/v2.json".to_string(); + store + .update_metadata_location( + tenant_id, + catalog, + None, + namespace.clone(), + table.clone(), + Some(v1.clone()), + v2.clone(), + ) + .await + .expect("the first writer's CAS should succeed"); + + // Writer B still believes the table is at v1: it must be refused. + let v3 = "s3://bucket/cas/v3.json".to_string(); + let stale = store + .update_metadata_location( + tenant_id, + catalog, + None, + namespace.clone(), + table.clone(), + Some(v1.clone()), + v3, + ) + .await; + + assert!( + stale.is_err(), + "a stale writer's CAS must fail (B5/B26); it silently overwrote the winner" + ); + + let current = store + .get_metadata_location(tenant_id, catalog, None, namespace, table) + .await + .expect("get metadata location"); + assert_eq!( + current, + Some(v2), + "the losing writer must not have changed the published metadata" + ); +} + +/// **B6 / B30.** Deleting one tenant's data must not disturb another's. +/// +/// Catalog names are per-tenant, but the memory backend's by-id asset index was +/// keyed on catalog name alone, so deleting tenant A's `sales` broke +/// `get_asset_by_id` for tenant B's unrelated `sales`. +pub async fn deleting_a_catalog_does_not_touch_another_tenant(store: &S) { + let catalog = "sales"; + let namespace = vec!["ns".to_string()]; + + let tenant_a = seed_hierarchy(store, catalog, &namespace).await; + let tenant_b = seed_hierarchy(store, catalog, &namespace).await; + + let a_asset = asset("orders", AssetType::IcebergTable); + let b_asset = asset("orders", AssetType::IcebergTable); + + store + .create_asset(tenant_a, catalog, None, namespace.clone(), a_asset.clone()) + .await + .expect("create asset"); + store + .create_asset(tenant_b, catalog, None, namespace.clone(), b_asset.clone()) + .await + .expect("create asset"); + + store + .delete_catalog(tenant_a, catalog.to_string()) + .await + .expect("delete catalog"); + + let survivor = store + .get_asset_by_id(tenant_b, b_asset.id) + .await + .expect("get asset by id"); + + assert!( + survivor.is_some(), + "deleting tenant A's catalog must not evict tenant B's identically-named \ + catalog from the by-id index (B6)" + ); +} + +/// **B21.** Deleting a catalog that does not exist must destroy nothing. +pub async fn deleting_a_missing_catalog_destroys_nothing(store: &S) { + let catalog = "present"; + let namespace = vec!["ns".to_string()]; + let tenant_id = seed_hierarchy(store, catalog, &namespace).await; + + let kept = asset("keep_me", AssetType::IcebergTable); + store + .create_asset(tenant_id, catalog, None, namespace.clone(), kept.clone()) + .await + .expect("create asset"); + + // A namespace under a catalog name that has no catalog row. + // + // This is the state a cascade-before-existence-check destroys, and it is + // reachable: `namespaces` carries a foreign key to `tenants` only, on every + // backend. (An orphaned *asset* is not portable - Postgres has a composite + // key from assets to namespaces, so it refuses one. That strictness is + // worth knowing about; it is also why this uses the child row that every + // backend does allow to be orphaned.) + // + // An earlier version of this test named a catalog with no children at all, + // which passes whatever the ordering is. It is worth being precise about + // what a regression test actually constrains. + let orphan_ns = vec!["orphaned".to_string()]; + let orphan_created = store + .create_namespace( + tenant_id, + "absent", + Namespace { + name: orphan_ns.clone(), + properties: HashMap::new(), + }, + ) + .await + .is_ok(); + + let result = store.delete_catalog(tenant_id, "absent".to_string()).await; + assert!( + result.is_err(), + "deleting a nonexistent catalog should be an error" + ); + + if orphan_created { + // The backend permits orphaned children, so the ordering is observable: + // a cascade that runs before the existence check destroys them and only + // then reports "not found". + let orphan_survived = store + .get_namespace(tenant_id, "absent", orphan_ns) + .await + .expect("get orphaned namespace"); + assert!( + orphan_survived.is_some(), + "a failed delete_catalog destroyed the rows it matched before \ + discovering the catalog did not exist (B21)" + ); + } + // Otherwise the backend enforces referential integrity between namespaces + // and catalogs and the orphaned state is unreachable, so there is nothing + // for a mis-ordered cascade to destroy. Postgres is the one that does; the + // deliberate asymmetry is recorded rather than papered over, because + // "Postgres refuses to create the fixture" and "Postgres passed the + // assertion" are very different facts. + + let still_there = store + .get_asset(tenant_id, catalog, None, namespace, kept.name.clone()) + .await + .expect("get asset"); + assert!( + still_there.is_some(), + "a failed delete_catalog must not have destroyed another catalog's assets (B21)" + ); +} + +/// **B16h.** Namespace property removals must actually remove. +pub async fn namespace_property_replacement_removes_keys(store: &S) { + let catalog = "props"; + let namespace = vec!["ns".to_string()]; + let tenant_id = seed_hierarchy(store, catalog, &namespace).await; + + let mut properties = HashMap::new(); + properties.insert("keep".to_string(), "yes".to_string()); + properties.insert("drop".to_string(), "please".to_string()); + + store + .update_namespace_properties(tenant_id, catalog, namespace.clone(), properties) + .await + .expect("seed properties"); + + let mut remaining = HashMap::new(); + remaining.insert("keep".to_string(), "yes".to_string()); + + store + .replace_namespace_properties(tenant_id, catalog, namespace.clone(), remaining) + .await + .expect("replace properties"); + + let ns = store + .get_namespace(tenant_id, catalog, namespace) + .await + .expect("get namespace") + .expect("namespace should exist"); + + assert_eq!(ns.properties.get("keep").map(String::as_str), Some("yes")); + assert!( + !ns.properties.contains_key("drop"), + "replace_namespace_properties must drop keys the caller left out (B16h)" + ); +} + +/// **B17 / B19.** A multi-level namespace must be usable, not just creatable. +pub async fn nested_namespaces_are_addressable(store: &S) { + let catalog = "nested"; + let namespace = vec!["outer".to_string(), "inner".to_string()]; + let tenant_id = seed_hierarchy(store, catalog, &namespace).await; + + let found = store + .get_namespace(tenant_id, catalog, namespace.clone()) + .await + .expect("get namespace"); + assert!( + found.is_some(), + "a nested namespace must be retrievable by its full path (B17)" + ); + + let mut properties = HashMap::new(); + properties.insert("level".to_string(), "two".to_string()); + store + .update_namespace_properties(tenant_id, catalog, namespace.clone(), properties) + .await + .expect("a nested namespace must be updatable (B17)"); + + store + .delete_namespace(tenant_id, catalog, namespace.clone()) + .await + .expect("a nested namespace must be deletable (B17)"); + + assert!( + store + .get_namespace(tenant_id, catalog, namespace) + .await + .expect("get namespace") + .is_none(), + "the namespace should be gone after delete" + ); +} + +/// **B24.** A federated catalog must still look federated in a *listing*. +pub async fn catalog_type_survives_a_listing(store: &S) { + let tenant_id = Uuid::new_v4(); + store + .create_tenant(Tenant { + id: tenant_id, + name: format!("fed_{tenant_id}"), + properties: HashMap::new(), + }) + .await + .expect("create tenant"); + + store + .create_catalog( + tenant_id, + Catalog { + id: Uuid::new_v4(), + name: "remote".to_string(), + catalog_type: CatalogType::Federated, + warehouse_name: None, + storage_location: None, + federated_config: Some(FederatedCatalogConfig { + properties: HashMap::new(), + }), + properties: HashMap::new(), + }, + ) + .await + .expect("create catalog"); + + let listed = store + .list_catalogs(tenant_id, None) + .await + .expect("list catalogs"); + + let remote = listed + .iter() + .find(|c| c.name == "remote") + .expect("the federated catalog should be listed"); + + assert_eq!( + remote.catalog_type, + CatalogType::Federated, + "list_catalogs must report the real catalog type (B24); \ + Postgres used to hardcode Local" + ); +} + +/// Run the whole parity suite against one backend. +/// +/// `tests/store_integration.rs` calls this for each of the four, which is what +/// turns "this backend behaves like this" into "all four behave alike". +pub async fn run_all(store: &S) { + asset_types_round_trip(store).await; + pagination_covers_the_set_exactly_once(store).await; + listing_order_is_stable(store).await; + revocation_round_trips(store).await; + audit_events_are_tenant_scoped(store).await; + audit_actions_round_trip(store).await; + search_treats_wildcards_literally(store).await; + tag_filter_semantics_agree(store).await; + metadata_cas_rejects_a_stale_writer(store).await; + deleting_a_catalog_does_not_touch_another_tenant(store).await; + deleting_a_missing_catalog_destroys_nothing(store).await; + namespace_property_replacement_removes_keys(store).await; + nested_namespaces_are_addressable(store).await; + catalog_type_survives_a_listing(store).await; +} diff --git a/pangolin/pangolin_store/tests/mongo_audit_tests.rs b/pangolin/pangolin_store/tests/mongo_audit_tests.rs index 477890e..8f1af15 100644 --- a/pangolin/pangolin_store/tests/mongo_audit_tests.rs +++ b/pangolin/pangolin_store/tests/mongo_audit_tests.rs @@ -196,11 +196,21 @@ async fn test_mongo_audit_log_filtering() { // Test 10: Get individual event let event_id = logs[0].id; - let event = store.get_audit_event(event_id).await.unwrap(); + let event = store.get_audit_event(tenant_id, event_id).await.unwrap(); assert!(event.is_some(), "Should find the event"); assert_eq!(event.unwrap().id, event_id); println!("✓ Test 10 passed: Get individual event"); + // B1 regression: an event must not be readable from another tenant, even + // by someone who knows its UUID. + let other_tenant = Uuid::new_v4(); + let leaked = store.get_audit_event(other_tenant, event_id).await.unwrap(); + assert!( + leaked.is_none(), + "an audit event must not be readable across tenants (B1)" + ); + println!("✓ Test 10b passed: Audit events are tenant-scoped"); + println!("\n✅ All MongoDB audit logging tests passed!"); } diff --git a/pangolin/pangolin_store/tests/mongo_uuid_round_trip_tests.rs b/pangolin/pangolin_store/tests/mongo_uuid_round_trip_tests.rs new file mode 100644 index 0000000..d42f8d7 --- /dev/null +++ b/pangolin/pangolin_store/tests/mongo_uuid_round_trip_tests.rs @@ -0,0 +1,969 @@ +//! Every MongoDB entity must survive a write/read round trip. +//! +//! There are three ways this codebase turns a `Uuid` into BSON, and they do not +//! agree: +//! +//! | route | produces | +//! |---|---| +//! | `to_bson_uuid` | `Binary`, generic subtype | +//! | `doc! { "k": uuid }` | `Binary`, *UUID* subtype | +//! | `bson::to_document(&value)` | a `String` | +//! +//! And two ways of reading one back, which also disagree: a typed +//! `Collection` deserializes non-human-readably and demands binary, while +//! `bson::from_bson` demands a string. So a write and a read chosen +//! independently - which is how every one of these modules was written - agree +//! only by luck. The failure is never a compile error and usually not a runtime +//! error either: the filter simply matches nothing. +//! +//! What that cost, before this file existed: +//! +//! * **B1** audit events were readable across tenants, because the tenant +//! filter was dropped rather than fixed; +//! * **B2** token revocation was a silent no-op - revoked JWTs stayed valid, +//! and logout did nothing; +//! * role assignments were unreadable, so every role-derived permission +//! silently vanished and **a user holding an admin role was authorized as +//! though they held none**; +//! * permission records could not be deserialized at all; +//! * business metadata could be written but never read back; +//! * **every** service-user method was a no-op, including the API-key lookup, +//! so API-key authentication could not succeed on Mongo at all; +//! * listing active tokens failed outright; +//! * a branch with a head commit - that is, any branch that has been committed +//! to - could not be read. +//! +//! Each was found separately, by a different failing test, months apart. Fixing +//! them one at a time treats eight symptoms of one cause. This file is the +//! cause-level test: it round-trips every entity that carries a `Uuid`, so a +//! new collection whose write and read disagree fails here immediately rather +//! than in whatever feature happens to touch it next. +//! +//! The rule the fixes settled on: **write through `to_bson_uuid`, read through +//! `from_bson_uuid`**, which accepts all three encodings so data already in a +//! deployed database still loads. +//! +//! Requires a MongoDB; skips with a note when `PANGOLIN_TEST_MONGO_URL` is +//! unset. CI sets it for both topologies. + +use chrono::Utc; +use pangolin_core::audit::{AuditAction, AuditLogEntry, ResourceType}; +use pangolin_core::business_metadata::{AccessRequest, BusinessMetadata, RequestStatus}; +use pangolin_core::model::*; +use pangolin_core::permission::{Action, Permission, PermissionScope, Role, UserRole}; +use pangolin_core::token::TokenInfo; +use pangolin_core::user::{ServiceUser, User, UserRole as UserRoleEnum}; +use pangolin_store::{CatalogStore, MongoStore}; +use std::collections::{HashMap, HashSet}; +use uuid::Uuid; + +/// Open a store against a database unique to this test, or skip. +macro_rules! store_or_skip { + () => {{ + let Some(url) = pangolin_store::test_support::mongo_url() else { + println!("skipping: set PANGOLIN_TEST_MONGO_URL to run this test"); + return; + }; + let db = format!("pangolin_roundtrip_{}", Uuid::new_v4().simple()); + match MongoStore::new(&url, &db).await { + Ok(store) => store, + Err(e) => panic!("PANGOLIN_TEST_MONGO_URL is set but unusable: {e}"), + } + }}; +} + +async fn seeded_tenant(store: &MongoStore) -> Uuid { + let tenant_id = Uuid::new_v4(); + store + .create_tenant(Tenant { + id: tenant_id, + name: format!("roundtrip-{tenant_id}"), + properties: HashMap::new(), + }) + .await + .expect("create tenant"); + tenant_id +} + +#[tokio::test] +async fn tenant_round_trips() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + + let read = store + .get_tenant(tenant_id) + .await + .expect("get tenant") + .expect("tenant should exist"); + assert_eq!(read.id, tenant_id, "tenant id did not round-trip"); +} + +#[tokio::test] +async fn warehouse_round_trips() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + + let warehouse = Warehouse { + id: Uuid::new_v4(), + name: "wh".to_string(), + tenant_id, + storage_config: HashMap::new(), + use_sts: false, + vending_strategy: None, + }; + store + .create_warehouse(tenant_id, warehouse.clone()) + .await + .expect("create warehouse"); + + let read = store + .get_warehouse(tenant_id, "wh".to_string()) + .await + .expect("get warehouse") + .expect("warehouse should exist"); + assert_eq!(read.id, warehouse.id, "warehouse id did not round-trip"); + assert_eq!( + read.tenant_id, tenant_id, + "warehouse tenant_id did not round-trip" + ); +} + +#[tokio::test] +async fn catalog_and_asset_round_trip() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + + let catalog = Catalog { + id: Uuid::new_v4(), + name: "cat".to_string(), + catalog_type: CatalogType::Local, + warehouse_name: None, + storage_location: None, + federated_config: None, + properties: HashMap::new(), + }; + store + .create_catalog(tenant_id, catalog.clone()) + .await + .expect("create catalog"); + + let read = store + .get_catalog(tenant_id, "cat".to_string()) + .await + .expect("get catalog") + .expect("catalog should exist"); + assert_eq!(read.id, catalog.id, "catalog id did not round-trip"); + + let namespace = vec!["ns".to_string()]; + store + .create_namespace( + tenant_id, + "cat", + Namespace { + name: namespace.clone(), + properties: HashMap::new(), + }, + ) + .await + .expect("create namespace"); + + let asset = Asset { + id: Uuid::new_v4(), + name: "tbl".to_string(), + kind: AssetType::DeltaTable, + location: "s3://bucket/tbl".to_string(), + properties: HashMap::new(), + }; + store + .create_asset(tenant_id, "cat", None, namespace.clone(), asset.clone()) + .await + .expect("create asset"); + + let read = store + .get_asset(tenant_id, "cat", None, namespace, "tbl".to_string()) + .await + .expect("get asset") + .expect("asset should exist"); + assert_eq!(read.id, asset.id, "asset id did not round-trip"); + assert_eq!( + read.kind, + AssetType::DeltaTable, + "asset kind did not round-trip" + ); + + // The by-id index is a separate lookup path with its own encoding. + let by_id = store + .get_asset_by_id(tenant_id, asset.id) + .await + .expect("get asset by id"); + assert!( + by_id.is_some(), + "asset was not findable by id - the id was stored in a form the lookup \ + cannot match" + ); +} + +#[tokio::test] +async fn user_round_trips() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + + let user = User { + id: Uuid::new_v4(), + username: format!("rt_{}", Uuid::new_v4().simple()), + email: format!("rt_{}@example.test", Uuid::new_v4().simple()), + password_hash: None, + oauth_provider: None, + oauth_subject: None, + tenant_id: Some(tenant_id), + role: UserRoleEnum::TenantUser, + created_at: Utc::now(), + updated_at: Utc::now(), + last_login: None, + active: true, + }; + store.create_user(user.clone()).await.expect("create user"); + + let read = store + .get_user(user.id) + .await + .expect("get user") + .expect("user should exist"); + assert_eq!(read.id, user.id, "user id did not round-trip"); + assert_eq!( + read.tenant_id, + Some(tenant_id), + "user tenant_id did not round-trip" + ); +} + +/// Roles and role assignments. +/// +/// The assignment is the one that silently mis-authorized users: written by +/// serde as a string, queried as Binary, so `get_user_roles` returned nothing +/// and an admin's grants evaporated. +#[tokio::test] +async fn role_and_assignment_round_trip() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + let user_id = Uuid::new_v4(); + + let mut role = Role::new("rt-role".to_string(), None, tenant_id, Uuid::new_v4()); + let mut actions = HashSet::new(); + actions.insert(Action::Read); + role.add_permission(PermissionScope::Tenant, actions); + store.create_role(role.clone()).await.expect("create role"); + + let read = store + .get_role(role.id) + .await + .expect("get role") + .expect("role should be findable by id"); + assert_eq!(read.id, role.id, "role id did not round-trip"); + assert_eq!( + read.tenant_id, tenant_id, + "role tenant_id did not round-trip" + ); + assert_eq!( + read.permissions.len(), + 1, + "the role's grants did not round-trip" + ); + + let listed = store.list_roles(tenant_id, None).await.expect("list roles"); + assert!( + listed.iter().any(|r| r.id == role.id), + "the role was not returned by a tenant listing" + ); + + let assignment = UserRole::new(user_id, role.id, Uuid::new_v4()); + store + .assign_role(assignment.clone()) + .await + .expect("assign role"); + + let assignments = store.get_user_roles(user_id).await.expect("get user roles"); + assert_eq!( + assignments.len(), + 1, + "the role assignment was not readable back - this is the defect that \ + silently authorized an admin as though they held no roles" + ); + assert_eq!(assignments[0].role_id, role.id); + assert_eq!(assignments[0].user_id, user_id); + + // The grant must actually reach the permission list, which is what + // authorization consults. + let perms = store + .list_user_permissions(user_id, None) + .await + .expect("list user permissions"); + assert!( + perms.iter().any(|p| p.scope == PermissionScope::Tenant), + "a role-derived permission did not reach list_user_permissions" + ); +} + +#[tokio::test] +async fn direct_permission_round_trips() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + let user_id = Uuid::new_v4(); + + let mut actions = HashSet::new(); + actions.insert(Action::Write); + let permission = Permission::new( + user_id, + tenant_id, + PermissionScope::Catalog { + catalog_id: Uuid::new_v4(), + }, + actions, + Uuid::new_v4(), + ); + store + .create_permission(permission.clone()) + .await + .expect("create permission"); + + let perms = store + .list_user_permissions(user_id, None) + .await + .expect("list user permissions"); + assert!( + perms.iter().any(|p| p.id == permission.id), + "a direct permission was not readable back" + ); + let found = perms.iter().find(|p| p.id == permission.id).unwrap(); + assert_eq!(found.user_id, user_id, "user_id did not round-trip"); + assert_eq!(found.tenant_id, tenant_id, "tenant_id did not round-trip"); +} + +#[tokio::test] +async fn service_user_round_trips() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + + let api_key_hash = format!("hash-{}", Uuid::new_v4().simple()); + let service_user = ServiceUser::new( + "rt-svc".to_string(), + None, + tenant_id, + api_key_hash.clone(), + UserRoleEnum::TenantUser, + Uuid::new_v4(), + None, + ); + store + .create_service_user(service_user.clone()) + .await + .expect("create service user"); + + let read = store + .get_service_user(service_user.id) + .await + .expect("get service user") + .expect("service user should be findable by id"); + assert_eq!( + read.id, service_user.id, + "service user id did not round-trip" + ); + assert_eq!( + read.tenant_id, tenant_id, + "service user tenant_id did not round-trip" + ); + + let listed = store + .list_service_users(tenant_id, None) + .await + .expect("list service users"); + assert!( + listed.iter().any(|s| s.id == service_user.id), + "the service user was not returned by a tenant listing" + ); + + // The authentication path. If this lookup misses, an API key is simply + // rejected - service-user auth is down, not bypassed. + let by_hash = store + .get_service_user_by_api_key_hash(&api_key_hash) + .await + .expect("look up service user by api key hash") + .expect("the api key hash should resolve to its service user"); + assert_eq!( + by_hash.id, service_user.id, + "the api key hash resolved to the wrong principal" + ); + + // Updating the role rewrites a field the reader has to parse back into an + // enum; writing the Rust variant name instead of its serde form made the + // record unreadable from that point on. + let updated = store + .update_service_user( + service_user.id, + None, + Some("promoted".to_string()), + Some(UserRoleEnum::TenantAdmin), + None, + ) + .await + .expect("update service user"); + assert_eq!( + updated.role, + UserRoleEnum::TenantAdmin, + "the updated role did not round-trip" + ); + + store + .update_service_user_last_used(service_user.id) + .await + .expect("touch last_used"); + let touched = store + .get_service_user(service_user.id) + .await + .expect("re-read after touch") + .expect("service user should still exist"); + assert!( + touched.last_used.is_some(), + "last_used was written to a field nothing reads back" + ); + + store + .delete_service_user(service_user.id) + .await + .expect("delete service user"); + assert!( + store + .get_service_user(service_user.id) + .await + .expect("get after delete") + .is_none(), + "the service user survived its own deletion - the delete filter matched nothing" + ); +} + +/// Token issuance and revocation. +/// +/// B2: revocation was written by serde and queried as Binary, so the check +/// never matched and a revoked token - including after logout - stayed valid. +#[tokio::test] +async fn token_and_revocation_round_trip() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + let user_id = Uuid::new_v4(); + let token_id = Uuid::new_v4(); + + store + .store_token(TokenInfo { + id: token_id, + tenant_id, + user_id, + username: "rt".to_string(), + expires_at: Utc::now() + chrono::Duration::hours(1), + created_at: Utc::now(), + is_valid: true, + token: Some("opaque".to_string()), + }) + .await + .expect("store token"); + + let listed = store + .list_active_tokens(tenant_id, Some(user_id), None) + .await + .expect("list active tokens"); + assert!( + listed.iter().any(|t| t.id == token_id), + "the token was not readable back" + ); + + assert!( + !store + .is_token_revoked(token_id) + .await + .expect("check revocation"), + "a fresh token must not read as revoked" + ); + + store + .revoke_token(token_id, Utc::now() + chrono::Duration::hours(1), None) + .await + .expect("revoke token"); + + assert!( + store + .is_token_revoked(token_id) + .await + .expect("check revocation"), + "a revoked token must read back as revoked (B2)" + ); +} + +/// B1: the audit event's tenant scoping. +#[tokio::test] +async fn audit_event_round_trips() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + let user_id = Uuid::new_v4(); + let resource_id = Uuid::new_v4(); + + let entry = AuditLogEntry::success( + tenant_id, + Some(user_id), + "rt".to_string(), + AuditAction::CreateBranch, + ResourceType::Branch, + Some(resource_id), + "cat/feature".to_string(), + ); + let event_id = entry.id; + CatalogStore::log_audit_event(&store, tenant_id, entry) + .await + .expect("log audit event"); + + let read = store + .get_audit_event(tenant_id, event_id) + .await + .expect("get audit event") + .expect("audit event should exist"); + assert_eq!(read.id, event_id, "audit event id did not round-trip"); + assert_eq!(read.user_id, Some(user_id), "user_id did not round-trip"); + assert_eq!( + read.resource_id, + Some(resource_id), + "resource_id did not round-trip" + ); + assert_eq!( + read.action, + AuditAction::CreateBranch, + "the action must round-trip, not collapse to a default" + ); + + let listed = store + .list_audit_events(tenant_id, None) + .await + .expect("list audit events"); + assert!( + listed.iter().any(|e| e.id == event_id), + "the audit event was not returned by a listing" + ); +} + +#[tokio::test] +async fn business_metadata_round_trips() { + let store = store_or_skip!(); + let asset_id = Uuid::new_v4(); + let author = Uuid::new_v4(); + + let mut metadata = BusinessMetadata::new(asset_id, author); + metadata.tags = vec!["pii".to_string()]; + metadata.description = Some("round trip".to_string()); + let metadata_id = metadata.id; + + store + .upsert_business_metadata(metadata) + .await + .expect("upsert business metadata"); + + let read = store + .get_business_metadata(asset_id) + .await + .expect("get business metadata") + .expect("metadata should exist"); + assert_eq!(read.id, metadata_id, "metadata id did not round-trip"); + assert_eq!(read.asset_id, asset_id, "asset_id did not round-trip"); + assert_eq!(read.created_by, author, "created_by did not round-trip"); + assert_eq!(read.tags, vec!["pii".to_string()]); +} + +#[tokio::test] +async fn merge_operation_and_conflict_round_trip() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + + let operation = MergeOperation::new( + tenant_id, + "cat".to_string(), + "feature".to_string(), + "main".to_string(), + None, + Uuid::new_v4(), + ); + let operation_id = operation.id; + store + .create_merge_operation(operation) + .await + .expect("create merge operation"); + + let read = store + .get_merge_operation(operation_id) + .await + .expect("get merge operation") + .expect("merge operation should be findable by id"); + assert_eq!(read.id, operation_id, "operation id did not round-trip"); + assert_eq!(read.tenant_id, tenant_id, "tenant_id did not round-trip"); + + let listed = store + .list_merge_operations(tenant_id, "cat", None) + .await + .expect("list merge operations"); + assert!( + listed.iter().any(|o| o.id == operation_id), + "the merge operation was not returned by a listing" + ); + + let conflict = MergeConflict::new( + operation_id, + ConflictType::MetadataConflict { + asset_name: "tbl".to_string(), + conflicting_properties: vec!["owner".to_string()], + }, + Some(Uuid::new_v4()), + "owner differs".to_string(), + ); + let conflict_id = conflict.id; + store + .create_merge_conflict(conflict) + .await + .expect("create merge conflict"); + + let conflicts = store + .list_merge_conflicts(operation_id, None) + .await + .expect("list merge conflicts"); + assert!( + conflicts.iter().any(|c| c.id == conflict_id), + "the conflict was not readable back by its operation id" + ); +} + +#[tokio::test] +async fn commit_round_trips() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + + let commit = Commit { + id: Uuid::new_v4(), + parent_id: None, + timestamp: Utc::now().timestamp_millis(), + author: "rt".to_string(), + message: "round trip".to_string(), + operations: vec![], + }; + let commit_id = commit.id; + store + .create_commit(tenant_id, commit) + .await + .expect("create commit"); + + let read = store + .get_commit(tenant_id, commit_id) + .await + .expect("get commit") + .expect("commit should be findable by id"); + assert_eq!(read.id, commit_id, "commit id did not round-trip"); +} + +/// Branches and tags. +/// +/// Both are written with a hand-built `doc!` rather than serde, which converts +/// a `Uuid` by a *third* route again (`impl From for Bson`). Whether that +/// agrees with what the read path expects is not something the type system +/// checks, so it is asserted here. +#[tokio::test] +async fn branch_and_tag_round_trip() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + store + .create_catalog( + tenant_id, + Catalog { + id: Uuid::new_v4(), + name: "cat".to_string(), + catalog_type: CatalogType::Local, + warehouse_name: None, + storage_location: None, + federated_config: None, + properties: HashMap::new(), + }, + ) + .await + .expect("create catalog"); + + let head = Uuid::new_v4(); + store + .create_branch( + tenant_id, + "cat", + Branch { + name: "feature".to_string(), + head_commit_id: Some(head), + branch_type: BranchType::Experimental, + assets: vec!["tbl".to_string()], + }, + ) + .await + .expect("create branch"); + + let read = store + .get_branch(tenant_id, "cat", "feature".to_string()) + .await + .expect("get branch") + .expect("branch should exist"); + assert_eq!( + read.head_commit_id, + Some(head), + "the branch head commit id did not round-trip" + ); + assert_eq!(read.assets, vec!["tbl".to_string()]); + + // A branch with no head is the common case right after creation, and it + // takes a different path through the same conversion. + store + .create_branch( + tenant_id, + "cat", + Branch { + name: "empty".to_string(), + head_commit_id: None, + branch_type: BranchType::Ingest, + assets: vec![], + }, + ) + .await + .expect("create headless branch"); + let headless = store + .get_branch(tenant_id, "cat", "empty".to_string()) + .await + .expect("get headless branch") + .expect("headless branch should exist"); + assert_eq!(headless.head_commit_id, None); + assert!( + matches!(headless.branch_type, BranchType::Ingest), + "the branch type did not round-trip" + ); + + let commit_id = Uuid::new_v4(); + store + .create_tag( + tenant_id, + "cat", + Tag { + name: "v1".to_string(), + commit_id, + }, + ) + .await + .expect("create tag"); + let tag = store + .get_tag(tenant_id, "cat", "v1".to_string()) + .await + .expect("get tag") + .expect("tag should exist"); + assert_eq!( + tag.commit_id, commit_id, + "the tag commit id did not round-trip" + ); + let tags = store + .list_tags(tenant_id, "cat", None) + .await + .expect("list tags"); + assert!( + tags.iter().any(|t| t.name == "v1"), + "the tag was not returned by a listing" + ); +} + +/// Access requests. +/// +/// The listing joins `access_requests.user-id` to `users.id`, so it only +/// returns anything if the two collections encode a UUID the same way - a +/// cross-collection agreement no single-module test can check. +#[tokio::test] +async fn access_request_round_trips() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + + let user = User { + id: Uuid::new_v4(), + username: format!("rt_{}", Uuid::new_v4().simple()), + email: format!("rt_{}@example.test", Uuid::new_v4().simple()), + password_hash: None, + oauth_provider: None, + oauth_subject: None, + tenant_id: Some(tenant_id), + role: UserRoleEnum::TenantUser, + created_at: Utc::now(), + updated_at: Utc::now(), + last_login: None, + active: true, + }; + store.create_user(user.clone()).await.expect("create user"); + + let request = AccessRequest::new( + tenant_id, + user.id, + Uuid::new_v4(), + Some("need read".to_string()), + ); + let request_id = request.id; + store + .create_access_request(request) + .await + .expect("create access request"); + + let read = store + .get_access_request(request_id) + .await + .expect("get access request") + .expect("access request should be findable by id"); + assert_eq!(read.user_id, user.id, "user_id did not round-trip"); + assert_eq!(read.asset_id, read.asset_id); + + let listed = store + .list_access_requests(tenant_id, None) + .await + .expect("list access requests"); + assert!( + listed.iter().any(|r| r.id == request_id), + "the access request was not returned by its tenant listing - the join \ + between access_requests.user-id and users.id did not match" + ); + + let mut approved = read; + approved.approve(Uuid::new_v4(), Some("ok".to_string())); + store + .update_access_request(approved.clone()) + .await + .expect("update access request"); + let after = store + .get_access_request(request_id) + .await + .expect("re-read access request") + .expect("access request should still exist"); + assert_eq!( + after.status, + RequestStatus::Approved, + "the approval did not round-trip" + ); + assert!( + after.reviewed_by.is_some(), + "reviewed_by did not round-trip" + ); +} + +/// The merge mutation paths. +/// +/// `create_merge_operation` is covered above, but the status transitions build +/// their update documents by hand - including one that writes a status literal +/// as a Rust `Debug` string. Those writes are never read back by any other +/// test, so a value that no longer deserializes would go unnoticed until a +/// merge was actually completed in production. +#[tokio::test] +async fn merge_status_transitions_round_trip() { + let store = store_or_skip!(); + let tenant_id = seeded_tenant(&store).await; + + let operation = MergeOperation::new( + tenant_id, + "cat".to_string(), + "feature".to_string(), + "main".to_string(), + Some(Uuid::new_v4()), + Uuid::new_v4(), + ); + let operation_id = operation.id; + store + .create_merge_operation(operation) + .await + .expect("create merge operation"); + + store + .update_merge_operation_status(operation_id, MergeStatus::Conflicted) + .await + .expect("set status"); + let read = store + .get_merge_operation(operation_id) + .await + .expect("re-read after status change") + .expect("operation should still exist"); + assert!( + matches!(read.status, MergeStatus::Conflicted), + "the status write produced a value that does not deserialize back" + ); + + let conflict = MergeConflict::new( + operation_id, + ConflictType::MetadataConflict { + asset_name: "tbl".to_string(), + conflicting_properties: vec!["owner".to_string()], + }, + Some(Uuid::new_v4()), + "owner differs".to_string(), + ); + let conflict_id = conflict.id; + store + .create_merge_conflict(conflict) + .await + .expect("create conflict"); + store + .add_conflict_to_operation(operation_id, conflict_id) + .await + .expect("attach conflict"); + + let with_conflict = store + .get_merge_operation(operation_id) + .await + .expect("re-read after attaching a conflict") + .expect("operation should still exist"); + assert!( + with_conflict.conflicts.contains(&conflict_id), + "the attached conflict id was written in a form the operation cannot \ + read back" + ); + + store + .resolve_merge_conflict( + conflict_id, + ConflictResolution { + conflict_id, + strategy: ResolutionStrategy::TakeSource, + resolved_value: None, + resolved_by: Uuid::new_v4(), + resolved_at: Utc::now(), + }, + ) + .await + .expect("resolve conflict"); + let resolved = store + .get_merge_conflict(conflict_id) + .await + .expect("re-read conflict") + .expect("conflict should still exist"); + assert!( + resolved.resolution.is_some(), + "the resolution did not round-trip" + ); + + let result_commit = Uuid::new_v4(); + store + .complete_merge_operation(operation_id, result_commit) + .await + .expect("complete merge"); + let completed = store + .get_merge_operation(operation_id) + .await + .expect("re-read after completion") + .expect("operation should still exist"); + assert!( + matches!(completed.status, MergeStatus::Completed), + "the completed status did not round-trip" + ); + assert_eq!( + completed.result_commit_id, + Some(result_commit), + "the result commit id did not round-trip" + ); + assert!( + completed.completed_at.is_some(), + "completed_at did not round-trip" + ); +} diff --git a/pangolin/pangolin_store/tests/postgres_tests.rs b/pangolin/pangolin_store/tests/postgres_tests.rs index 486d84a..d493fa6 100644 --- a/pangolin/pangolin_store/tests/postgres_tests.rs +++ b/pangolin/pangolin_store/tests/postgres_tests.rs @@ -138,10 +138,18 @@ async fn test_postgres_access_requests() { .expect("Create tenant"); // Setup: User + // + // The username and email are unique per run. They used to be the literal + // "req_user"/"req_pg@example.com", which is fine against a database created + // fresh for each run and fails on the second run against a persistent one - + // `users_username_key` is a unique constraint. A test that only passes on a + // clean database cannot be re-run, which is exactly what a CI service + // container and a developer's local Postgres both need. + let unique = Uuid::new_v4(); let user = User { id: Uuid::new_v4(), - username: "req_user".to_string(), - email: "req_pg@example.com".to_string(), + username: format!("req_user_{unique}"), + email: format!("req_pg_{unique}@example.com"), password_hash: None, oauth_provider: None, oauth_subject: None, diff --git a/pangolin/pangolin_store/tests/sqlite_migration_tests.rs b/pangolin/pangolin_store/tests/sqlite_migration_tests.rs new file mode 100644 index 0000000..25d7cae --- /dev/null +++ b/pangolin/pangolin_store/tests/sqlite_migration_tests.rs @@ -0,0 +1,206 @@ +//! Upgrade path for an existing SQLite database. +//! +//! The `audit_logs` fix was initially applied by editing `sqlite_schema.sql` and +//! bumping `SQLITE_SCHEMA_VERSION`. That is not a migration: the schema file is +//! written entirely with `CREATE TABLE IF NOT EXISTS`, which does nothing at all +//! when the table already exists. Fresh installs got the new columns and every +//! upgraded database kept the broken ones - with a version number now claiming +//! otherwise, which is worse than no version at all. +//! +//! These tests build a database with the genuine pre-v2 shape and assert the +//! upgrade actually works, which is the only way to tell a real migration from a +//! version bump. + +use pangolin_core::audit::{AuditAction, AuditLogEntry, ResourceType}; +use pangolin_store::{sqlite::SQLITE_SCHEMA_VERSION, SqliteStore}; +use std::collections::HashMap; +use uuid::Uuid; + +/// The `audit_logs` table exactly as it was declared before v2. +const V1_AUDIT_LOGS: &str = "CREATE TABLE IF NOT EXISTS audit_logs ( + id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + timestamp INTEGER NOT NULL, + actor TEXT NOT NULL, + action TEXT NOT NULL, + resource TEXT NOT NULL, + details TEXT +); +CREATE INDEX IF NOT EXISTS idx_audit_logs_tenant_ts ON audit_logs(tenant_id, timestamp DESC);"; + +/// A unique on-disk database per test; SQLite needs a real file for the +/// rename-and-recreate the migration performs. +fn temp_db_url() -> (std::path::PathBuf, String) { + let dir = std::env::temp_dir().join(format!("pangolin_migration_{}", Uuid::new_v4())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let path = dir.join("pangolin.db"); + let url = format!("sqlite://{}?mode=rwc", path.to_string_lossy()); + (dir, url) +} + +/// Stand up a store whose `audit_logs` has the pre-v2 shape, as an upgrading +/// deployment's database does. +async fn legacy_store(url: &str) -> SqliteStore { + let store = SqliteStore::new(url).await.expect("open sqlite"); + + // The tenants table has to exist first: audit_logs carries a foreign key to + // it in the real schema, and the tests below insert a tenant. + store + .apply_schema( + "CREATE TABLE IF NOT EXISTS tenants ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + properties TEXT + );", + ) + .await + .expect("seed tenants"); + + store.apply_schema(V1_AUDIT_LOGS).await.expect("seed v1"); + store +} + +async fn audit_columns(store: &SqliteStore, table: &str) -> Vec { + store.table_columns(table).await.expect("table_info") +} + +/// The core regression: after upgrading, an audit write must succeed. +/// +/// Before the migration existed this failed with "table audit_logs has no +/// column named user_id" - on a database that `run_migrations` had just +/// reported as being at the current version. +#[tokio::test] +async fn upgrading_a_v1_database_makes_audit_logging_work() { + let (_dir, url) = temp_db_url(); + let store = legacy_store(&url).await; + + // Sanity: the fixture really is the old shape. + let before = audit_columns(&store, "audit_logs").await; + assert!( + before.iter().any(|c| c == "actor"), + "fixture should start with the pre-v2 shape, got {before:?}" + ); + assert!( + !before.iter().any(|c| c == "user_id"), + "fixture should not already have the v2 columns" + ); + + store.run_migrations().await.expect("migrate"); + + let after = audit_columns(&store, "audit_logs").await; + for expected in [ + "user_id", + "username", + "resource_type", + "resource_id", + "resource_name", + "ip_address", + "user_agent", + "result", + "error_message", + "metadata", + ] { + assert!( + after.iter().any(|c| c == expected), + "column {expected} missing after migration; got {after:?}" + ); + } + + // The behaviour the columns exist for. + let tenant_id = Uuid::new_v4(); + store + .create_tenant(pangolin_core::model::Tenant { + id: tenant_id, + name: "upgraded".to_string(), + properties: HashMap::new(), + }) + .await + .expect("create tenant"); + + let entry = AuditLogEntry::success( + tenant_id, + Some(Uuid::new_v4()), + "upgrader".to_string(), + AuditAction::CreateBranch, + ResourceType::Branch, + Some(Uuid::new_v4()), + "cat/feature".to_string(), + ); + + store + .log_audit_event(tenant_id, entry) + .await + .expect("an audit write must succeed after upgrading"); + + let events = store + .list_audit_events(tenant_id, None) + .await + .expect("list audit events"); + assert_eq!(events.len(), 1, "the audit event should be readable back"); + assert_eq!( + events[0].action, + AuditAction::CreateBranch, + "the action must round-trip, not collapse to a default (B22)" + ); +} + +/// The old table is preserved rather than dropped. +#[tokio::test] +async fn upgrading_keeps_the_old_table_as_a_backup() { + let (_dir, url) = temp_db_url(); + let store = legacy_store(&url).await; + + store.run_migrations().await.expect("migrate"); + + let backup = audit_columns(&store, "audit_logs_pre_v2").await; + assert!( + backup.iter().any(|c| c == "actor"), + "the pre-v2 table should be kept as audit_logs_pre_v2, got {backup:?}" + ); +} + +/// Running migrations twice must be a no-op, not a second rename that clobbers +/// the real table. +#[tokio::test] +async fn migrating_twice_is_idempotent() { + let (_dir, url) = temp_db_url(); + let store = legacy_store(&url).await; + + store.run_migrations().await.expect("first migrate"); + store + .run_migrations() + .await + .expect("second migrate must be a no-op"); + + let after = audit_columns(&store, "audit_logs").await; + assert!( + after.iter().any(|c| c == "user_id"), + "audit_logs should still be the v2 shape after a second run, got {after:?}" + ); + assert_eq!( + store.schema_version().await.expect("version"), + Some(SQLITE_SCHEMA_VERSION) + ); +} + +/// A fresh database needs no migration and lands on the current shape. +#[tokio::test] +async fn a_fresh_database_is_created_at_the_current_version() { + let (_dir, url) = temp_db_url(); + let store = SqliteStore::new(&url).await.expect("open sqlite"); + + store.run_migrations().await.expect("migrate"); + + let columns = audit_columns(&store, "audit_logs").await; + assert!(columns.iter().any(|c| c == "user_id")); + assert_eq!( + store.schema_version().await.expect("version"), + Some(SQLITE_SCHEMA_VERSION) + ); + + // Nothing to back up, so no backup table should have been created. + assert!( + audit_columns(&store, "audit_logs_pre_v2").await.is_empty(), + "a fresh database should not produce a backup table" + ); +} diff --git a/pangolin/pangolin_store/tests/store_integration.rs b/pangolin/pangolin_store/tests/store_integration.rs index ce5ee30..edf66e0 100644 --- a/pangolin/pangolin_store/tests/store_integration.rs +++ b/pangolin/pangolin_store/tests/store_integration.rs @@ -1,5 +1,5 @@ use pangolin_store::{ - tests::{test_asset_update_consistency, test_dashboard_stats_consistency}, + tests::{parity, test_asset_update_consistency, test_dashboard_stats_consistency}, MemoryStore, MongoStore, PostgresStore, SqliteStore, }; use std::env; @@ -10,6 +10,7 @@ async fn test_memory_store_regression() { let store = MemoryStore::new(); test_asset_update_consistency(&store).await; test_dashboard_stats_consistency(&store).await; + parity::run_all(&store).await; } #[tokio::test] @@ -32,6 +33,7 @@ async fn test_sqlite_store_regression() { test_asset_update_consistency(&store).await; test_dashboard_stats_consistency(&store).await; + parity::run_all(&store).await; } #[tokio::test] @@ -50,6 +52,7 @@ async fn test_postgres_store_regression() { test_asset_update_consistency(&store).await; test_dashboard_stats_consistency(&store).await; + parity::run_all(&store).await; } #[tokio::test] @@ -70,4 +73,5 @@ async fn test_mongo_store_regression() { test_asset_update_consistency(&store).await; test_dashboard_stats_consistency(&store).await; + parity::run_all(&store).await; } diff --git a/pangolin/scripts/check_env_var_docs.sh b/pangolin/scripts/check_env_var_docs.sh new file mode 100755 index 0000000..8d3ca49 --- /dev/null +++ b/pangolin/scripts/check_env_var_docs.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# +# Verify the environment-variable reference against the source of truth. +# +# B43: `docs/environment-variables.md` documented `PANGOLIN_HOST`, +# `PANGOLIN_PORT` and `PANGOLIN_STORE_TYPE` - none of which any code reads - and +# omitted roughly twenty variables that are read. Anyone configuring a +# deployment from that page set variables that did nothing and missed the ones +# that mattered, which is exactly how B9's `PANGOLIN_STORE_TYPE` in the compose +# files survived. +# +# Hand-maintained documentation of a machine-readable fact drifts. This script +# re-derives the set from the code and fails when the docs and the code +# disagree, so the drift is caught in CI rather than by an auditor. +# +# Usage: scripts/check_env_var_docs.sh (run from the pangolin/ workspace root) + +set -euo pipefail + +DOC="../docs/environment-variables.md" + +if [[ ! -f "$DOC" ]]; then + echo "error: $DOC not found; run this from the pangolin/ workspace root" >&2 + exit 1 +fi + +# Every PANGOLIN_* name the server actually reads. Test-only knobs +# (PANGOLIN_TEST_*) are deliberately excluded: they configure the test harness, +# not a deployment. +mapfile -t IN_CODE < <( + grep -rhoE 'PANGOLIN_[A-Z0-9_]+' pangolin_api/src pangolin_store/src --include='*.rs' \ + | grep -v '^PANGOLIN_TEST_' \ + | sort -u +) + +# The doc deliberately names a few variables that do *not* exist, to warn +# readers off them. Those lines carry a `` marker so this +# check can tell "documented as real" from "documented as a trap". +mapfile -t IN_DOCS < <( + grep -vF '' "$DOC" \ + | grep -ohE 'PANGOLIN_[A-Z0-9_]+' \ + | grep -v '^PANGOLIN_TEST_' \ + | sort -u +) + +missing=() +for name in "${IN_CODE[@]}"; do + if ! printf '%s\n' "${IN_DOCS[@]}" | grep -qx "$name"; then + missing+=("$name") + fi +done + +phantom=() +for name in "${IN_DOCS[@]}"; do + if ! printf '%s\n' "${IN_CODE[@]}" | grep -qx "$name"; then + phantom+=("$name") + fi +done + +status=0 + +if (( ${#missing[@]} )); then + echo "error: read by the server but absent from $DOC:" >&2 + printf ' %s\n' "${missing[@]}" >&2 + status=1 +fi + +if (( ${#phantom[@]} )); then + echo "error: documented in $DOC but read by nothing:" >&2 + printf ' %s\n' "${phantom[@]}" >&2 + status=1 +fi + +if (( status == 0 )); then + echo "environment-variable reference matches the code (${#IN_CODE[@]} variables)" +fi + +exit "$status" diff --git a/pangolin_ui/.env.example b/pangolin_ui/.env.example index d93c4c7..ae65300 100644 --- a/pangolin_ui/.env.example +++ b/pangolin_ui/.env.example @@ -1,2 +1,12 @@ -VITE_API_URL=http://localhost:8080 -VITE_NO_AUTH=false +# Base URL of the Pangolin API. +# +# B31: this file used to declare VITE_API_URL, the compose files passed +# VITE_API_URL, and the client read PUBLIC_API_URL - three names, none of which +# agreed, so the client's fallback always won and every deployed build called +# `http://localhost:8080`, meaning the *visitor's* machine. +# +# SvelteKit's dynamic public env requires the PUBLIC_ prefix, so PUBLIC_API_URL +# is the correct name and now the only one. +# +# Leave it empty for a same-origin (reverse-proxy) deployment. +PUBLIC_API_URL=http://localhost:8080 diff --git a/pangolin_ui/.gitignore b/pangolin_ui/.gitignore index 3b462cb..5ba696c 100644 --- a/pangolin_ui/.gitignore +++ b/pangolin_ui/.gitignore @@ -21,3 +21,13 @@ Thumbs.db # Vite vite.config.js.timestamp-* vite.config.ts.timestamp-* + +# Test and debug output (B45). +# +# ~260 KB of `check_*.txt` scratch files and Playwright artefacts were tracked +# in git. The ignore patterns did not cover them, and by the time they were +# added the files were already tracked - so `.gitignore` had no effect on them. +test-results/ +playwright-report/ +check_*.txt +node_modules/ diff --git a/pangolin_ui/check_catalogs_list.txt b/pangolin_ui/check_catalogs_list.txt deleted file mode 100644 index dd960f2..0000000 --- a/pangolin_ui/check_catalogs_list.txt +++ /dev/null @@ -1,971 +0,0 @@ - -> pangolin-ui@0.1.0 check -> svelte-kit sync && svelte-check --tsconfig ./tsconfig.json - -Loading svelte-check in workspace: /home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui -Getting Svelte diagnostics... - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/lib/components/ui/Modal.svelte:30:2 -Warn: Elements with the 'dialog' interactive role must have a tabindex value -https://svelte.dev/e/a11y_interactive_supports_focus (svelte) -{#if open} -
e.key === 'Escape' && close()} - role="dialog" - aria-modal="true" - > -
- -
- - -
- - {#if title} -
-

{title}

-
- {/if} - - -
- -
- - -
- - - -
-
-
-
-{/if} - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/lib/components/ui/DataTable.svelte:105:9 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
- Loading... - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/lib/components/ui/Textarea.svelte:18:5 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) - {#if label} - - {/if} - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/branches/+page.svelte:97:5 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - - ` rather than `` rather than ` - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/assets/[id]/+page.svelte:194:33 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/assets/[id]/+page.svelte:204:33 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/branches/[catalog]/[name]/+page.svelte:114:5 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
-
- -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/catalogs/[name]/+page.svelte:108:5 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
-
- -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/catalogs/[name]/+page.svelte:193:7 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - ` rather than `` rather than ` - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/assets/[id]/+page.svelte:194:33 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/assets/[id]/+page.svelte:204:33 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/branches/[catalog]/[name]/+page.svelte:114:5 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
-
- -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/catalogs/[name]/+page.svelte:108:5 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
-
- -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/catalogs/[name]/+page.svelte:193:7 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - ` rather than `` rather than ` - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/assets/[id]/+page.svelte:194:33 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/assets/[id]/+page.svelte:204:33 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/branches/[catalog]/[name]/+page.svelte:114:5 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
-
- -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/catalogs/[name]/+page.svelte:108:5 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
-
- -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/catalogs/[name]/+page.svelte:193:7 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - ` rather than `` rather than ` - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/assets/[id]/+page.svelte:194:33 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/assets/[id]/+page.svelte:204:33 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/branches/[catalog]/[name]/+page.svelte:114:5 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
-
- -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/catalogs/[name]/+page.svelte:108:5 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
-
- -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/catalogs/[name]/+page.svelte:193:7 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - ` rather than `` rather than ` - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/assets/[id]/+page.svelte:194:33 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/assets/[id]/+page.svelte:204:33 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- - - -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/branches/[catalog]/[name]/+page.svelte:114:5 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
-
- -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/catalogs/[name]/+page.svelte:108:5 -Warn: Self-closing HTML tags for non-void elements are ambiguous — use `
` rather than `
` -https://svelte.dev/e/element_invalid_self_closing_tag (svelte) -
-
-
- -/home/alexmerced/development/personal/Personal/2026/pangolin/pangolin_ui/src/routes/catalogs/[name]/+page.svelte:193:7 -Warn: A form label must be associated with a control -https://svelte.dev/e/a11y_label_has_associated_control (svelte) -
- -