diff --git a/.github/actions/save-build-caches/action.yml b/.github/actions/save-build-caches/action.yml new file mode 100644 index 000000000..c1d4d3e60 --- /dev/null +++ b/.github/actions/save-build-caches/action.yml @@ -0,0 +1,47 @@ +name: Save completed build work +description: Preserve compiler objects and uv wheels when later tests fail, including PR runs. +inputs: + rust-key: + description: Boxington primary key, empty when Rust caching is disabled. + default: '' + uv-key: + description: setup-uv primary key, empty when uv caching is disabled. + default: '' +runs: + using: composite + steps: + # The upstream action saves successful default-branch pushes. Preserve + # completed objects for failed jobs and PRs as well; GitHub scopes PR + # caches to the PR merge ref, so they cannot populate the trunk cache. + - name: Export completed Rust builds + id: rust + if: inputs.rust-key != '' && (github.event_name != 'push' || failure()) + shell: bash + run: | + archive="$(mbx cache dir)/github-actions-cache-v1.tar" + if mbx cache export --group "$MBX_CACHE_EXPORT_GROUP" "$archive" > "$RUNNER_TEMP/mbx-export.log" 2>&1; then + echo "archive=$archive" >> "$GITHUB_OUTPUT" + elif grep -q 'no completed mbx builds are recorded for export group' "$RUNNER_TEMP/mbx-export.log"; then + cat "$RUNNER_TEMP/mbx-export.log" + else + cat "$RUNNER_TEMP/mbx-export.log" >&2 + exit 1 + fi + + - uses: actions/cache/save@v4 + if: ${{ !cancelled() && steps.rust.outputs.archive != '' }} + with: + path: ${{ steps.rust.outputs.archive }} + key: ${{ inputs.rust-key }}-${{ github.sha }} + + # setup-uv saves successful jobs itself, but its post step skips failures. + - name: Prune uv cache after failure + if: inputs.uv-key != '' && failure() + shell: bash + run: uv cache prune --ci + + - uses: actions/cache/save@v4 + if: inputs.uv-key != '' && failure() + with: + path: ${{ runner.temp }}/uv-cache + key: ${{ inputs.uv-key }} diff --git a/.github/actions/setup-rust-cache/action.yml b/.github/actions/setup-rust-cache/action.yml new file mode 100644 index 000000000..e63375433 --- /dev/null +++ b/.github/actions/setup-rust-cache/action.yml @@ -0,0 +1,66 @@ +name: Cache Rust compilation +description: Share completed compiler objects across Cargo, uv, maturin, and Make builds. +inputs: + suffix: + description: Job and matrix partition for the compiler cache. + required: true +outputs: + key: + description: Key used by the Boxington action. + value: ${{ steps.cache.outputs.cache-primary-key }} +runs: + using: composite + steps: + # Upstream 1.11.0 publishes ARM macOS binaries, but no Intel macOS binary. + # Bootstrap that host once, then reuse the pinned executable across jobs. + - uses: actions/cache/restore@v4 + if: runner.os == 'macOS' && runner.arch == 'X64' + id: intel + with: + path: ${{ runner.temp }}/mbx-intel/bin/mbx + key: mbx-1.11.0-macos-x64-v1 + + - name: Build Boxington for Intel macOS + if: runner.os == 'macOS' && runner.arch == 'X64' && steps.intel.outputs.cache-hit != 'true' + shell: bash + run: cargo install --git https://github.com/jdx/mr-boxington --rev 4ebd2af1eb7e70fee37ec8678c30c0619f5ed5af --locked --root "$RUNNER_TEMP/mbx-intel" mbx + + - uses: actions/cache/save@v4 + if: runner.os == 'macOS' && runner.arch == 'X64' && steps.intel.outputs.cache-hit != 'true' + with: + path: ${{ runner.temp }}/mbx-intel/bin/mbx + key: mbx-1.11.0-macos-x64-v1 + + - name: Enable pinned Intel Boxington + if: runner.os == 'macOS' && runner.arch == 'X64' + shell: bash + run: | + test "$("$RUNNER_TEMP/mbx-intel/bin/mbx" --version)" = 'mbx 1.11.0' + echo "$RUNNER_TEMP/mbx-intel/bin" >> "$GITHUB_PATH" + + - uses: jdx/mr-boxington-action@a20e1ffcd962370fb2b6045c13b7b349f7b03386 + id: cache + env: + MBX_CACHE_DIR: ${{ github.workspace }}/.ci-mbx + MBX_TARGET_VIEWS: '0' + with: + version: ${{ (runner.os != 'macOS' || runner.arch != 'X64') && '1.11.0' || '' }} + github-cache-mode: objects + cache-generation: sidemantic-v1-${{ inputs.suffix }} + + - name: Enable Cargo subprocess caching + shell: bash + run: | + # mbx dispatches as a Cargo shim when invoked under Cargo's filename. + # Keep the shim in the workspace so maturin's workspace mount can use it too. + shim_dir="$GITHUB_WORKSPACE/.ci-cache-bin" + mkdir -p "$shim_dir" + if [ "$RUNNER_OS" = Windows ]; then + cp "$(command -v mbx)" "$shim_dir/cargo.exe" + else + cp "$(command -v mbx)" "$shim_dir/cargo" + chmod +x "$shim_dir/cargo" + fi + echo "$shim_dir" >> "$GITHUB_PATH" + echo "MBX_CACHE_DIR=$GITHUB_WORKSPACE/.ci-mbx" >> "$GITHUB_ENV" + echo "MBX_TARGET_VIEWS=0" >> "$GITHUB_ENV" diff --git a/.github/actions/setup-uv/action.yml b/.github/actions/setup-uv/action.yml new file mode 100644 index 000000000..434a2c7a0 --- /dev/null +++ b/.github/actions/setup-uv/action.yml @@ -0,0 +1,41 @@ +name: Set up uv with native source caching +description: Cache downloaded dependencies and built wheels, including their Rust build inputs. +inputs: + enable-cache: + description: Disable in production release jobs. + default: 'true' + suffix: + description: Job and matrix partition for the dependency cache. + required: true +outputs: + cache-key: + description: Key used by setup-uv. + value: ${{ steps.uv.outputs.cache-key }} +runs: + using: composite + steps: + - uses: astral-sh/setup-uv@v7 + id: uv + with: + version: '0.12.14' + enable-cache: ${{ inputs.enable-cache }} + cache-local-path: ${{ runner.temp }}/uv-cache + cache-suffix: ${{ inputs.suffix }} + cache-dependency-glob: | + uv.lock + pyproject.toml + sidemantic-rs/pyproject.toml + crates/dax-pyo3/pyproject.toml + crates/dax-pyo3/README.md + .cargo/config.toml + rust-toolchain* + Cargo.lock + Cargo.toml + sidemantic-rs/Cargo.toml + sidemantic-rs/build.rs + sidemantic-rs/src/** + crates/*/Cargo.toml + crates/*/src/** + crates/dax-pyo3/python/**/*.py + crates/dax-pyo3/python/**/*.pyi + crates/dax-pyo3/python/**/py.typed diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f570ce83..dd52777b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,14 +12,19 @@ concurrency: jobs: python: - name: Python ${{ matrix.python-version }} + name: Python ${{ matrix.python-version }} (${{ matrix.engine }} engine) runs-on: ubuntu-latest env: UV_PYTHON: ${{ matrix.python-version }} + SIDEMANTIC_ENGINE: ${{ matrix.engine }} strategy: fail-fast: false matrix: python-version: ["3.11", "3.12", "3.13", "3.14"] + engine: [python] + include: + - python-version: "3.12" + engine: rust steps: - uses: actions/checkout@v4 @@ -27,10 +32,18 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} @@ -83,11 +96,24 @@ jobs: uv build --wheel unzip -l dist/*.whl | grep -q 'sidemantic/py.typed' - - name: Run tests - run: uv run pytest -v --cov-fail-under=76 + - name: Run Python reference suite + if: matrix.engine == 'python' + run: uv run pytest -v --test-engine python --cov-fail-under=76 --junitxml=test-results.xml + + - name: Run full shared suite against Rust + if: matrix.engine == 'rust' + run: uv run pytest -v --test-engine rust --no-cov --junitxml=test-results.xml + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: tests-${{ matrix.python-version }}-${{ matrix.engine }} + path: test-results.xml - name: Run installed-wheel DAX smoke run: | + uv build sidemantic-rs --out-dir /tmp/sidemantic-dax-dist uv build crates/dax-pyo3 --out-dir /tmp/sidemantic-dax-dist uv build --out-dir /tmp/sidemantic-dist SIDEMANTIC_WHEEL=$(realpath "$(find /tmp/sidemantic-dist -name 'sidemantic-[0-9]*.whl' -print -quit)") @@ -98,6 +124,13 @@ jobs: sidemantic_dax.parse_expression("1") PY + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + dependency-compatibility: name: Dependencies (${{ matrix.name }}) runs-on: ubuntu-latest @@ -119,10 +152,18 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} @@ -151,20 +192,41 @@ jobs: -q -o addopts="-m 'not integration'" + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + base-install-cli: name: Base Install CLI runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python run: uv python install 3.12 + - name: Build runtime distribution + run: uv build sidemantic-rs --out-dir /tmp/sidemantic-runtime-dist + - name: Smoke base CLI install run: | set -euo pipefail @@ -183,12 +245,20 @@ jobs: agg: count YAML - uv run --no-project --with . sidemantic --version - uv run --no-project --with . sidemantic validate "$tmpdir/models" - uv run --no-project --with . sidemantic query "SELECT order_count, status FROM orders" --models "$tmpdir/models" --dry-run + uv run --no-project --find-links /tmp/sidemantic-runtime-dist --with . sidemantic --version + uv run --no-project --find-links /tmp/sidemantic-runtime-dist --with . sidemantic validate "$tmpdir/models" + uv run --no-project --find-links /tmp/sidemantic-runtime-dist --with . sidemantic query "SELECT order_count, status FROM orders" --models "$tmpdir/models" --dry-run + + uv run --no-project --find-links /tmp/sidemantic-runtime-dist --with . python - <<'PYTHON' + from sidemantic import SemanticLayer, Model, Metric + layer = SemanticLayer() + layer.add_model(Model(name="orders", table="orders", metrics=[Metric(name="count", agg="count")])) + layer.compile(metrics=["orders.count"]) + assert layer.last_engine_selection == {"engine": "rust", "reason": None} + PYTHON set +e - timeout 10s uv run --no-project --with . sidemantic serve "$tmpdir/models" >"$tmpdir/serve.out" 2>"$tmpdir/serve.err" + timeout 10s uv run --no-project --find-links /tmp/sidemantic-runtime-dist --with . sidemantic serve "$tmpdir/models" >"$tmpdir/serve.out" 2>"$tmpdir/serve.err" serve_status=$? set -e if [ "$serve_status" -eq 0 ]; then @@ -203,6 +273,13 @@ jobs: fi grep -q "sidemantic\\[api\\]" "$tmpdir/serve.err" + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + update-schema: name: Update JSON Schema needs: python @@ -215,17 +292,25 @@ jobs: with: ref: ${{ github.head_ref }} + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python run: uv python install 3.12 - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - name: Install dependencies run: uv sync --extra dev --extra dax @@ -240,16 +325,34 @@ jobs: git push fi + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + native-compat: name: Native Compatibility runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python run: uv python install 3.12 @@ -257,14 +360,6 @@ jobs: - name: Install dependencies run: uv sync --extra dev - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - - name: Cache cargo - uses: Swatinem/rust-cache@v2 - with: - workspaces: sidemantic-rs - - name: Run Python native fixtures run: uv run pytest tests/native_compat -v @@ -326,6 +421,13 @@ jobs: if-no-files-found: warn retention-days: 7 + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + check-rust-changes: name: Check Rust/DuckDB changes runs-on: ubuntu-latest @@ -348,6 +450,7 @@ jobs: - '.cargo/**' - 'rust-toolchain*' - '.github/workflows/ci.yml' + - '.github/actions/**' # The installed-extension conformance job exercises Python callers, # not just the crate. Keep its contract and fixture inputs gated. - 'pyproject.toml' @@ -395,9 +498,12 @@ jobs: - '.cargo/**' - 'rust-toolchain*' - '.github/workflows/ci.yml' + - '.github/actions/**' - 'pyproject.toml' - 'uv.lock' dax: + - '.github/actions/**' + - '.github/workflows/ci.yml' - 'crates/**' - 'Cargo.toml' - 'Cargo.lock' @@ -428,10 +534,11 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable - - name: Cache cargo - uses: Swatinem/rust-cache@v2 + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache with: - workspaces: . + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Run cargo fmt check run: cargo fmt --check @@ -442,6 +549,12 @@ jobs: - name: Run cargo test run: cargo test -p dax-parser -p dax-pyo3 + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + rust: name: Rust (sidemantic-rs) needs: check-rust-changes @@ -473,15 +586,18 @@ jobs: with: targets: wasm32-unknown-unknown - - name: Install uv - uses: astral-sh/setup-uv@v5 + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache with: - enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - - name: Cache cargo - uses: Swatinem/rust-cache@v2 + - name: Install uv + id: uv + uses: ./.github/actions/setup-uv with: - workspaces: sidemantic-rs + enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Run cargo fmt check run: cargo fmt --check @@ -572,6 +688,8 @@ jobs: run: uv run --no-project --with dist/*.whl tests/python_wheel_smoke.py - name: Check lightweight Python extension build + env: + MBX_VERIFY: "1" run: uvx maturin build --no-default-features --features python --out dist-python - name: Smoke lightweight Python extension wheel @@ -598,6 +716,13 @@ jobs: - name: Smoke Python extension ADBC wheel run: uv run --no-project --with dist-adbc/*.whl tests/python_wheel_adbc_smoke.py + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + duckdb-extension: name: DuckDB Extension needs: check-rust-changes @@ -609,17 +734,18 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Fetch DuckDB dependencies - working-directory: sidemantic-duckdb - run: make deps DUCKDB_VERSION=v1.5.5 - - name: Install Rust uses: dtolnay/rust-toolchain@stable - - name: Cache cargo - uses: Swatinem/rust-cache@v2 + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache with: - workspaces: sidemantic-rs + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + + - name: Fetch DuckDB dependencies + working-directory: sidemantic-duckdb + run: make deps DUCKDB_VERSION=v1.5.5 - name: Install build dependencies run: sudo apt-get update && sudo apt-get install -y ninja-build @@ -633,7 +759,11 @@ jobs: run: make test - name: Install uv for SQL host acceptance - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv + with: + enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Execute SemanticInput in the loaded DuckDB extension env: @@ -641,6 +771,13 @@ jobs: SIDEMANTIC_DUCKDB_EXTENSION: sidemantic-duckdb/build/release/extension/sidemantic/sidemantic.duckdb_extension run: uv run sidemantic-duckdb/test/test_semantic_input_host.py + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + webapp: name: Webapp (types + unit) needs: check-rust-changes diff --git a/.github/workflows/duckdb-extension-release.yml b/.github/workflows/duckdb-extension-release.yml index 695837f79..a59741b72 100644 --- a/.github/workflows/duckdb-extension-release.yml +++ b/.github/workflows/duckdb-extension-release.yml @@ -19,6 +19,7 @@ on: pull_request: paths: + - ".github/actions/**" - ".github/workflows/duckdb-extension-release.yml" permissions: @@ -35,6 +36,16 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + if: github.event_name == 'pull_request' + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Resolve DuckDB version id: duckdb env: @@ -61,9 +72,6 @@ jobs: DUCKDB_VERSION: ${{ steps.duckdb.outputs.version }} run: make deps DUCKDB_VERSION="$DUCKDB_VERSION" - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - name: Install build dependencies run: sudo apt-get update && sudo apt-get install -y ninja-build @@ -91,6 +99,12 @@ jobs: path: sidemantic-duckdb/dist/*.duckdb_extension if-no-files-found: error + - name: Save completed build caches + if: ${{ !cancelled() && github.event_name == 'pull_request' }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + github-release: name: Attach extension artifact to GitHub release needs: build diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9ab015ab5..aba072e9b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -33,10 +33,21 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python run: uv python install 3.12 @@ -65,12 +76,28 @@ jobs: POSTGRES_PASSWORD: "test" run: uv run pytest -m integration tests/db/test_postgres_cli_e2e.py -v + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + bigquery-integration: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Start BigQuery emulator run: | docker run -d --name bigquery-emulator \ @@ -82,9 +109,11 @@ jobs: sleep 5 - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python run: uv python install 3.12 @@ -100,16 +129,34 @@ jobs: BIGQUERY_DATASET: "test_dataset" run: uv run pytest -m integration tests/db/test_bigquery_integration.py -v + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + snowflake-integration: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python run: uv python install 3.12 @@ -125,6 +172,13 @@ jobs: SNOWFLAKE_TEST: "1" run: uv run pytest -m integration tests/db/test_snowflake_integration.py -v + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + clickhouse-integration: runs-on: ubuntu-latest @@ -147,10 +201,21 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python run: uv python install 3.12 @@ -166,6 +231,13 @@ jobs: CLICKHOUSE_PASSWORD: "clickhouse" run: uv run pytest -m integration tests/db/test_clickhouse_integration.py -v + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + adbc-integration: name: ADBC integration (${{ matrix.db }}) runs-on: ubuntu-latest @@ -205,6 +277,15 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install Rust for Rust ADBC probe + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Start BigQuery emulator if: matrix.db == 'bigquery' run: | @@ -215,9 +296,11 @@ jobs: sleep 5 - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python run: uv python install 3.12 @@ -257,9 +340,6 @@ jobs: SNOWFLAKE_TEST: "1" run: uv run pytest -m integration tests/db/test_adbc_ci_smoke.py -v - - name: Install Rust for Rust ADBC probe - uses: dtolnay/rust-toolchain@stable - - name: Export SQLite ADBC driver for Rust if: matrix.db == 'sqlite' run: | @@ -335,3 +415,10 @@ jobs: - name: Run Rust ADBC probe run: cargo test --manifest-path sidemantic-rs/Cargo.toml --features adbc-exec --test adbc_driver_matrix + + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c4afe4307..70593f856 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -39,6 +39,8 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v5 + with: + enable-cache: false - name: Set up Python run: uv python install 3.12 @@ -125,6 +127,7 @@ jobs: version_files = ( Path("pyproject.toml"), Path("crates/dax-pyo3/pyproject.toml"), + Path("sidemantic-rs/pyproject.toml"), Path("crates/dax-pyo3/Cargo.toml"), ) for path in version_files: @@ -152,6 +155,7 @@ jobs: pyproject_path.write_text( "\n".join( f' "sidemantic-dax>={version}",' if line.strip().startswith('"sidemantic-dax>=') + else f' "sidemantic-rs=={version}; sys_platform != \'emscripten\'",' if line.strip().startswith('"sidemantic-rs==') else line for line in pyproject_lines ) @@ -180,6 +184,7 @@ jobs: - name: Build packages run: | + uv build sidemantic-rs --sdist --out-dir dist uv build crates/dax-pyo3 --sdist --out-dir dist uv build --out-dir dist unzip -l "$(find dist -name 'sidemantic-[0-9]*.whl' -print -quit)" | grep -q 'sidemantic/py.typed' @@ -200,7 +205,7 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git add pyproject.toml sidemantic/__init__.py sidemantic/man/sidemantic.1 crates/dax-pyo3/pyproject.toml crates/dax-pyo3/Cargo.toml Cargo.lock uv.lock + git add pyproject.toml sidemantic/__init__.py sidemantic/man/sidemantic.1 sidemantic-rs/pyproject.toml crates/dax-pyo3/pyproject.toml crates/dax-pyo3/Cargo.toml Cargo.lock uv.lock git commit -m "Bump version to ${{ steps.version.outputs.new_version }}" git tag -a "v${{ steps.version.outputs.new_version }}" -m "Release v${{ steps.version.outputs.new_version }}" git push origin main @@ -257,11 +262,65 @@ jobs: path: crates/dax-pyo3/dist/*.whl if-no-files-found: error + build-runtime-wheels: + name: Build sidemantic-rs wheel (${{ matrix.target }}) + needs: publish + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + target: x86_64 + - os: ubuntu-24.04-arm + target: aarch64 + - os: macos-15-intel + target: x86_64 + - os: macos-14 + target: aarch64 + - os: windows-latest + target: x86_64-pc-windows-msvc + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.publish.outputs.tag }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Build abi3 wheel + uses: PyO3/maturin-action@v1 + with: + command: build + args: --release --manifest-path sidemantic-rs/Cargo.toml --out sidemantic-rs/dist + manylinux: auto + target: ${{ matrix.target }} + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: false + + - name: Smoke runtime wheel + shell: bash + run: uv run --no-project --with sidemantic-rs/dist/*.whl sidemantic-rs/tests/python_wheel_python_smoke.py + + - name: Upload abi3 wheel + uses: actions/upload-artifact@v4 + with: + name: dist-runtime-${{ matrix.os }}-${{ matrix.target }} + path: sidemantic-rs/dist/*.whl + if-no-files-found: error + release: name: Publish to PyPI and create release needs: - publish - build-dax-wheels + - build-runtime-wheels runs-on: ubuntu-latest permissions: id-token: write @@ -270,6 +329,8 @@ jobs: steps: - name: Install uv uses: astral-sh/setup-uv@v5 + with: + enable-cache: false - name: Download distributions uses: actions/download-artifact@v4 @@ -281,7 +342,10 @@ jobs: - name: Publish to PyPI env: UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }} - run: uv publish dist/* --token $UV_PUBLISH_TOKEN + run: | + uv publish dist/sidemantic_rs-* --token "$UV_PUBLISH_TOKEN" + uv publish dist/sidemantic_dax-* --token "$UV_PUBLISH_TOKEN" + uv publish dist/sidemantic-[0-9]* --token "$UV_PUBLISH_TOKEN" - name: Create GitHub Release env: diff --git a/.github/workflows/pyodide-test.yml b/.github/workflows/pyodide-test.yml index aff048ee2..725753a5e 100644 --- a/.github/workflows/pyodide-test.yml +++ b/.github/workflows/pyodide-test.yml @@ -17,7 +17,11 @@ jobs: node-version: '20' - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv + with: + enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Build sidemantic wheel run: uv build @@ -129,3 +133,9 @@ jobs: }); EOF node test_pyodide.mjs + + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + uv-key: ${{ steps.uv.outputs.cache-key }} diff --git a/.github/workflows/rust-runtime-release.yml b/.github/workflows/rust-runtime-release.yml index 731ee65ca..227204e92 100644 --- a/.github/workflows/rust-runtime-release.yml +++ b/.github/workflows/rust-runtime-release.yml @@ -20,6 +20,7 @@ on: pull_request: paths: + - ".github/actions/**" - ".github/workflows/rust-runtime-release.yml" - "sidemantic-rs/tests/packaged_cli_smoke.py" @@ -43,6 +44,13 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable + - name: Cache Rust compilation + if: github.event_name == 'pull_request' + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Run package metadata tests run: cargo test --locked --test package_metadata @@ -64,6 +72,12 @@ jobs: cargo publish --locked + - name: Save completed build caches + if: ${{ !cancelled() && github.event_name == 'pull_request' }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + cli-artifacts: name: Build CLI artifact (${{ matrix.artifact }}) runs-on: ${{ matrix.os }} @@ -90,6 +104,13 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable + - name: Cache Rust compilation + if: github.event_name == 'pull_request' + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Build CLI run: cargo build --release --locked --bin sidemantic @@ -106,7 +127,11 @@ jobs: fi - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv + with: + enable-cache: ${{ github.event_name == 'pull_request' }} + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Smoke packaged CLI shell: bash @@ -127,6 +152,13 @@ jobs: path: sidemantic-rs/dist/* if-no-files-found: error + - name: Save completed build caches + if: ${{ !cancelled() && github.event_name == 'pull_request' }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + github-release: name: Attach CLI artifacts to GitHub release needs: diff --git a/.github/workflows/sidemantic-rs-wheels.yml b/.github/workflows/sidemantic-rs-wheels.yml index 24fff09bd..4b2886e53 100644 --- a/.github/workflows/sidemantic-rs-wheels.yml +++ b/.github/workflows/sidemantic-rs-wheels.yml @@ -19,6 +19,7 @@ on: pull_request: paths: + - ".github/actions/**" - ".github/workflows/sidemantic-rs-wheels.yml" - "sidemantic-rs/pyproject.toml" - "sidemantic-rs/tests/python_wheel_smoke.py" @@ -51,6 +52,16 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + if: github.event_name == 'pull_request' + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Set up Python uses: actions/setup-python@v5 with: @@ -62,14 +73,33 @@ jobs: command: build args: --release --manifest-path sidemantic-rs/Cargo.toml --out sidemantic-rs/dist manylinux: auto + docker-options: ${{ github.event_name == 'pull_request' && '--env MBX_CACHE_DIR --env MBX_CACHE_EXPORT_GROUP --env MBX_TARGET_VIEWS=0 --env MBX_CACHE_LINKS --env MBX_GC_AUTO' || '' }} + before-script-linux: | + if [ -x "$GITHUB_WORKSPACE/.ci-cache-bin/cargo" ]; then + export PATH="$GITHUB_WORKSPACE/.ci-cache-bin:$PATH" + fi target: ${{ matrix.target }} + - name: Restore ownership of the container compiler cache + if: ${{ !cancelled() && runner.os == 'Linux' && github.event_name == 'pull_request' }} + shell: bash + run: | + if [ -d "$MBX_CACHE_DIR" ]; then + sudo chown -R "$(id -u):$(id -g)" "$MBX_CACHE_DIR" + fi + - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv + with: + enable-cache: ${{ github.event_name == 'pull_request' }} + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Smoke built wheel on its target host shell: bash - run: uv run --no-project --with sidemantic-rs/dist/*.whl sidemantic-rs/tests/python_wheel_smoke.py + run: | + uv run --no-project --with sidemantic-rs/dist/*.whl sidemantic-rs/tests/python_wheel_smoke.py + uv run --no-project --with sidemantic-rs/dist/*.whl sidemantic-rs/tests/python_wheel_python_smoke.py - name: Upload wheel uses: actions/upload-artifact@v4 @@ -78,6 +108,13 @@ jobs: path: sidemantic-rs/dist/*.whl if-no-files-found: error + - name: Save completed build caches + if: ${{ !cancelled() && github.event_name == 'pull_request' }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} + build-sdist: name: Build sdist runs-on: ubuntu-latest @@ -89,9 +126,11 @@ jobs: - uses: actions/checkout@v4 - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: - enable-cache: true + enable-cache: ${{ github.event_name == 'pull_request' }} + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python run: uv python install 3.12 @@ -106,6 +145,12 @@ jobs: path: sidemantic-rs/dist/*.tar.gz if-no-files-found: error + - name: Save completed build caches + if: ${{ !cancelled() && github.event_name == 'pull_request' }} + uses: ./.github/actions/save-build-caches + with: + uv-key: ${{ steps.uv.outputs.cache-key }} + release: name: Publish or attach wheel artifacts needs: @@ -123,7 +168,7 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' && inputs.publish_pypi == true }} uses: astral-sh/setup-uv@v5 with: - enable-cache: true + enable-cache: false - name: Download distributions uses: actions/download-artifact@v4 diff --git a/.github/workflows/wasm-package-release.yml b/.github/workflows/wasm-package-release.yml index 0b6c2e9b1..43b5628f5 100644 --- a/.github/workflows/wasm-package-release.yml +++ b/.github/workflows/wasm-package-release.yml @@ -13,6 +13,7 @@ on: - "sidemantic-wasm-v*" pull_request: paths: + - ".github/actions/**" - "sidemantic-wasm/**" - "sidemantic-rs/**" - "Cargo.toml" @@ -41,6 +42,13 @@ jobs: with: targets: wasm32-unknown-unknown + - name: Cache Rust compilation + if: github.event_name == 'pull_request' + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Install wasm-bindgen-cli matching the crate working-directory: . run: | @@ -103,3 +111,9 @@ jobs: fi # Publish the exact tarball exercised by the isolated consumer above. npm publish sidemantic-wasm.tgz --access public --ignore-scripts + + - name: Save completed build caches + if: ${{ !cancelled() && github.event_name == 'pull_request' }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} diff --git a/.github/workflows/yardstick-upstream.yml b/.github/workflows/yardstick-upstream.yml index e89ad8ab3..f86bccbdb 100644 --- a/.github/workflows/yardstick-upstream.yml +++ b/.github/workflows/yardstick-upstream.yml @@ -11,6 +11,7 @@ on: - cron: "17 10 * * *" pull_request: paths: + - ".github/actions/**" - ".github/workflows/yardstick-upstream.yml" - "docs/compatibility/yardstick.md" - "sidemantic/adapters/yardstick.py" @@ -38,10 +39,21 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust compilation + id: rust-cache + uses: ./.github/actions/setup-rust-cache + with: + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} + - name: Install uv - uses: astral-sh/setup-uv@v5 + id: uv + uses: ./.github/actions/setup-uv with: enable-cache: true + suffix: ${{ github.workflow }}-${{ github.job }}-${{ strategy.job-index || 0 }} - name: Set up Python run: uv python install 3.12 @@ -51,3 +63,10 @@ jobs: - name: Replay upstream Yardstick tests run: uv run pytest -q tests/queries/test_yardstick_measures_replay.py -m yardstick_upstream + + - name: Save completed build caches + if: ${{ !cancelled() }} + uses: ./.github/actions/save-build-caches + with: + rust-key: ${{ steps.rust-cache.outputs.key }} + uv-key: ${{ steps.uv.outputs.cache-key }} diff --git a/.gitignore b/.gitignore index dce2b92cd..071d041bc 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,7 @@ package.json # Rust workspace build artifacts (dax local wheel builds) /target/ + +# GitHub Actions compiler cache and Cargo shim +/.ci-mbx/ +/.ci-cache-bin/ diff --git a/crates/dax-pyo3/pyproject.toml b/crates/dax-pyo3/pyproject.toml index b213f9cb6..8af2d1caa 100644 --- a/crates/dax-pyo3/pyproject.toml +++ b/crates/dax-pyo3/pyproject.toml @@ -15,3 +15,23 @@ module-name = "sidemantic_dax._native" python-source = "python" features = ["pyo3/extension-module", "pyo3/abi3-py311"] exclude = ["dist/*", "target/*"] + +# uv otherwise only checks Python packaging metadata for local wheel rebuilds. +[tool.uv] +cache-keys = [ + { file = "pyproject.toml" }, + { file = "README.md" }, + { file = "Cargo.toml" }, + { file = "../../Cargo.toml" }, + { file = "../../Cargo.lock" }, + { file = "src/**/*" }, + { file = "python/**/*.py" }, + { file = "python/**/*.pyi" }, + { file = "python/**/py.typed" }, + { file = "../dax-parser/Cargo.toml" }, + { file = "../dax-parser/src/**/*" }, + { file = "../../.cargo/config.toml" }, + { file = "../../rust-toolchain*" }, + { env = "RUSTFLAGS" }, + { env = "CARGO_ENCODED_RUSTFLAGS" }, +] diff --git a/docs/rust-runtime.md b/docs/rust-runtime.md index 60034e55e..44fc7298a 100644 --- a/docs/rust-runtime.md +++ b/docs/rust-runtime.md @@ -48,6 +48,21 @@ Rust does not parse these source formats directly: Those remain Python-owned import paths. +## Default runtime and installation + +A normal `uv tool install sidemantic` includes the matching `sidemantic-rs` +package. Native validation, compilation, and semantic SQL rewriting use Rust by +default and report an error if it is unavailable or a capability is unsupported. +Use `--engine python` to select Python, or `--engine auto` to allow fallback for +known unsupported capabilities. Invalid input and unexpected failures still propagate. +`SIDEMANTIC_ENGINE=python` sets a process-wide default; explicit engine selection +and project runtime configuration take precedence. + +Pyodide excludes the native dependency and defaults to Python. The distributed +Python extension uses the lightweight `python` feature; database execution +continues through Python adapters. Rust ADBC execution requires a separate +source build with `python-adbc`. + ## Python API Engine Selection Python users can select the native runtime explicitly: diff --git a/docs/semantic-input.md b/docs/semantic-input.md index af166a1ad..ecddd50dd 100644 --- a/docs/semantic-input.md +++ b/docs/semantic-input.md @@ -351,8 +351,9 @@ synthetic fixtures, not production workload or performance qualification. Rust's `semantic_input` tests check decoding, keys, scope, dialects and rejection without relying on Python preprocessing. -This contract does not change the default engine, retire the Python compiler, -or claim that WASM, the DuckDB extension, and the Python binding already expose +Native Python installations now default to Rust with a matching runtime package; +Pyodide retains Python. This does not retire the Python compiler or claim that +WASM, the DuckDB extension, and the Python binding already expose identical capabilities. Each host needs corresponding acceptance evidence before its default or implementation ownership changes. diff --git a/pyproject.toml b/pyproject.toml index afe0ce787..2c32abde1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ readme = "README.md" license = {file = "LICENSE"} requires-python = ">=3.11" dependencies = [ + "sidemantic-rs==0.12.0; sys_platform != 'emscripten'", "antlr4-python3-runtime>=4.13.2", "sqlglot>=30.1.0", "pyyaml>=6.0", @@ -244,9 +245,14 @@ warn_unused_ignores = true [tool.uv] prerelease = "if-necessary" -required-environments = ["sys_platform == 'linux' and platform_machine == 'x86_64'"] +# Require binary wheels for the CPython versions exercised by CI, without +# imposing a Python upper bound on users or requiring future/PyPy wheels. +required-environments = [ + "sys_platform == 'linux' and platform_machine == 'x86_64' and implementation_name == 'cpython' and platform_python_implementation == 'CPython' and python_version < '3.15'", +] [tool.uv.sources] +sidemantic-rs = { path = "sidemantic-rs" } sidemantic-dax = { path = "crates/dax-pyo3" } [dependency-groups] diff --git a/sidemantic-rs/pyproject.toml b/sidemantic-rs/pyproject.toml index 824482b9d..dd3784c5f 100644 --- a/sidemantic-rs/pyproject.toml +++ b/sidemantic-rs/pyproject.toml @@ -4,11 +4,26 @@ build-backend = "maturin" [project] name = "sidemantic-rs" -version = "0.1.0" +version = "0.12.0" description = "Standalone Rust runtime bindings for Sidemantic" license = "AGPL-3.0-only" requires-python = ">=3.11" [tool.maturin] module-name = "sidemantic_rs" -features = ["python-adbc"] +features = ["python"] + +# uv otherwise only checks Python packaging metadata for local wheel rebuilds. +[tool.uv] +cache-keys = [ + { file = "pyproject.toml" }, + { file = "Cargo.toml" }, + { file = "../Cargo.toml" }, + { file = "../Cargo.lock" }, + { file = "build.rs" }, + { file = "src/**/*" }, + { file = "../.cargo/config.toml" }, + { file = "../rust-toolchain*" }, + { env = "RUSTFLAGS" }, + { env = "CARGO_ENCODED_RUSTFLAGS" }, +] diff --git a/sidemantic-rs/src/core/dependency.rs b/sidemantic-rs/src/core/dependency.rs index fb4d7ef4e..d03d383ab 100644 --- a/sidemantic-rs/src/core/dependency.rs +++ b/sidemantic-rs/src/core/dependency.rs @@ -31,8 +31,8 @@ impl SemanticColumnReference { pub fn parse_semantic_expression(sql: &str) -> crate::error::Result { #[cfg(target_arch = "wasm32")] crate::wasm_sql_guard::check(sql, DialectType::DuckDB)?; - let statement = polyglot_sql::parse_one(&format!("SELECT {sql}"), DialectType::DuckDB) - .map_err(|error| crate::error::SidemanticError::SqlParse(error.to_string()))?; + let statement = + crate::semantic_input::dialects::parse(&format!("SELECT {sql}"), DialectType::DuckDB)?; let Expression::Select(mut select) = statement else { return Err(crate::error::SidemanticError::SqlParse( "expected scalar expression".into(), @@ -69,11 +69,18 @@ fn column_references( let ast = serde_json::to_value(expression) .map_err(|error| crate::error::SidemanticError::SqlParse(error.to_string()))?; let mut references = Vec::new(); - let mut stack = vec![(&ast, false)]; - while let Some((node, aggregate_input)) = stack.pop() { + let mut stack = vec![(&ast, false, HashSet::::new())]; + while let Some((node, aggregate_input, bindings)) = stack.pop() { match node { serde_json::Value::Object(fields) => { let kind = (fields.len() == 1).then(|| fields.keys().next().unwrap().as_str()); + if kind == Some("lambda") { + let lambda = &fields["lambda"]; + let mut bindings = bindings; + bindings.extend(lambda_parameter_names(lambda)); + stack.push((&lambda["body"], aggregate_input, bindings)); + continue; + } if allow_subqueries && matches!( kind, @@ -88,67 +95,107 @@ fn column_references( }); } let aggregate_input = aggregate_input - || matches!( - kind, - Some( - "count" - | "sum" - | "avg" - | "min" - | "max" - | "median" - | "mode" - | "stddev" - | "stddev_pop" - | "stddev_samp" - | "variance" - | "var_pop" - | "var_samp" - | "aggregate_function" - | "group_concat" - | "string_agg" - | "list_agg" - | "array_agg" - | "count_if" - | "sum_if" - | "first" - | "last" - | "any_value" - | "approx_distinct" - | "approx_count_distinct" - | "approx_percentile" - | "percentile" - | "logical_and" - | "logical_or" - | "skewness" - | "array_concat_agg" - | "array_unique_agg" - | "bool_xor_agg" - ) - ); + || is_aggregate_ast_node(node) + || matches!(kind, Some("window" | "window_function")); if kind == Some("column") { let column: polyglot_sql::expressions::Column = serde_json::from_value(fields["column"].clone()).map_err(|error| { crate::error::SidemanticError::SqlParse(error.to_string()) })?; + if bindings.contains( + column + .table + .as_ref() + .map_or(column.name.name.as_str(), |table| table.name.as_str()), + ) { + continue; + } references.push(SemanticColumnReference { model: column.table.map(|table| table.name), field: column.name.name, aggregate_input, }); } else { - stack.extend(fields.values().map(|child| (child, aggregate_input))); + stack.extend( + fields + .values() + .map(|child| (child, aggregate_input, bindings.clone())), + ); } } - serde_json::Value::Array(children) => { - stack.extend(children.iter().map(|child| (child, aggregate_input))) - } + serde_json::Value::Array(children) => stack.extend( + children + .iter() + .map(|child| (child, aggregate_input, bindings.clone())), + ), _ => {} } } Ok(references) } +/// Aggregate expression variants in the pinned SQL AST. +pub(crate) fn is_aggregate_ast_kind(kind: &str) -> bool { + matches!( + kind, + "count" + | "sum" + | "avg" + | "min" + | "max" + | "median" + | "mode" + | "stddev" + | "stddev_pop" + | "stddev_samp" + | "variance" + | "var_pop" + | "var_samp" + | "aggregate_function" + | "group_concat" + | "string_agg" + | "list_agg" + | "array_agg" + | "count_if" + | "sum_if" + | "first" + | "last" + | "any_value" + | "approx_distinct" + | "approx_count_distinct" + | "approx_percentile" + | "percentile" + | "logical_and" + | "logical_or" + | "skewness" + | "array_concat_agg" + | "array_unique_agg" + | "bool_xor_agg" + ) +} + +pub(crate) fn is_aggregate_ast_node(node: &serde_json::Value) -> bool { + let Some(fields) = node.as_object().filter(|fields| fields.len() == 1) else { + return false; + }; + let (kind, value) = fields.iter().next().unwrap(); + is_aggregate_ast_kind(kind) + // polyglot 0.1.15 leaves bare DuckDB LIST as a generic function, + // while LIST with DISTINCT/ORDER/FILTER is an AggregateFunction. + || (kind == "function" + && value["quoted"].as_bool() != Some(true) + && value["name"].as_str().is_some_and(|name| name.eq_ignore_ascii_case("LIST"))) +} + +fn lambda_parameter_names(lambda: &serde_json::Value) -> Vec { + lambda["parameters"] + .as_array() + .into_iter() + .flatten() + .filter_map(|parameter| parameter["name"].as_str().map(str::to_owned)) + .collect() +} + /// Replace column nodes in every AST child, including typed functions that the /// pinned polyglot transform visitor does not descend into. pub fn replace_semantic_columns( @@ -176,9 +223,21 @@ fn replace_columns( value: &mut serde_json::Value, replacements: &std::collections::HashMap<(Option, String), String>, skip_subqueries: bool, + bindings: &HashSet, ) -> crate::error::Result<()> { match value { serde_json::Value::Object(fields) => { + if fields.len() == 1 && fields.contains_key("lambda") { + let lambda = fields.get_mut("lambda").unwrap(); + let mut bindings = bindings.clone(); + bindings.extend(lambda_parameter_names(lambda)); + return replace( + &mut lambda["body"], + replacements, + skip_subqueries, + &bindings, + ); + } if skip_subqueries && fields.len() == 1 && fields.keys().any(|name| { @@ -195,6 +254,9 @@ fn replace_columns( serde_json::from_value(fields["column"].clone()) .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; let key = (column.table.map(|table| table.name), column.name.name); + if bindings.contains(key.0.as_deref().unwrap_or(&key.1)) { + return Ok(()); + } if let Some(sql) = replacements.get(&key) { *value = serde_json::to_value(Expression::Raw(polyglot_sql::expressions::Raw { @@ -204,13 +266,13 @@ fn replace_columns( } } else { for child in fields.values_mut() { - replace(child, replacements, skip_subqueries)?; + replace(child, replacements, skip_subqueries, bindings)?; } } } serde_json::Value::Array(children) => { for child in children { - replace(child, replacements, skip_subqueries)?; + replace(child, replacements, skip_subqueries, bindings)?; } } _ => {} @@ -219,7 +281,7 @@ fn replace_columns( } let mut value = serde_json::to_value(expression) .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; - replace(&mut value, replacements, skip_subqueries)?; + replace(&mut value, replacements, skip_subqueries, &HashSet::new())?; serde_json::from_value(value).map_err(|error| SidemanticError::SqlGeneration(error.to_string())) } @@ -643,6 +705,69 @@ mod tests { #[allow(unused_imports)] use crate::core::model::{Aggregation, Dimension, Model}; + #[test] + fn lambda_bindings_are_local_to_the_body_for_dependencies_and_replacement() { + let sql = "list_transform(orders.items, x -> x.value + external.bias) + x.value"; + let references = semantic_column_references(sql).unwrap(); + let names: Vec<_> = references + .iter() + .map(SemanticColumnReference::name) + .collect(); + assert_eq!(names.len(), 3, "{names:?}"); + for name in ["orders.items", "external.bias", "x.value"] { + assert!(names.contains(&name.to_owned()), "{names:?}"); + } + let replacements = std::collections::HashMap::from([ + ((Some("x".into()), "value".into()), "outer_value".into()), + ((Some("external".into()), "bias".into()), "free_bias".into()), + ]); + let replaced = + replace_semantic_columns(parse_semantic_expression(sql).unwrap(), &replacements) + .unwrap(); + let rendered = polyglot_sql::generate(&replaced, DialectType::DuckDB).unwrap(); + assert_eq!(rendered.matches("outer_value").count(), 1, "{rendered}"); + assert!(rendered.contains("x.value"), "{rendered}"); + assert!(rendered.contains("free_bias"), "{rendered}"); + } + + #[test] + fn list_aggregate_inputs_stay_physical_through_scalar_list_functions() { + let sql = "LIST_AGGREGATE(LIST_TRANSFORM(LIST_DISTINCT(LIST(STRUCT_PACK(k := orders.id, v := orders.amount))), x -> x.v), 'quantile_cont', 0.5) + orders.revenue"; + let references = semantic_column_references(sql).unwrap(); + assert_eq!(references.len(), 3, "{references:?}"); + for reference in references { + assert_eq!( + reference.aggregate_input, + reference.field != "revenue", + "{reference:?}" + ); + } + assert!(validate_row_expression( + &parse_semantic_expression("LIST(amount)").unwrap(), + "test.row_scope" + ) + .is_err()); + let scalar_list = + semantic_column_references("list_transform(orders.amounts, x -> x + 1)").unwrap(); + assert_eq!(scalar_list.len(), 1); + assert!(!scalar_list[0].aggregate_input); + } + + #[test] + fn authored_window_partition_and_order_columns_are_physical_inputs() { + let references = semantic_column_references( + "sum(amount) over (partition by category order by year) + revenue", + ) + .unwrap(); + for reference in references { + assert_eq!( + reference.aggregate_input, + reference.field != "revenue", + "{reference:?}" + ); + } + } + #[test] fn outer_filter_dependencies_do_not_capture_nested_columns_or_literals() { let sql = "orders.status IN (SELECT status FROM allowed WHERE note = 'orders.revenue AND customers.id')"; @@ -767,49 +892,12 @@ pub fn validate_row_expression( match value { serde_json::Value::Object(fields) => { let kind = (fields.len() == 1).then(|| fields.keys().next().unwrap().as_str()); - matches!( - kind, - Some( - "select" - | "subquery" - | "raw" - | "window" - | "window_function" - | "count" - | "sum" - | "avg" - | "min" - | "max" - | "median" - | "mode" - | "stddev" - | "stddev_pop" - | "stddev_samp" - | "variance" - | "var_pop" - | "var_samp" - | "aggregate_function" - | "group_concat" - | "string_agg" - | "list_agg" - | "array_agg" - | "count_if" - | "sum_if" - | "first" - | "last" - | "any_value" - | "approx_distinct" - | "approx_count_distinct" - | "approx_percentile" - | "percentile" - | "logical_and" - | "logical_or" - | "skewness" - | "array_concat_agg" - | "array_unique_agg" - | "bool_xor_agg" - ) - ) || fields.values().any(visit) + (is_aggregate_ast_node(value) + || matches!( + kind, + Some("select" | "subquery" | "raw" | "window" | "window_function") + )) + || fields.values().any(visit) } serde_json::Value::Array(values) => values.iter().any(visit), _ => false, diff --git a/sidemantic-rs/src/core/graph.rs b/sidemantic-rs/src/core/graph.rs index bebd5a666..bff391d29 100644 --- a/sidemantic-rs/src/core/graph.rs +++ b/sidemantic-rs/src/core/graph.rs @@ -363,6 +363,33 @@ impl SemanticGraph { Some(owner) } + /// Whether this SQL instance is a declared many-to-many junction population. + pub(crate) fn is_bridge_instance(&self, instance: &str) -> bool { + self.relationship_instances + .iter() + .any(|((source, name), target)| { + let Some(model) = self.get_model(source) else { + return false; + }; + model.relationships.iter().any(|relationship| { + if !relationship.active + || relationship.name != *name + || relationship.r#type != RelationshipType::ManyToMany + { + return false; + } + let Some(through) = &relationship.through else { + return false; + }; + if source != &model.name || relationship.target_model.is_some() { + instance == format!("{target}$through") + } else { + instance == through + } + }) + }) + } + pub fn relationship_target_instance( &self, source: &str, @@ -798,12 +825,10 @@ impl SemanticGraph { let fk_keys = rel.foreign_key_columns(); let pk_keys = if rel.primary_key.is_some() || rel.primary_key_columns.is_some() { rel.primary_key_columns() - } else if (instance != canonical || rel.target_model.is_some()) - && matches!( - rel.r#type, - RelationshipType::OneToMany | RelationshipType::OneToOne - ) - { + } else if matches!( + rel.r#type, + RelationshipType::OneToMany | RelationshipType::OneToOne + ) { model.primary_keys() } else { self.models @@ -821,15 +846,10 @@ impl SemanticGraph { (model.primary_keys(), fk_keys.clone()) } } - RelationshipType::OneToOne - if instance != canonical || rel.target_model.is_some() => - { + RelationshipType::ManyToOne => (fk_keys.clone(), pk_keys.clone()), + RelationshipType::OneToMany | RelationshipType::OneToOne => { (pk_keys.clone(), fk_keys.clone()) } - RelationshipType::ManyToOne | RelationshipType::OneToOne => { - (fk_keys.clone(), pk_keys.clone()) - } - RelationshipType::OneToMany => (pk_keys.clone(), fk_keys.clone()), }; self.adjacency.entry(instance.clone()).or_default().push(( @@ -844,7 +864,13 @@ impl SemanticGraph { if rel.r#type == RelationshipType::Cross { None } else { - rel.sql.clone() + // Legacy column-name SQL leaves the keyed join intact. + // Custom predicates use the same placeholder contract + // as semantic-input decoding and the Python graph. + rel.sql + .as_ref() + .filter(|sql| sql.contains("{from}") || sql.contains("{to}")) + .cloned() }, rel.edge_id.clone(), )); @@ -894,6 +920,7 @@ impl SemanticGraph { .sql .as_ref() .filter(|_| rel.r#type != RelationshipType::Cross) + .filter(|sql| sql.contains("{from}") || sql.contains("{to}")) .map(|sql| { sql.replace("{from}", "__TEMP__") .replace("{to}", "{from}") @@ -903,12 +930,10 @@ impl SemanticGraph { let fk_keys = rel.foreign_key_columns(); let pk_keys = if rel.primary_key.is_some() || rel.primary_key_columns.is_some() { rel.primary_key_columns() - } else if (instance != canonical || rel.target_model.is_some()) - && matches!( - rel.r#type, - RelationshipType::OneToMany | RelationshipType::OneToOne - ) - { + } else if matches!( + rel.r#type, + RelationshipType::OneToMany | RelationshipType::OneToOne + ) { model.primary_keys() } else { self.models @@ -926,15 +951,10 @@ impl SemanticGraph { (fk_keys.clone(), model.primary_keys()) } } - RelationshipType::OneToOne - if instance != canonical || rel.target_model.is_some() => - { + RelationshipType::ManyToOne => (pk_keys.clone(), fk_keys.clone()), + RelationshipType::OneToMany | RelationshipType::OneToOne => { (fk_keys.clone(), pk_keys.clone()) } - RelationshipType::ManyToOne | RelationshipType::OneToOne => { - (pk_keys.clone(), fk_keys.clone()) - } - RelationshipType::OneToMany => (fk_keys.clone(), pk_keys.clone()), }; self.adjacency @@ -1067,24 +1087,27 @@ impl SemanticGraph { let model_name = parts[0]; let field_with_granularity = parts[1]; - // Check for granularity suffix (e.g., order_date__month) - let (field_name, granularity) = - if let Some((field, gran)) = field_with_granularity.rsplit_once("__") { - if field.is_empty() || gran.is_empty() { - return Err(SidemanticError::InvalidReference { - reference: reference.to_string(), - }); - } - (field.to_string(), Some(gran.to_string())) - } else { - (field_with_granularity.to_string(), None) - }; - - // Verify model exists - if self.get_model(model_name).is_none() { + let model = self.get_model(model_name).ok_or_else(|| { let available: Vec<&str> = self.models.keys().map(|s| s.as_str()).collect(); - return Err(SidemanticError::model_not_found(model_name, &available)); - } + SidemanticError::model_not_found(model_name, &available) + })?; + + // Exact public fields win over the granularity syntax, including names + // resembling internal helpers such as __fanout_rank_0. + let (field_name, granularity) = if model.get_metric(field_with_granularity).is_some() + || model.get_dimension(field_with_granularity).is_some() + { + (field_with_granularity.to_string(), None) + } else if let Some((field, gran)) = field_with_granularity.rsplit_once("__") { + if field.is_empty() || gran.is_empty() { + return Err(SidemanticError::InvalidReference { + reference: reference.to_string(), + }); + } + (field.to_string(), Some(gran.to_string())) + } else { + (field_with_granularity.to_string(), None) + }; Ok((model_name.to_string(), field_name, granularity)) } @@ -1098,6 +1121,37 @@ mod tests { }; use crate::core::parameter::{Parameter, ParameterType}; + #[test] + fn exact_field_names_take_precedence_over_granularity_suffixes() { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("orders", "id") + .with_table("orders") + .with_metric(Metric::count("__fanout_rank_0")) + .with_metric(Metric::count("__sidemantic_filtered_2")) + .with_dimension(Dimension::categorical("__sidemantic_filtered_1_raw")) + .with_dimension(Dimension::categorical("literal__month")) + .with_dimension(Dimension::time("day")), + ) + .unwrap(); + for name in [ + "__fanout_rank_0", + "__sidemantic_filtered_2", + "__sidemantic_filtered_1_raw", + "literal__month", + ] { + assert_eq!( + graph.parse_reference(&format!("orders.{name}")).unwrap(), + ("orders".into(), name.into(), None) + ); + } + assert_eq!( + graph.parse_reference("orders.day__month").unwrap(), + ("orders".into(), "day".into(), Some("month".into())) + ); + } + fn role(name: &str, target: &str, key: &str) -> Relationship { let mut relationship = Relationship::many_to_one(name).with_keys(key, "id"); relationship.target_model = Some(target.into()); @@ -1256,7 +1310,11 @@ mod tests { assert_eq!(path.steps[1].from_keys, vec![key.to_string()]); assert_eq!(graph.get_model(&bridge).unwrap().name, "links"); assert_eq!(graph.role_root_owner(&bridge), Some("orders")); + assert!(graph.is_bridge_instance(&bridge)); + assert!(!graph.is_bridge_instance(name)); } + assert!(!graph.is_bridge_instance("links")); + assert!(!graph.is_bridge_instance("orders")); } #[test] @@ -1692,6 +1750,35 @@ mod tests { assert_eq!(path.steps[0].to_keys, vec!["customer_uid".to_string()]); } + #[test] + fn one_to_one_uses_local_key_and_remote_foreign_key_in_both_directions() { + for explicit_key in [false, true] { + let mut graph = SemanticGraph::new(); + let mut relationship = Relationship::new("regions"); + relationship.r#type = RelationshipType::OneToOne; + relationship.foreign_key = Some("region_record_id".into()); + if explicit_key { + relationship.primary_key = Some("region_id".into()); + } + graph + .add_model( + Model::new("sales", "region_id") + .with_table("sales") + .with_relationship(relationship), + ) + .unwrap(); + graph + .add_model(Model::new("regions", "region_record_id").with_table("regions")) + .unwrap(); + let forward = graph.find_join_path("sales", "regions").unwrap(); + assert_eq!(forward.steps[0].from_keys, vec!["region_id"]); + assert_eq!(forward.steps[0].to_keys, vec!["region_record_id"]); + let reverse = graph.find_join_path("regions", "sales").unwrap(); + assert_eq!(reverse.steps[0].from_keys, vec!["region_record_id"]); + assert_eq!(reverse.steps[0].to_keys, vec!["region_id"]); + } + } + #[test] fn test_one_to_one_omitted_key_defaults_to_id() { let mut graph = SemanticGraph::new(); @@ -1745,6 +1832,33 @@ mod tests { assert_eq!(path.fan_out_boundary(), Some("orders")); } + #[test] + fn legacy_relationship_sql_preserves_forward_and_reverse_keys() { + let mut graph = SemanticGraph::new(); + let mut relationship = Relationship::many_to_one("customers").with_condition("customer_id"); + relationship.foreign_key = Some("customer_id".into()); + graph + .add_model( + Model::new("orders", "order_id") + .with_table("orders") + .with_relationship(relationship), + ) + .unwrap(); + graph + .add_model(Model::new("customers", "customer_key").with_table("customers")) + .unwrap(); + for (from, to, from_key, to_key) in [ + ("orders", "customers", "customer_id", "customer_key"), + ("customers", "orders", "customer_key", "customer_id"), + ] { + let path = graph.find_join_path(from, to).unwrap(); + assert_eq!(path.steps.len(), 1); + assert!(path.steps[0].custom_condition.is_none()); + assert_eq!(path.steps[0].from_keys, vec![from_key]); + assert_eq!(path.steps[0].to_keys, vec![to_key]); + } + } + #[test] fn test_custom_join_condition() { let mut graph = SemanticGraph::new(); @@ -1844,6 +1958,8 @@ mod tests { let path = graph.find_join_path("orders", "products").unwrap(); assert_eq!(path.steps.len(), 2); + assert!(graph.is_bridge_instance("order_items")); + assert!(!graph.is_bridge_instance("products")); // orders -> order_items assert_eq!(path.steps[0].from_model, "orders"); diff --git a/sidemantic-rs/src/core/key_expression.rs b/sidemantic-rs/src/core/key_expression.rs index 6369a76cf..38bc2636b 100644 --- a/sidemantic-rs/src/core/key_expression.rs +++ b/sidemantic-rs/src/core/key_expression.rs @@ -42,6 +42,19 @@ fn deterministic_scalar(expression: &Expression) -> bool { && deterministic_scalar(&concat.expression) } Expression::Coalesce(arguments) => arguments.expressions.iter().all(deterministic_scalar), + Expression::DateTrunc(trunc) | Expression::TimestampTrunc(trunc) => { + deterministic_scalar(&trunc.this) + } + Expression::Function(function) + if !function.quoted + && function.name.eq_ignore_ascii_case("DATE_TRUNC") + && function.args.len() == 2 => + { + matches!( + &function.args[0], + Expression::Literal(polyglot_sql::expressions::Literal::String(_)) + ) && deterministic_scalar(&function.args[1]) + } _ => false, } } @@ -154,6 +167,26 @@ mod tests { use super::*; use crate::core::Dimension; + #[test] + fn date_bucket_keys_keep_physical_input_scope() { + let model = Model::new("monthly_sales", "month") + .with_dimension(Dimension::new("month").with_sql("DATE_TRUNC('month', order_date)")); + let expression = key_expression(&model, "month", Some("s"), DialectType::DuckDB).unwrap(); + let sql = polyglot_sql::generate(&expression, DialectType::DuckDB).unwrap(); + assert!( + sql.to_ascii_uppercase() + .contains("DATE_TRUNC('MONTH', S.ORDER_DATE)"), + "{sql}" + ); + assert!(is_computed_key(&model, "month").unwrap()); + for input in ["random()", "SUM(order_date)", "other.order_date"] { + let model = Model::new("monthly_sales", "month").with_dimension( + Dimension::new("month").with_sql(format!("DATE_TRUNC('month', {input})")), + ); + assert!(key_expression(&model, "month", None, DialectType::DuckDB).is_err()); + } + } + #[test] fn key_classification_uses_resolved_direction_and_composite_defaults() { use crate::core::{Relationship, RelationshipType}; diff --git a/sidemantic-rs/src/core/mod.rs b/sidemantic-rs/src/core/mod.rs index b206e03ff..b06202f11 100644 --- a/sidemantic-rs/src/core/mod.rs +++ b/sidemantic-rs/src/core/mod.rs @@ -13,6 +13,7 @@ mod segment; pub mod symmetric_agg; mod table_calc; +pub(crate) use dependency::is_aggregate_ast_node; pub use dependency::{ check_circular_dependencies, extract_column_references_from_expr, extract_dependencies, extract_dependencies_with_context, outer_semantic_column_references, parse_semantic_expression, @@ -30,7 +31,7 @@ pub use model::{ pub use parameter::{Parameter, ParameterType}; pub use policy::{AccessRule, PolicyError, PreparedPolicies, SecurityPolicy}; pub(crate) use preaggregation::{ - materialization_sql as preaggregation_materialization_sql, + materialization_sql as preaggregation_materialization_sql, replace_model_placeholder, source_expression as preaggregation_source_expression, }; pub use relative_date::RelativeDate; diff --git a/sidemantic-rs/src/core/model.rs b/sidemantic-rs/src/core/model.rs index 52e613174..fddcfb419 100644 --- a/sidemantic-rs/src/core/model.rs +++ b/sidemantic-rs/src/core/model.rs @@ -190,6 +190,7 @@ pub enum ComparisonType { Wow, // Week over week Dod, // Day over day Qoq, // Quarter over quarter + #[serde(alias = "prior_period")] PriorPeriod, } @@ -1137,6 +1138,15 @@ impl Model { mod tests { use super::*; + #[test] + fn prior_period_accepts_python_and_legacy_spellings() { + for name in ["prior_period", "priorperiod"] { + let comparison: ComparisonType = + serde_json::from_value(serde_json::json!(name)).unwrap(); + assert_eq!(comparison, ComparisonType::PriorPeriod); + } + } + #[test] fn test_dimension_sql_expr() { let dim = Dimension::new("status"); diff --git a/sidemantic-rs/src/core/preaggregation.rs b/sidemantic-rs/src/core/preaggregation.rs index b768052bb..12540d3a3 100644 --- a/sidemantic-rs/src/core/preaggregation.rs +++ b/sidemantic-rs/src/core/preaggregation.rs @@ -162,7 +162,7 @@ pub(crate) fn materialization_sql( } /// Rewrite only actual placeholder tokens, leaving literal and subquery text intact. -fn replace_model_placeholder(sql: &str, owner: Option<&str>) -> Result { +pub(crate) fn replace_model_placeholder(sql: &str, owner: Option<&str>) -> Result { use polyglot_sql::{dialects::Dialect, DialectType, TokenType}; let tokens = Dialect::get(DialectType::DuckDB) .tokenize(sql) diff --git a/sidemantic-rs/src/lib.rs b/sidemantic-rs/src/lib.rs index b277c3dc3..12b01ea19 100644 --- a/sidemantic-rs/src/lib.rs +++ b/sidemantic-rs/src/lib.rs @@ -105,13 +105,13 @@ pub use wasm::{ wasm_analyze_migrator_query, wasm_build_preaggregation_refresh_statements, wasm_build_symmetric_aggregate_sql, wasm_calculate_preaggregation_benefit_score, wasm_chart_auto_detect_columns, wasm_chart_encoding_type, wasm_chart_format_label, - wasm_chart_select_type, wasm_compile_with_yaml_query, wasm_detect_adapter_kind, - wasm_dimension_sql_expr_with_yaml, wasm_dimension_with_granularity_with_yaml, - wasm_evaluate_table_calculation_expression, wasm_extract_column_references, - wasm_extract_metric_dependencies_from_yaml, wasm_extract_preaggregation_patterns, - wasm_find_models_for_query, wasm_find_relationship_path_with_yaml, - wasm_format_parameter_value_with_yaml, wasm_generate_catalog_metadata_with_yaml, - wasm_generate_preaggregation_definition, + wasm_chart_select_type, wasm_compile_with_semantic_input, wasm_compile_with_yaml_query, + wasm_detect_adapter_kind, wasm_dimension_sql_expr_with_yaml, + wasm_dimension_with_granularity_with_yaml, wasm_evaluate_table_calculation_expression, + wasm_extract_column_references, wasm_extract_metric_dependencies_from_yaml, + wasm_extract_preaggregation_patterns, wasm_find_models_for_query, + wasm_find_relationship_path_with_yaml, wasm_format_parameter_value_with_yaml, + wasm_generate_catalog_metadata_with_yaml, wasm_generate_preaggregation_definition, wasm_generate_preaggregation_materialization_sql_with_yaml, wasm_generate_preaggregation_name, wasm_generate_time_comparison_sql, wasm_interpolate_sql_with_parameters_with_yaml, wasm_is_relative_date, wasm_is_sql_template, wasm_load_graph_with_sql, @@ -128,7 +128,8 @@ pub use wasm::{ wasm_relationship_primary_key_columns_with_yaml, wasm_relationship_related_key_with_yaml, wasm_relationship_sql_expr_with_yaml, wasm_relative_date_to_range, wasm_render_sql_template, wasm_resolve_metric_inheritance, wasm_resolve_model_inheritance_with_yaml, - wasm_rewrite_with_yaml, wasm_segment_get_sql_with_yaml, wasm_summarize_preaggregation_patterns, + wasm_rewrite_with_semantic_input_context, wasm_rewrite_with_yaml, + wasm_segment_get_sql_with_yaml, wasm_summarize_preaggregation_patterns, wasm_time_comparison_offset_interval, wasm_time_comparison_sql_offset, wasm_trailing_period_sql_interval, wasm_validate_engine_refresh_sql_compatibility, wasm_validate_metric_payload, wasm_validate_model_payload, wasm_validate_models_yaml, diff --git a/sidemantic-rs/src/python.rs b/sidemantic-rs/src/python.rs index b5a63d2f6..add971dca 100644 --- a/sidemantic-rs/src/python.rs +++ b/sidemantic-rs/src/python.rs @@ -104,7 +104,7 @@ static REGISTRY_CONTEXTVAR: PyOnceLock> = PyOnceLock::new(); pyo3::create_exception!( sidemantic_rs, UnsupportedSemanticFeaturesError, - PyRuntimeError + PyValueError ); pyo3::create_exception!(sidemantic_rs, SecurityError, PyRuntimeError); pyo3::create_exception!(sidemantic_rs, QueryValidationError, PyValueError); @@ -124,9 +124,10 @@ fn semantic_input_error(py: Python<'_>, error: SidemanticError) -> PyErr { } error } - SidemanticError::Validation(_) | SidemanticError::ValidationIssue { .. } => { - QueryValidationError::new_err(error.to_string()) - } + SidemanticError::Validation(_) + | SidemanticError::ValidationIssue { .. } + | SidemanticError::NoJoinPath { .. } + | SidemanticError::SqlParse(_) => QueryValidationError::new_err(error.to_string()), SidemanticError::InvalidConfig(_) => PyValueError::new_err(error.to_string()), _ => PyRuntimeError::new_err(error.to_string()), } @@ -515,12 +516,14 @@ fn validate_query_references( /// Generate materialization SQL for a model pre-aggregation using sidemantic-rs schema. #[pyfunction] fn generate_preaggregation_materialization_sql( + py: Python<'_>, yaml: &str, model_name: &str, preagg_name: &str, ) -> PyResult { generate_preaggregation_materialization_sql_with_yaml_native(yaml, model_name, preagg_name) .map_err(|e| match e { + SidemanticError::UnsupportedSemanticFeatures { .. } => semantic_input_error(py, e), SidemanticError::Validation(_) | SidemanticError::YamlParse(_) | SidemanticError::InvalidConfig(_) diff --git a/sidemantic-rs/src/runtime.rs b/sidemantic-rs/src/runtime.rs index e3eb4b6f5..08bc44444 100644 --- a/sidemantic-rs/src/runtime.rs +++ b/sidemantic-rs/src/runtime.rs @@ -5804,6 +5804,12 @@ pub fn validate_query_references( let mut errors = Vec::new(); for metric_ref in metrics { + // Graph metric names are opaque, and may themselves contain dots. + if graph.get_metric(metric_ref).is_some() + || context.top_level_metric_names.contains(metric_ref) + { + continue; + } if let Some((model_name, metric_name)) = metric_ref.split_once('.') { if graph.get_model(model_name).is_none() { errors.push(format!( @@ -5852,11 +5858,10 @@ pub fn validate_query_references( )); continue; } - if graph - .get_model(model_name) - .and_then(|model| model.get_dimension(dim_name)) - .is_none() - { + if graph.get_model(model_name).is_some_and(|model| { + model.get_dimension(dim_name).is_none() + && !crate::core::semantic_key_names(graph, model).contains(dim_name) + }) { errors.push(format!( "Dimension '{dim_name}' not found in model '{model_name}' (referenced in '{dim_ref_for_lookup}')" )); @@ -5870,6 +5875,16 @@ pub fn validate_query_references( let mut model_names: BTreeSet = BTreeSet::new(); for metric_ref in metrics { + if graph.get_metric(metric_ref).is_some() + || context.top_level_metric_names.contains(metric_ref) + { + if let Some(sql_ref) = context.top_level_metric_sql_refs.get(metric_ref) { + if let Some((model_name, _)) = sql_ref.split_once('.') { + model_names.insert(model_name.to_string()); + } + } + continue; + } if let Some((model_name, _)) = metric_ref.split_once('.') { model_names.insert(model_name.to_string()); continue; @@ -5919,6 +5934,51 @@ mod tests { use super::*; use std::time::{SystemTime, UNIX_EPOCH}; + #[test] + fn validation_resolves_exact_graph_names_and_declared_relationship_keys() { + let yaml = r#" +models: + - name: orders + table: orders + primary_key: id + metrics: + - name: revenue + agg: sum + sql: amount + - name: order_count + agg: count + relationships: + - name: customers + type: many_to_one + foreign_key: customer_id + - name: customers + table: customers + primary_key: id +metrics: + - name: finance.revenue_per_order + type: ratio + numerator: orders.revenue + denominator: orders.order_count + - name: company.sales.revenue + sql: orders.revenue +"#; + let runtime = SidemanticRuntime::from_yaml(yaml).unwrap(); + for metric in ["finance.revenue_per_order", "company.sales.revenue"] { + let errors = + runtime.validate_query_references(&[metric.into()], &["orders.customer_id".into()]); + assert!(errors.is_empty(), "{errors:?}"); + } + let errors = runtime.validate_query_references( + &["company.sales.missing".into()], + &["orders.unknown_key".into()], + ); + assert_eq!(errors.len(), 2, "{errors:?}"); + assert!(errors.iter().any(|error| error.contains("unknown_key"))); + assert!(errors + .iter() + .any(|error| error.contains("company.sales.missing"))); + } + #[test] fn test_export_osi_yaml_accepts_out_of_order_graph_metrics() { // A ratio metric listed before the graph metrics it references must not diff --git a/sidemantic-rs/src/runtime/parameters.rs b/sidemantic-rs/src/runtime/parameters.rs index a808b829f..3d243b31b 100644 --- a/sidemantic-rs/src/runtime/parameters.rs +++ b/sidemantic-rs/src/runtime/parameters.rs @@ -91,6 +91,42 @@ fn string_literal(value: &str, dialect: DialectType) -> std::result::Result std::result::Result { + let text = text + .strip_prefix("E:") + .or_else(|| text.strip_prefix("e:")) + .ok_or("Invalid SQL escape-string token")?; + let mut characters = text.chars(); + let mut result = String::new(); + while let Some(character) = characters.next() { + if character != '\\' { + result.push(character); + continue; + } + let escaped = characters.next().ok_or("Invalid SQL escape-string token")?; + let decoded = match escaped { + '\\' | '\'' => escaped, + 'a' => '\u{0007}', + 'b' => '\u{0008}', + 'f' => '\u{000c}', + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + 'v' => '\u{000b}', + other => { + // SQLGlot preserves escapes outside its recognized table. + result.push('\\'); + other + } + }; + result.push(decoded); + } + Ok(result) +} + /// Wrap bare-name output expressions so declared string/unquoted/yesno types /// remain available to the formatter. Tokenization leaves conditions, complex /// expressions, quoted Jinja strings, comments and raw blocks untouched. @@ -223,20 +259,24 @@ fn replace_outputs( .ok_or("Invalid SQL token span")?; if matches!( token.token_type, - TokenType::String | TokenType::DollarString | TokenType::ByteString + TokenType::String + | TokenType::DollarString + | TokenType::ByteString + | TokenType::EscapeString ) { let raw = sql.get(start..end).ok_or("Invalid SQL token span")?; - if token.token_type == TokenType::ByteString - || matches!( - dialect, - DialectType::MySQL - | DialectType::BigQuery - | DialectType::Snowflake - | DialectType::Spark - | DialectType::Databricks - | DialectType::Hive - ) - { + if matches!( + token.token_type, + TokenType::ByteString | TokenType::EscapeString + ) || matches!( + dialect, + DialectType::MySQL + | DialectType::BigQuery + | DialectType::Snowflake + | DialectType::Spark + | DialectType::Databricks + | DialectType::Hive + ) { let prefix = &sql[start..position]; if !(prefix.len() - prefix.trim_end_matches('\\').len()).is_multiple_of(2) { return Err( @@ -247,22 +287,28 @@ fn replace_outputs( let value = match literals.entry((start, end)) { std::collections::btree_map::Entry::Occupied(entry) => entry.into_mut(), std::collections::btree_map::Entry::Vacant(entry) => { - let normalized = dialects::fragment(raw, dialect, dialects::Fragment::Scalar) - .map_err(|error| error.to_string())?; - let expression = crate::core::parse_semantic_expression(&normalized) - .map_err(|error| error.to_string())?; - let text = match expression { - Expression::Literal(Literal::String(text)) => text, - Expression::Literal(Literal::DollarString(text)) => { - polyglot_sql::tokens::parse_dollar_string_token(&text).1 - } - _ => { - return Err( - "Parameter output requires an ordinary SQL string literal".into() - ) - } - }; - entry.insert(text) + if token.token_type == TokenType::EscapeString { + entry.insert(escape_string_text(&token.text)?) + } else { + let normalized = + dialects::fragment(raw, dialect, dialects::Fragment::Scalar) + .map_err(|error| error.to_string())?; + let expression = crate::core::parse_semantic_expression(&normalized) + .map_err(|error| error.to_string())?; + let text = match expression { + Expression::Literal(Literal::String(text)) => text, + Expression::Literal(Literal::DollarString(text)) => { + polyglot_sql::tokens::parse_dollar_string_token(&text).1 + } + _ => { + return Err( + "Parameter output requires an ordinary SQL string literal" + .into(), + ) + } + }; + entry.insert(text) + } } }; *value = value.replace(&output.marker, &output.text); @@ -291,6 +337,45 @@ mod tests { serde_json::from_value(json!({"name":name, "type":kind})).unwrap() } + #[test] + fn escape_string_outputs_keep_static_escapes_separate_from_values() { + for kind in ["string", "date"] { + let parameter = parameter("value", kind); + let definitions = HashMap::from([("value".to_owned(), ¶meter)]); + for dialect in [DialectType::DuckDB, DialectType::PostgreSQL] { + for value in ["signup", "\\' OR 1=1 -- ", "O'Reilly\\folder"] { + let values = HashMap::from([( + "value".to_owned(), + serde_yaml::Value::String(value.into()), + )]); + for (template, expected) in [ + ("{# c #}E'{{ value }}'", value.to_owned()), + ( + r"{# c #}e'é prefix\'{{ value }}'", + format!("é prefix'{value}"), + ), + ( + r"{# c #}E'line\n{{ value }}\\{{ value }}'", + format!("line\n{value}\\{value}"), + ), + ] { + let rendered = render(template, &definitions, &values, dialect).unwrap(); + assert_eq!(rendered, string_literal(&expected, dialect).unwrap()); + let expression = crate::core::parse_semantic_expression(&rendered).unwrap(); + assert_eq!(expression, Expression::Literal(Literal::String(expected))); + } + } + let values = + HashMap::from([("value".to_owned(), serde_yaml::Value::String("ok".into()))]); + for template in [r"{# c #}E'\{{ value }}'", r"{# c #}e'\\\{{ value }}'"] { + assert!(render(template, &definitions, &values, dialect) + .unwrap_err() + .contains("unpaired SQL escape")); + } + } + } + } + #[test] fn output_literals_preserve_quotes_backslashes_and_quoted_context() { let parameter = parameter("value", "string"); diff --git a/sidemantic-rs/src/semantic_input.rs b/sidemantic-rs/src/semantic_input.rs index f0df6a586..41b9bf2fb 100644 --- a/sidemantic-rs/src/semantic_input.rs +++ b/sidemantic-rs/src/semantic_input.rs @@ -20,6 +20,8 @@ use crate::sql::{QueryRewriter, SemanticQuery, SqlGenerator}; mod calculations; mod dates; pub(crate) mod dialects; +mod fragments; +mod inheritance; mod literals; mod policies; @@ -177,13 +179,20 @@ fn expression_language(raw: &mut Map, path: &str) -> Result<()> { } } } + // DAX source is provenance once the host has supplied an executable SQL, + // table, or simple aggregate translation. Never run untranslated DAX here. + let translated = ["sql", "table", "agg"].iter().any(|field| { + raw.get(*field) + .and_then(Value::as_str) + .is_some_and(|value| !value.is_empty()) + }); if let Some(value) = raw.remove("dax") { - if !value.is_null() { + if !value.is_null() && !translated { return Err(unsupported(format!("{path}.dax"))); } } if let Some(value) = raw.remove("expression_language") { - if !value.is_null() && value != "sql" { + if !value.is_null() && value != "sql" && !(value == "dax" && translated) { return Err(unsupported(format!("{path}.expression_language"))); } } @@ -517,6 +526,16 @@ fn decode_metric( }; raw.insert("type".into(), json!(kind)); } + // Imported definitions may retain `derived` alongside a concrete agg. + // The aggregation still consumes physical row inputs, as in Python. + if raw.get("type").and_then(Value::as_str) == Some("derived") + && raw + .get("agg") + .and_then(Value::as_str) + .is_some_and(|agg| agg != "expression") + { + raw.insert("type".into(), json!("simple")); + } if complete && raw.get("agg").is_some_and(|value| !value.is_null()) { return Err(invalid( path, @@ -552,7 +571,10 @@ fn decode_metric( exemplar.logical_data_type = Some(String::new()); exemplar.sql_is_complete = true; let metric = project(raw, exemplar, path)?; - if metric.agg == Some(crate::core::Aggregation::ApproxCountDistinct) && !model_local { + if metric.agg == Some(crate::core::Aggregation::ApproxCountDistinct) + && !model_local + && !(metric.r#type == crate::core::MetricType::Cohort && owner.is_some()) + { return Err(unsupported("metric.approx_count_distinct_model_scope")); } if metric.non_additive_dimension.is_some() { @@ -573,6 +595,16 @@ fn decode_metric( fn decode_relationship(value: Value, path: &str) -> Result { let mut raw = object(value, path)?; + // Python's native relationship contract reserves SQL join predicates for + // {from}/{to} expressions. A legacy SQL field such as "id" does not + // replace the declared FK/PK join with a bare boolean expression. + if raw + .get("sql") + .and_then(Value::as_str) + .is_some_and(|sql| !sql.contains("{from}") && !sql.contains("{to}")) + { + raw.remove("sql"); + } if raw.get("type") == Some(&json!("many_to_many")) && raw.get("through").is_none_or(Value::is_null) && raw.get("foreign_key").is_none_or(Value::is_null) @@ -603,14 +635,12 @@ fn decode_relationship(value: Value, path: &str) -> Result { fn decode_model(value: Value, path: &str) -> Result { let mut raw = object(value, path)?; expression_language(&mut raw, path)?; - for (field, capability) in [ - ("schema_exposure", "model.schema_exposure"), - ("auto_dimensions", "model.auto_dimensions"), - ("extends", "model.inheritance"), - ] { - reject_active(&mut raw, field, capability)?; - } + reject_active(&mut raw, "extends", "model.inheritance")?; // These declarations are retained and checked separately from core models. + // Schema exposure is applied by host introspection before dimensions enter + // the handoff. Native compilation uses only those explicit dimensions. + raw.remove("schema_exposure"); + raw.remove("auto_dimensions"); raw.remove("security"); raw.remove("invariant_filters"); if let Some(value) = raw.remove("pre_aggregations") { @@ -699,7 +729,7 @@ fn decode_model(value: Value, path: &str) -> Result { .as_deref() .is_none_or(|sql| sql.is_empty() || sql == "*") { - if primary_keys.len() != 1 { + if primary_keys.is_empty() { return Err(unsupported("metric.count_distinct_primary_key")); } // Keep the default distinct input distinct from explicit raw SQL. @@ -731,16 +761,65 @@ fn validate_semantic_dependencies(graph: &SemanticGraph, graph_metrics: &[Metric let mut dependencies: HashMap> = HashMap::new(); for (name, metric, context) in definitions { let mut metric_dependencies = Vec::new(); + let source_qualifier = context + .and_then(|owner| graph.get_model(owner)) + .filter(|model| model.sql.is_none()) + .and_then(|model| model.table.as_deref()) + .map(|table| dialects::parse(&format!("SELECT * FROM {table}"), DialectType::DuckDB)) + .transpose()? + .and_then(|expression| match expression { + polyglot_sql::Expression::Select(select) => select.from, + _ => None, + }) + .and_then(|from| from.expressions.into_iter().next()) + .and_then(|expression| match expression { + polyglot_sql::Expression::Table(table) => Some(table.name.name), + _ => None, + }); + let deferred_input_aliases: Vec<&str> = metric + .metadata + .as_ref() + .and_then(|metadata| metadata.get("input_metrics")) + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter(|input| { + ["offset_window", "offset_to_grain", "filter"] + .iter() + .any(|field| input.get(*field).is_some_and(|value| !value.is_null())) + }) + .filter_map(|input| input.get("alias").and_then(Value::as_str)) + .collect(); + if metric.r#type == crate::core::MetricType::Cumulative { + // Cumulative SQL is a base metric reference, not a scalar SQL + // expression. Imports can retain unresolved bases; the temporal + // generator resolves them when selected. Keep known edges here so + // cycles remain invalid even before a query is compiled. + if let Some(reference) = metric.sql.as_ref().or(metric.base_metric.as_ref()) { + let mut dependency = reference.clone(); + if let Some(model_name) = context { + if graph + .get_model(model_name) + .is_some_and(|model| model.get_metric(reference).is_some()) + { + dependency = format!("{model_name}.{reference}"); + } + } + metric_dependencies.push(dependency); + } + } // `base` is the generated period-output relation, not a model. Resolve // its metric name with the same ownership and cycle rules as other refs. let window_dependency = SqlGenerator::window_output_dependency(metric)?; for expression in [ // Cohort SQL consumes inner-result aliases; the cohort generator // validates that separate namespace instead of metric dependencies. - metric - .sql - .as_deref() - .filter(|_| metric.r#type != crate::core::MetricType::Cohort), + metric.sql.as_deref().filter(|_| { + !matches!( + metric.r#type, + crate::core::MetricType::Cohort | crate::core::MetricType::Cumulative + ) + }), metric.numerator.as_deref(), metric.denominator.as_deref(), window_dependency.as_deref(), @@ -748,21 +827,42 @@ fn validate_semantic_dependencies(graph: &SemanticGraph, graph_metrics: &[Metric .into_iter() .flatten() { - for column in crate::core::semantic_column_references(expression)? { - let model_name = column.model.as_deref().or(context); + let expression = crate::core::replace_model_placeholder(expression, context)?; + for column in crate::core::semantic_column_references(&expression)? { + // MetricFlow retains modified input aliases for round-trip; + // they are not ordinary metric definitions. Unrelated queries + // remain valid, while selected derived SQL still resolves every + // input through the generator and rejects unsupported aliases. + if column.model.is_none() && deferred_input_aliases.contains(&column.field.as_str()) + { + continue; + } + let physical_input = metric.r#type == crate::core::MetricType::Simple + || metric.agg.is_some() + || metric.sql_is_complete + || column.aggregate_input; + // Complete aggregates may use the generated source CTE qualifier. + // Resolve that alias only for physical inputs, and prefer a real + // model of that name when one exists. + let model_name = column.model.as_deref().or(context).map(|name| { + if physical_input && graph.get_model(name).is_none() { + if source_qualifier.as_deref() == Some(name) { + context.unwrap_or(name) + } else { + name.strip_suffix("_cte").unwrap_or(name) + } + } else { + name + } + }); if let Some(model_name) = model_name { - let model = graph + graph .get_model(model_name) .ok_or_else(|| invalid(&name, format!("unknown model '{model_name}'")))?; - if column.aggregate_input - && model - .get_dimension(&column.field) - .is_some_and(|dimension| dimension.sql_expr() != column.field) - { - return Err(unsupported("metric.raw_computed_column")); - } + // Complete aggregate SQL names physical columns. A semantic + // dimension with the same name does not redefine that input. } - if metric.r#type == crate::core::MetricType::Simple || column.aggregate_input { + if physical_input { continue; } let dependency = if let Some(model_name) = &column.model { @@ -854,6 +954,7 @@ impl SemanticInput { return Err(invalid("version", "supported semantic input version is 1")); } let input_dialect = dialects::parse_dialect(&envelope.input_dialect)?; + envelope.models = inheritance::resolve(&envelope.models)?; let policies = policies::decode_with_dialect(&envelope.models, input_dialect)?; let deferred_segment_dialects = dialects::normalize(&mut envelope, input_dialect)?; let unsupported_capabilities: Vec = envelope @@ -877,6 +978,8 @@ impl SemanticInput { | "relationship.inactive" | "model.security" | "model.invariant_filters" + | "model.schema_exposure" + | "preaggregation.lambda" | "visibility" ) }) @@ -928,10 +1031,10 @@ impl SemanticInput { continue; } if !keys.contains_key(relationship.related_model()) { - return Err(invalid( - "relationships", - format!("unknown target {}", relationship.related_model()), - )); + // Models can be loaded independently of their join targets. + // The graph retains this declaration but only builds edges + // between loaded models; queries still resolve every reference. + continue; } if relationship.r#type == crate::core::RelationshipType::ManyToMany && relationship.through.is_some() @@ -985,9 +1088,13 @@ impl SemanticInput { // An explicit primary_key records a local/remote key pair. (foreign, primary) }; - if relationship.sql.is_none() - && (local.is_empty() || remote.is_empty() || local.len() != remote.len()) - { + if relationship.sql.is_none() && (local.is_empty() || remote.is_empty()) { + // An incomplete direct edge is not traversable, but it + // must not block queries against the local model. + relationship.active = false; + continue; + } + if relationship.sql.is_none() && local.len() != remote.len() { return Err(invalid( "relationships", "direct many-to-many join key arity mismatch", @@ -1015,8 +1122,7 @@ impl SemanticInput { if primary.is_empty() { let primary_model = if relationship.r#type == crate::core::RelationshipType::OneToMany - || (relationship.target_model.is_some() - && relationship.r#type == crate::core::RelationshipType::OneToOne) + || relationship.r#type == crate::core::RelationshipType::OneToOne { &model.name } else { @@ -1109,6 +1215,8 @@ struct QueryInput { #[serde(default)] use_preaggregations: bool, #[serde(default)] + allow_non_additive_unsafe: bool, + #[serde(default)] skip_default_time_dimensions: bool, preagg_database: Option, preagg_schema: Option, @@ -1140,12 +1248,45 @@ fn prepare_query_input(mut query: QueryInput, input: &mut SemanticInput) -> Resu ) .map_err(|error| invalid("query.parameter_values", error))? .iter() - .map(|sql| dialects::fragment(sql, dialect, dialects::Fragment::Scalar)) + .map(|sql| { + fragments::validate_request_expression(sql, dialect, false)?; + dialects::fragment(sql, dialect, dialects::Fragment::Scalar) + }) .collect::>>()?; + let known_names: Vec<_> = query + .aliases + .values() + .chain(query.metrics.iter()) + .chain(query.dimensions.iter()) + .map(String::as_str) + .collect(); query.order_by = query .order_by .iter() - .map(|sql| dialects::fragment(sql, dialect, dialects::Fragment::Order)) + .map(|sql| { + // Selected references and output aliases are names, not authored SQL. + // Bind before dialect parsing and policy collection so spaces and + // ordering keywords inside an alias retain their literal meaning. + let (field, suffix) = crate::sql::split_order_field(sql, &known_names); + // Keep canonical semantic names in the request. Specialized + // planners and table calculations bind these names before rendering + // SQL identifiers; quoting here changes the lookup key. + let render_reference = + |reference: &str| format!("{reference} {suffix}").trim_end().to_owned(); + if let Some((reference, _)) = query.aliases.iter().find(|(_, alias)| *alias == field) { + return Ok(render_reference(reference)); + } + if query + .metrics + .iter() + .chain(&query.dimensions) + .any(|name| name == field) + { + return Ok(render_reference(field)); + } + fragments::validate_request_expression(sql, dialect, true)?; + dialects::fragment(sql, dialect, dialects::Fragment::Order) + }) .collect::>>()?; let mut prepared_segments = std::collections::HashSet::new(); for reference in &query.segments { @@ -1235,6 +1376,7 @@ fn compile_semantic_input(input_json: &str, query_json: &str) -> Result offset: payload.offset, ungrouped: payload.ungrouped, use_preaggregations: payload.use_preaggregations, + allow_non_additive_unsafe: payload.allow_non_additive_unsafe, skip_default_time_dimensions: payload.skip_default_time_dimensions, preagg_database: payload.preagg_database, preagg_schema: payload.preagg_schema, @@ -1285,14 +1427,18 @@ fn validate_semantic_input(input_json: &str, query_json: &str) -> Result Result>, #[serde(default)] enforce_visibility: bool, + #[serde(default)] + use_preaggregations: bool, + #[serde(default)] + allow_non_additive_unsafe: bool, } pub fn rewrite_with_semantic_input_context( @@ -1404,6 +1555,8 @@ fn rewrite_semantic_input_diagnostics( .any(|policy| policy.security.is_some() || !policy.invariant_filters.is_empty()); let requires_policies = context.user_attributes.is_some() || security_controls; let prepare = |graph: &SemanticGraph, query: &mut SemanticQuery| { + query.use_preaggregations = context.use_preaggregations; + query.allow_non_additive_unsafe = context.allow_non_additive_unsafe; query.prepared_policies = policies::prepare_for_rewrite( graph, &input.policies, @@ -1419,7 +1572,7 @@ fn rewrite_semantic_input_diagnostics( // their source names too, before the rewriter allocates any user CTE. let policy_definitions = serde_json::to_string(&input.policies) .map_err(|error| invalid("rewrite.policy_definitions", error))?; - if requires_policies { + if requires_policies || context.use_preaggregations || context.allow_non_additive_unsafe { rewriter = rewriter.with_query_preparer(&prepare, &policy_definitions, security_controls); } @@ -1445,6 +1598,245 @@ mod tests { }) } + #[test] + fn unloaded_relationship_targets_do_not_block_local_queries() { + let mut source = input(); + source["models"][0]["relationships"] = json!([ + {"name":"customers", "type":"many_to_one", "foreign_key":"customer_id"} + ]); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.revenue"]}"#) + .unwrap(); + assert!(!sql.contains("JOIN"), "{sql}"); + assert!(compile_with_semantic_input( + &source.to_string(), + r#"{"metrics":["orders.revenue"],"dimensions":["customers.region"]}"#, + ) + .is_err()); + } + + #[test] + fn translated_dax_remains_provenance_without_blocking_sql() { + let mut source = input(); + source["models"][0]["dax"] = json!("FILTER(Orders, TRUE())"); + source["models"][0]["expression_language"] = json!("dax"); + source["models"][0]["dimensions"][0]["dax"] = json!("UPPER(Orders[status])"); + source["models"][0]["dimensions"][0]["expression_language"] = json!("dax"); + source["models"][0]["dimensions"][0]["sql"] = json!("UPPER(status)"); + source["models"][0]["metrics"][0]["dax"] = json!("SUM(Orders[amount])"); + source["models"][0]["metrics"][0]["expression_language"] = json!("dax"); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.revenue"]}"#) + .unwrap(); + assert!(sql.contains("SUM("), "{sql}"); + assert_eq!( + SemanticInput::from_json(&source.to_string()) + .unwrap() + .source, + source + ); + let metric = source["models"][0]["metrics"][0].as_object_mut().unwrap(); + metric.remove("sql"); + metric.remove("agg"); + assert!(matches!( + SemanticInput::from_json(&source.to_string()), + Err(SidemanticError::UnsupportedSemanticFeatures { .. }) + )); + } + + #[test] + fn incomplete_direct_many_to_many_does_not_block_local_queries() { + let mut source = input(); + source["models"][0]["relationships"] = + json!([{"name":"regions", "type":"many_to_many", "foreign_key":"order_id"}]); + source["models"].as_array_mut().unwrap().push(json!({ + "name":"regions", "table":"regions", "primary_key":"order_id", + "dimensions":[{"name":"label", "type":"categorical"}] + })); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.revenue"]}"#) + .unwrap(); + assert!(!sql.contains("JOIN"), "{sql}"); + assert!(compile_with_semantic_input( + &source.to_string(), + r#"{"metrics":["orders.revenue"],"dimensions":["regions.label"]}"# + ) + .is_err()); + assert_eq!( + SemanticInput::from_json(&source.to_string()) + .unwrap() + .source, + source + ); + } + + #[test] + fn dependency_validation_binds_model_tokens_without_rewriting_literals() { + let mut source = input(); + source["models"][0]["metrics"] = json!([ + {"name":"average", "sql":"sum({model}.amount) / count(*)", "sql_is_complete":true}, + {"name":"literal", "sql":"sum(CASE WHEN '{model}.missing' = 'x' THEN 1 ELSE 0 END)", "sql_is_complete":true} + ]); + let query = r#"{"metrics":["orders.average"]}"#; + assert!(validate_with_semantic_input(&source.to_string(), query) + .unwrap() + .is_empty()); + } + + #[test] + fn unresolved_cumulative_base_only_blocks_queries_that_select_it() { + let mut source = input(); + source["models"][0]["metrics"] + .as_array_mut() + .unwrap() + .push(json!({ + "name":"running", "type":"cumulative", "agg":"sum", "sql":"hours" + })); + source["models"][0]["dimensions"] + .as_array_mut() + .unwrap() + .push(json!({ + "name":"day", "type":"time", "granularity":"day" + })); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.revenue"]}"#) + .unwrap(); + assert!(sql.contains("SUM("), "{sql}"); + assert!(compile_with_semantic_input( + &source.to_string(), + r#"{"metrics":["orders.running"],"dimensions":["orders.day"]}"#, + ) + .is_err()); + source["models"][0]["metrics"] + .as_array_mut() + .unwrap() + .push(json!({ + "name":"hours", "type":"cumulative", "sql":"running" + })); + assert!(matches!( + SemanticInput::from_json(&source.to_string()), + Err(SidemanticError::CircularDependency(_)) + )); + } + + #[test] + fn deferred_imported_input_alias_does_not_block_unrelated_metrics() { + let mut source = input(); + source["models"][0]["metrics"].as_array_mut().unwrap().push(json!({ + "name":"growth", "type":"derived", "sql":"revenue - prior_revenue", + "metadata":{"input_metrics":[{"name":"revenue", "alias":"prior_revenue", "offset_window":"7 days"}]} + })); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.revenue"]}"#) + .unwrap(); + assert!(sql.contains("SUM("), "{sql}"); + assert!(compile_with_semantic_input( + &source.to_string(), + r#"{"metrics":["orders.growth"]}"# + ) + .is_err()); + assert_eq!( + SemanticInput::from_json(&source.to_string()) + .unwrap() + .source, + source + ); + } + + #[test] + fn caller_filters_cannot_borrow_trusted_segment_or_policy_scope() { + let predicate = "status IN (SELECT status FROM allowed_statuses)"; + for temporal in [false, true] { + let mut source = input(); + source["models"][0]["segments"] = json!([{"name":"allowed", "sql":predicate}]); + source["models"][0]["dimensions"] + .as_array_mut() + .unwrap() + .push(json!({ + "name":"day", "type":"time", "granularity":"day" + })); + source["models"][0]["metrics"] + .as_array_mut() + .unwrap() + .push(json!({ + "name":"running", "type":"cumulative", "sql":"orders.revenue" + })); + let mut query = if temporal { + json!({"metrics":["orders.running"], "dimensions":["orders.day"]}) + } else { + json!({"metrics":["orders.revenue"]}) + }; + query["segments"] = json!(["orders.allowed"]); + let sql = compile_with_semantic_input(&source.to_string(), &query.to_string()).unwrap(); + assert!(sql.contains("allowed_statuses"), "{sql}"); + query.as_object_mut().unwrap().remove("segments"); + source["models"][0]["invariant_filters"] = json!([predicate]); + let sql = compile_with_semantic_input(&source.to_string(), &query.to_string()).unwrap(); + assert!(sql.contains("allowed_statuses"), "{sql}"); + query["filters"] = json!([predicate]); + for result in [ + compile_with_semantic_input(&source.to_string(), &query.to_string()).map(|_| ()), + validate_with_semantic_input(&source.to_string(), &query.to_string()).map(|_| ()), + ] { + let error = result.unwrap_err(); + assert!( + error.to_string().contains("physical data sources"), + "{error}" + ); + } + } + } + + #[test] + fn complete_ordered_set_aggregate_uses_physical_sort_input() { + let mut source = input(); + source["models"][0]["metrics"] = json!([{ + "name":"p95", "sql_is_complete":true, + "sql":"PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY orders.amount)" + }]); + let sql = compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.p95"]}"#) + .unwrap(); + assert!( + sql.contains("WITHIN GROUP") || sql.contains("QUANTILE_CONT"), + "{sql}" + ); + assert!(sql.contains("amount"), "{sql}"); + } + + #[test] + fn physical_metric_input_can_use_its_declared_source_table_name() { + let mut source = input(); + source["models"][0]["table"] = json!("sales"); + source["models"][0]["metrics"][0]["sql"] = json!("sales.amount"); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.revenue"]}"#) + .unwrap(); + assert!(sql.contains("amount AS revenue_raw"), "{sql}"); + assert!(sql.contains("SUM(orders_cte.revenue_raw)"), "{sql}"); + assert!(sql.contains("FROM sales"), "{sql}"); + source["models"][0]["metrics"][0]["sql"] = json!("other.amount"); + assert!(SemanticInput::from_json(&source.to_string()).is_err()); + } + + #[test] + fn schema_exposure_uses_only_host_discovered_dimensions() { + let mut source = input(); + source["required_capabilities"] = json!(["model.schema_exposure"]); + source["models"][0]["schema_exposure"] = json!({"private":["secret"]}); + source["models"][0]["auto_dimensions"] = json!(true); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"dimensions":["orders.status"]}"#) + .unwrap(); + assert!(!sql.contains("secret"), "{sql}"); + assert!(compile_with_semantic_input( + &source.to_string(), + r#"{"dimensions":["orders.secret"]}"#, + ) + .is_err()); + let decoded = SemanticInput::from_json(&source.to_string()).unwrap(); + assert_eq!(decoded.source, source); + } + #[test] fn resolved_explore_anchor_is_shared_by_compilation_and_validation() { let mut source = input(); @@ -1580,6 +1972,83 @@ mod tests { assert_eq!(model.get_segment("unused").unwrap().sql, "{% if missing"); } + #[test] + fn selected_order_names_remain_canonical_before_sql_rendering() { + let mut source = input(); + source["models"][0]["dimensions"] = json!([ + {"name":"Order Status", "type":"categorical", "sql":"status"}, + {"name":"Rank DESC", "type":"categorical", "sql":"status"} + ]); + let mut input = SemanticInput::decode_scoped(&source.to_string(), true).unwrap(); + for (order, expected) in [ + ("orders.Order Status DESC", "orders.Order Status DESC"), + ("orders.Rank DESC", "orders.Rank DESC"), + ("Public Status ASC", "orders.Order Status ASC"), + ] { + let request = json!({ + "metrics":["orders.revenue"], + "dimensions":["orders.Order Status", "orders.Rank DESC"], + "aliases":{"orders.Order Status":"Public Status"}, + "order_by":[order] + }) + .to_string(); + let query = prepare_query_input(query_input(&request).unwrap(), &mut input).unwrap(); + assert_eq!(query.order_by, vec![expected]); + // Exercise policy dependency parsing and final output binding too. + let sql = compile_with_semantic_input(&source.to_string(), &request).unwrap(); + assert!(sql.contains("ORDER BY"), "{sql}"); + } + } + + #[test] + fn explicit_aggregation_and_authored_windows_accept_physical_columns() { + let mut source = input(); + source["models"][0]["metrics"] = json!([ + {"name":"value", "type":"derived", "agg":"sum", "sql":"amount * quantity"}, + {"name":"running", "type":"derived", "sql":"sum(amount) over (order by year)"} + ]); + let decoded = SemanticInput::from_json(&source.to_string()).unwrap(); + assert_eq!( + decoded + .graph + .get_model("orders") + .unwrap() + .get_metric("value") + .unwrap() + .r#type, + crate::core::MetricType::Simple, + ); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.value"]}"#) + .unwrap(); + assert!(sql.contains("SUM("), "{sql}"); + assert!(sql.contains("quantity"), "{sql}"); + } + + #[test] + fn dotted_graph_aggregate_name_declares_an_existing_source_model() { + let mut source = input(); + source["metrics"] = json!([ + {"name":"orders.p95.amount", "agg":"max", "sql":"amount"} + ]); + let sql = compile_with_semantic_input( + &source.to_string(), + &json!({ + "metrics":["orders.p95.amount"], + "order_by":["orders.p95.amount DESC"] + }) + .to_string(), + ) + .unwrap(); + assert!(sql.contains("\"orders.p95.amount\" DESC"), "{sql}"); + source["metrics"][0]["name"] = json!("missing.p95.amount"); + assert!(compile_with_semantic_input( + &source.to_string(), + r#"{"metrics":["missing.p95.amount"]}"#, + ) + .is_err()); + } + #[test] fn handoff_preserves_unknown_keys_and_original_declarations() { let source = input(); @@ -1612,10 +2081,10 @@ mod tests { )); } let mut source = input(); - source["models"][0]["schema_exposure"] = json!({}); + source["models"][0]["extends"] = json!("base_orders"); assert!(matches!( SemanticInput::from_json(&source.to_string()), - Err(SidemanticError::UnsupportedSemanticFeatures { .. }) + Err(SidemanticError::ValidationIssue { .. }) )); let mut source = input(); source["models"][0]["invariant_filters"] = json!(["tenant_id = 1"]); @@ -1653,6 +2122,7 @@ mod tests { .remove("unexpected"); source["models"][0]["pre_aggregations"][0]["union_with_source_data"] = json!(true); source["models"][0]["pre_aggregations"][0]["rollups"] = json!(["orders.historical"]); + source["required_capabilities"] = json!(["preaggregation.lambda"]); let decoded = SemanticInput::from_json(&source.to_string()).unwrap(); assert!( decoded.graph.get_model("orders").unwrap().pre_aggregations[0].union_with_source_data @@ -1664,6 +2134,82 @@ mod tests { ); } + #[test] + fn sql_rewrite_preaggregations_are_opt_in_and_respect_policies() { + let mut source = input(); + source["models"][0]["pre_aggregations"] = json!([ + {"name":"total", "measures":["revenue"]} + ]); + for query in [ + "SELECT revenue FROM orders", + "SELECT revenue AS total FROM orders", + "SELECT * FROM (SELECT revenue FROM orders) AS totals", + ] { + let sql = + rewrite_with_semantic_input_context(&source.to_string(), query, "{}").unwrap(); + assert!(!sql.contains("orders_preagg_total"), "{sql}"); + assert!(!sql.contains("used_preagg=true"), "{sql}"); + dialects::parse(&sql, DialectType::DuckDB).unwrap(); + let sql = rewrite_with_semantic_input_context( + &source.to_string(), + query, + r#"{"use_preaggregations":true}"#, + ) + .unwrap(); + assert!(sql.contains("orders_preagg_total"), "{sql}"); + assert!(sql.contains("used_preagg=true"), "{sql}"); + dialects::parse(&sql, DialectType::DuckDB).unwrap(); + } + source["models"][0]["invariant_filters"] = json!(["not deleted"]); + let sql = rewrite_with_semantic_input_context( + &source.to_string(), + "SELECT revenue FROM orders", + r#"{"use_preaggregations":true}"#, + ) + .unwrap(); + assert!(!sql.contains("orders_preagg_total"), "{sql}"); + assert!(!sql.contains("used_preagg=true"), "{sql}"); + dialects::parse(&sql, DialectType::DuckDB).unwrap(); + assert!(sql.contains("NOT deleted"), "{sql}"); + } + + #[test] + fn raw_inheritance_preserves_source_and_enforces_inherited_policies() { + let mut source = input(); + source["models"][0]["security"] = json!({"row_filters":["tenant = {{ user.tenant }}"]}); + source["models"][0]["invariant_filters"] = json!(["not deleted"]); + source["models"].as_array_mut().unwrap().push(json!({ + "name":"vip_orders", "extends":"orders", "security":null, + "invariant_filters":["amount > 100"] + })); + let decoded = SemanticInput::from_json(&source.to_string()).unwrap(); + assert_eq!(decoded.source, source); + let parent = decoded.graph.get_model("orders").unwrap(); + let child = decoded.graph.get_model("vip_orders").unwrap(); + assert_eq!(child.table, parent.table); + assert_eq!(child.metrics.len(), parent.metrics.len()); + assert!(matches!( + compile_with_semantic_input( + &source.to_string(), + r#"{"metrics":["vip_orders.revenue"]}"# + ), + Err(SidemanticError::Security(_)) + )); + let sql = compile_with_semantic_input( + &source.to_string(), + r#"{"metrics":["vip_orders.revenue"],"user_attributes":{"tenant":1}}"#, + ) + .unwrap(); + assert!(sql.contains("tenant = 1"), "{sql}"); + assert!(sql.contains("NOT deleted"), "{sql}"); + assert!(sql.contains("amount > 100"), "{sql}"); + source["models"][1]["unexpected"] = json!(true); + assert!(matches!( + SemanticInput::from_json(&source.to_string()), + Err(SidemanticError::ValidationIssue { .. }) + )); + } + #[test] fn handoff_policies_are_enforced_and_cannot_be_forged() { let mut source = input(); @@ -1975,6 +2521,40 @@ mod tests { assert!(sql.contains("NULLIF")); } + #[test] + fn legacy_relationship_sql_keeps_keyed_join_in_both_directions() { + let mut source = input(); + source["models"][0]["primary_key"] = json!("id"); + source["models"][0]["relationships"] = json!([ + {"name":"items", "type":"one_to_many", "sql":"id", "foreign_key":"order_id"} + ]); + source["models"].as_array_mut().unwrap().push(json!({ + "name":"items", "table":"items", "primary_key":"item_id", + "dimensions":[{"name":"category", "type":"categorical"}] + })); + let decoded = SemanticInput::from_json(&source.to_string()).unwrap(); + for (from, to, local, remote) in [ + ("orders", "items", "id", "order_id"), + ("items", "orders", "order_id", "id"), + ] { + let path = decoded.graph.find_join_path(from, to).unwrap(); + assert_eq!(path.steps[0].from_keys, vec![local]); + assert_eq!(path.steps[0].to_keys, vec![remote]); + assert!(path.steps[0].custom_condition.is_none()); + } + let sql = compile_with_semantic_input( + &source.to_string(), + r#"{"metrics":["orders.revenue"],"dimensions":["items.category"]}"#, + ) + .unwrap(); + // Either operand may anchor the query's dimension population. + assert!( + sql.contains("orders_cte.id = items_cte.order_id") + || sql.contains("items_cte.order_id = orders_cte.id"), + "{sql}" + ); + } + #[test] fn handoff_relationships_preserve_composite_keys_and_roles() { let mut source = input(); @@ -2057,10 +2637,10 @@ mod tests { } #[test] - fn count_distinct_uses_only_a_known_single_key() { + fn count_distinct_uses_known_single_and_composite_keys() { let mut source = input(); source["models"][0]["metrics"] = json!([{"name":"unique_orders", "agg":"count_distinct"}]); - for key in [Value::Null, json!(["tenant_id", "order_id"])] { + for key in [Value::Null, json!([])] { source["models"][0]["primary_key"] = key; assert!(matches!( SemanticInput::from_json(&source.to_string()), @@ -2076,6 +2656,20 @@ mod tests { assert!(sql.contains("COUNT(DISTINCT")); assert!(sql.contains("order_id")); assert!(!sql.contains("CONCAT")); + source["models"][0]["primary_key"] = json!(["tenant_key", "order_key"]); + source["models"][0]["dimensions"] = json!([ + {"name":"tenant_key", "type":"numeric", "sql":"tenant_id + 10"}, + {"name":"order_key", "type":"numeric", "sql":"order_id + 100"} + ]); + let sql = compile_with_semantic_input( + &source.to_string(), + r#"{"metrics":["orders.unique_orders"]}"#, + ) + .unwrap(); + assert!(sql.contains("COUNT(DISTINCT"), "{sql}"); + assert!(sql.contains("CONCAT"), "{sql}"); + assert!(sql.contains("tenant_id + 10"), "{sql}"); + assert!(sql.contains("order_id + 100"), "{sql}"); } #[test] @@ -2484,11 +3078,62 @@ mod tests { source["models"][0]["metrics"][0]["sql"] = json!("SUM(amount)"); source["models"][0]["dimensions"] = json!([{"name":"amount", "type":"numeric", "sql":"amount * 10"}]); - let error = SemanticInput::from_json(&source.to_string()).err(); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.paid"]}"#) + .unwrap(); + assert!(sql.contains("SUM("), "{sql}"); + assert!(!sql.contains("* 10"), "{sql}"); + } + + #[test] + fn compilation_binds_model_placeholders_in_metric_dependency_walks() { + for metric in [ + json!({"name":"value", "agg":"sum", "sql":"({model}.amount)"}), + json!({"name":"value", "sql":"SUM({model}.amount)", "sql_is_complete":true}), + json!({"name":"value", "type":"derived", "sql":"{model}.revenue * 2"}), + ] { + let mut source = input(); + source["models"][0]["metrics"] + .as_array_mut() + .unwrap() + .push(metric); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.value"]}"#) + .unwrap(); + assert!(!sql.contains("{model}"), "{sql}"); + assert!(sql.contains("orders_cte"), "{sql}"); + } + } + + #[test] + fn complete_aggregates_accept_source_cte_aliases_without_weakening_metric_binding() { + let mut source = input(); + source["models"][0]["metrics"] = json!([ + {"name":"average", "sql":"AVG(orders_cte.amount)", "sql_is_complete":true} + ]); + let sql = + compile_with_semantic_input(&source.to_string(), r#"{"metrics":["orders.average"]}"#) + .unwrap(); + // Complete aggregates may project their physical inputs into an inner + // population before averaging. Validate the resulting query, not that + // planner's temporary input name. + assert!(sql.contains("AVG("), "{sql}"); assert!( - matches!(&error, Some(SidemanticError::UnsupportedSemanticFeatures { capabilities }) if capabilities == &vec!["metric.raw_computed_column"]), - "{error:?}" + polyglot_sql::parse_one(&sql, DialectType::DuckDB).is_ok(), + "{sql}" ); + + source["models"][0]["metrics"][0]["sql"] = json!("AVG(missing_cte.amount)"); + assert!(matches!( + SemanticInput::from_json(&source.to_string()), + Err(SidemanticError::ValidationIssue { .. }) + )); + source["models"][0]["metrics"][0] = + json!({"name":"average", "type":"derived", "sql":"orders_cte.revenue"}); + assert!(matches!( + SemanticInput::from_json(&source.to_string()), + Err(SidemanticError::ValidationIssue { .. }) + )); } #[test] diff --git a/sidemantic-rs/src/semantic_input/dates.rs b/sidemantic-rs/src/semantic_input/dates.rs index d5d4267fc..a7fb27691 100644 --- a/sidemantic-rs/src/semantic_input/dates.rs +++ b/sidemantic-rs/src/semantic_input/dates.rs @@ -18,8 +18,7 @@ pub(super) fn normalize_transpiled( if source != DialectType::Snowflake || target != DialectType::DuckDB { return Ok(sql.to_owned()); } - let expression = polyglot_sql::parse_one(sql, target) - .map_err(|error| SidemanticError::SqlParse(error.to_string()))?; + let expression = super::dialects::parse(sql, target)?; let convert_error = |error: serde_json::Error| SidemanticError::SqlGeneration(error.to_string()); let mut value = serde_json::to_value(expression).map_err(convert_error)?; diff --git a/sidemantic-rs/src/semantic_input/dialects.rs b/sidemantic-rs/src/semantic_input/dialects.rs index 17fe9d993..cf37115d5 100644 --- a/sidemantic-rs/src/semantic_input/dialects.rs +++ b/sidemantic-rs/src/semantic_input/dialects.rs @@ -86,8 +86,164 @@ pub(crate) fn parse(sql: &str, source: DialectType) -> Result { let sql = super::literals::source_sql(sql, source)?; #[cfg(target_arch = "wasm32")] crate::wasm_sql_guard::check(&sql, source)?; - polyglot_sql::parse_one(&sql, source) - .map_err(|error| SidemanticError::SqlParse(error.to_string())) + let mut statements = parse_many(&sql, source)?; + if statements.len() != 1 { + return Err(SidemanticError::SqlParse( + "Expected one SQL statement".into(), + )); + } + Ok(statements.remove(0)) +} + +/// QUANTILE_CONT/DISC are absent from the pinned parser's aggregate registry. +/// Parse their argument lists with its generic aggregate grammar, then restore +/// the actual name in the AST. ORDER BY, DISTINCT, FILTER and window clauses +/// must survive; deleting the aggregate ordering changes percentile semantics. +pub(crate) fn parse_many(sql: &str, dialect: DialectType) -> Result> { + use polyglot_sql::expressions::AggregateFunction; + use polyglot_sql::tokens::TokenType; + + let mut replacements: HashMap = HashMap::new(); + let mut prepared = sql.to_owned(); + if sql.to_ascii_uppercase().contains("QUANTILE_") { + let stream = Dialect::get(dialect) + .tokenize(sql) + .map_err(|error| SidemanticError::SqlParse(error.to_string()))?; + let offsets = sql + .char_indices() + .map(|(index, _)| index) + .chain([sql.len()]) + .collect::>(); + let mut prefix = "__sidemantic_quantile_".to_owned(); + while sql.to_ascii_lowercase().contains(&prefix) { + prefix.push('_'); + } + prepared.clear(); + let mut cursor = 0; + let mut index = 0; + while index + 1 < stream.len() { + let token = &stream[index]; + if token.token_type == TokenType::Identifier + || !["QUANTILE_CONT", "QUANTILE_DISC"] + .contains(&token.text.to_ascii_uppercase().as_str()) + || stream[index + 1].token_type != TokenType::LParen + || index > 0 && stream[index - 1].token_type == TokenType::Dot + { + index += 1; + continue; + } + let open = index + 1; + let mut end = open + 1; + let mut depth = 1; + while end < stream.len() { + match stream[end].token_type { + TokenType::LParen => depth += 1, + TokenType::RParen => depth -= 1, + _ => {} + } + if depth == 0 { + break; + } + end += 1; + } + if end == stream.len() { + return Err(SidemanticError::SqlParse( + "Unclosed quantile aggregate".into(), + )); + } + let body = &sql[offsets[stream[open].span.end]..offsets[stream[end].span.start]]; + // RESERVOIR_SAMPLE uses the generic aggregate grammar without a + // specialized argument parser in the pinned registry. Only + // this root node is renamed; nested calls keep their own identity. + let mut parsed = parse_many(&format!("SELECT RESERVOIR_SAMPLE({body})"), dialect)?; + let Expression::Select(select) = parsed.remove(0) else { + unreachable!() + }; + let Expression::AggregateFunction(mut aggregate) = select.expressions[0].clone() else { + return Err(SidemanticError::SqlParse( + "Expected quantile aggregate arguments".into(), + )); + }; + aggregate.name = token.text.to_ascii_uppercase(); + let placeholder = format!("{prefix}{}", replacements.len()); + replacements.insert(placeholder.clone(), *aggregate); + prepared.push_str(&sql[cursor..offsets[token.span.start]]); + prepared.push_str(&format!("{placeholder}()")); + cursor = offsets[stream[end].span.end]; + index = end + 1; + } + prepared.push_str(&sql[cursor..]); + } + let statements = polyglot_sql::parse(&prepared, dialect) + .map_err(|error| SidemanticError::SqlParse(error.to_string()))?; + let complete_conditionals = + dialect == DialectType::DuckDB && prepared.to_ascii_uppercase().contains("IF"); + if replacements.is_empty() && !complete_conditionals { + return Ok(statements); + } + fn restore( + value: &mut Value, + replacements: &HashMap, + complete_conditionals: bool, + ) -> Result<()> { + match value { + Value::Object(fields) => { + for child in fields.values_mut() { + restore(child, replacements, complete_conditionals)?; + } + if complete_conditionals { + if let Some(function) = fields.get_mut("if_func").and_then(Value::as_object_mut) + { + if function.get("false_value").is_none_or(Value::is_null) { + // The parser accepts IF(condition, value), but the + // DuckDB emitter preserves that invalid two-argument + // call. Its omitted false branch has NULL semantics. + function.insert( + "false_value".into(), + serde_json::to_value(Expression::null()).map_err(|error| { + SidemanticError::SqlParse(error.to_string()) + })?, + ); + } + } + } + for kind in ["function", "aggregate_function"] { + let Some(function) = fields.get(kind) else { + continue; + }; + let Some(name) = function.get("name").and_then(Value::as_str) else { + continue; + }; + let Some(saved) = replacements.get(&name.to_ascii_lowercase()) else { + continue; + }; + let mut aggregate = saved.clone(); + if kind == "aggregate_function" { + let wrapper: AggregateFunction = + serde_json::from_value(function.clone()) + .map_err(|error| SidemanticError::SqlParse(error.to_string()))?; + aggregate.filter = wrapper.filter; + aggregate.ignore_nulls = wrapper.ignore_nulls.or(aggregate.ignore_nulls); + } + *value = + serde_json::to_value(Expression::AggregateFunction(Box::new(aggregate))) + .map_err(|error| SidemanticError::SqlParse(error.to_string()))?; + break; + } + } + Value::Array(children) => { + for child in children { + restore(child, replacements, complete_conditionals)?; + } + } + _ => {} + } + Ok(()) + } + let mut value = serde_json::to_value(statements) + .map_err(|error| SidemanticError::SqlParse(error.to_string()))?; + restore(&mut value, &replacements, complete_conditionals)?; + serde_json::from_value(value).map_err(|error| SidemanticError::SqlParse(error.to_string())) } pub(crate) fn query(sql: &str, source: DialectType) -> Result { @@ -358,6 +514,73 @@ pub(super) fn normalize( mod tests { use super::*; + #[test] + fn ordered_quantiles_preserve_arguments_order_filter_and_window() { + for name in ["quantile_cont", "quantile_disc"] { + let sql = format!("SELECT {name}(DISTINCT value, 0.25 ORDER BY sort_key DESC NULLS FIRST) FILTER (WHERE included) OVER (PARTITION BY category), 'é QUANTILE_CONT(x ORDER BY y)' AS label"); + let parsed = parse(&sql, DialectType::DuckDB).unwrap(); + let generated = polyglot_sql::generate(&parsed, DialectType::DuckDB).unwrap(); + assert!( + generated.contains(&format!( + "{}(DISTINCT value, 0.25 ORDER BY sort_key DESC NULLS FIRST)", + name.to_ascii_uppercase() + )), + "{generated}" + ); + assert!( + generated.contains("FILTER(WHERE included)") + || generated.contains("FILTER (WHERE included)"), + "{generated}" + ); + assert!(generated.contains("PARTITION BY category"), "{generated}"); + assert!( + generated.contains("'é QUANTILE_CONT(x ORDER BY y)'"), + "{generated}" + ); + let reparsed = parse(&generated, DialectType::DuckDB).unwrap(); + assert_eq!( + polyglot_sql::generate(&reparsed, DialectType::DuckDB).unwrap(), + generated + ); + } + } + + #[test] + fn omitted_conditional_false_branches_are_explicit_nulls_for_duckdb() { + for (input, expected) in [ + ("IF(flag, 1)", "IF(flag, 1, NULL)"), + ("COUNT(IF(flag, 1))", "COUNT(IF(flag, 1, NULL))"), + ( + "IF(flag, IF(other, 1), 0)", + "IF(flag, IF(other, 1, NULL), 0)", + ), + ("IF(flag, 1, 0)", "IF(flag, 1, 0)"), + ] { + let parsed = parse(&format!("SELECT {input}"), DialectType::DuckDB).unwrap(); + let generated = polyglot_sql::generate(&parsed, DialectType::DuckDB).unwrap(); + assert_eq!(generated, format!("SELECT {expected}")); + } + let sql = "SELECT 'IF(flag, 1)' AS label"; + let parsed = parse(sql, DialectType::DuckDB).unwrap(); + assert_eq!( + polyglot_sql::generate(&parsed, DialectType::DuckDB).unwrap(), + sql + ); + } + + #[test] + fn nested_quantiles_keep_each_function_identity() { + let parsed = parse("SELECT quantile_cont((SELECT quantile_disc(value, 0.5 ORDER BY value) FROM raw), 0.25 ORDER BY rank)", DialectType::DuckDB).unwrap(); + let generated = polyglot_sql::generate(&parsed, DialectType::DuckDB).unwrap(); + assert!(generated.contains("QUANTILE_CONT("), "{generated}"); + assert!( + generated.contains("QUANTILE_DISC(value, 0.5 ORDER BY value)"), + "{generated}" + ); + assert!(!generated.contains("RESERVOIR_SAMPLE"), "{generated}"); + assert!(!generated.contains("__sidemantic_quantile_"), "{generated}"); + } + #[test] fn nested_query_emission_uses_production_stack_on_standard_test_thread() { // Construct the AST directly to isolate final emission from parsing. diff --git a/sidemantic-rs/src/semantic_input/fragments.rs b/sidemantic-rs/src/semantic_input/fragments.rs new file mode 100644 index 000000000..3c0092d71 --- /dev/null +++ b/sidemantic-rs/src/semantic_input/fragments.rs @@ -0,0 +1,297 @@ +//! Caller expressions cannot introduce physical reads or escape their clause. +//! Trusted segment and security definitions do not cross this request boundary. + +use polyglot_sql::{expressions::Select, DialectType, Expression}; +use serde_json::Value; + +use crate::error::Result; + +// Keep this contract aligned with sidemantic/sql/fragment.py. Unknown/UDF +// functions belong in trusted model SQL, not caller-supplied filters. +const SCALAR_FUNCTIONS: &str = " +ABS ACOS ASIN ATAN ATAN2 CEIL CEILING FLOOR ROUND SIGN SQRT CBRT POWER POW EXP LN LOG LOG2 LOG10 +SIN COS TAN COT DEGREES RADIANS PI MOD GREATEST LEAST COALESCE NULLIF IF IIF CASE CAST TRY_CAST +LOWER UPPER LENGTH CHAR_LENGTH CHARACTER_LENGTH CONCAT CONCAT_WS SUBSTRING SUBSTR LEFT RIGHT +TRIM LTRIM RTRIM REPLACE REPEAT REVERSE LPAD RPAD SPLIT SPLIT_PART STARTS_WITH ENDS_WITH +CONTAINS POSITION STR_POSITION REGEXP_LIKE REGEXP_REPLACE REGEXP_EXTRACT REGEXP_SPLIT +COUNT SUM AVG MIN MAX MEDIAN STDDEV STDDEV_POP STDDEV_SAMP VARIANCE VAR_POP VAR_SAMP +DATE TIME TIMESTAMP DATE_TRUNC TIMESTAMP_TRUNC DATETIME_TRUNC TIME_TRUNC DATE_ADD DATE_SUB +DATE_DIFF DATEDIFF TIMESTAMP_ADD TIMESTAMP_SUB TIMESTAMP_DIFF EXTRACT YEAR MONTH DAY +DAY_OF_MONTH DAY_OF_WEEK DAY_OF_YEAR WEEK WEEK_OF_YEAR QUARTER HOUR MINUTE SECOND +CURRENT_DATE CURRENT_TIME CURRENT_TIMESTAMP CURRENT_DATETIME TIME_TO_STR STR_TO_TIME +TS_OR_DS_TO_DATE TS_OR_DS_TO_TIMESTAMP TS_OR_DS_TO_DATE_STR TIME_TO_UNIX UNIX_TO_TIME +DATE_TO_DATE_STR LAST_DAY DATE_FROM_PARTS TIMESTAMP_FROM_PARTS INTERVAL +ARRAY ARRAY_SIZE ARRAY_LENGTH ARRAY_CONTAINS ARRAY_SLICE ARRAY_TO_STRING +JSON_EXTRACT JSON_EXTRACT_SCALAR JSONB_EXTRACT JSONB_EXTRACT_SCALAR JSON_TYPE +STRUCT MAP EXISTS ISNULL IFNULL NVL"; + +fn normalized_name(name: &str) -> String { + name.chars() + .filter(|character| *character != '_') + .flat_map(char::to_uppercase) + .collect() +} + +fn allowed_function(name: &str) -> bool { + let name = normalized_name(name); + SCALAR_FUNCTIONS + .split_whitespace() + .any(|allowed| normalized_name(allowed) == name) +} + +fn validate_nodes(value: &Value, path: &str) -> Result<()> { + // The pinned public AST walker omits typed function children. Inspect the + // serialized tree, deserializing enum nodes to distinguish them from their + // payload objects (which can also have a single field). + if let Some(fields) = value.as_object().filter(|fields| fields.len() == 1) { + if let Ok(expression) = serde_json::from_value::(value.clone()) { + let kind = fields.keys().next().unwrap().as_str(); + match expression { + Expression::Table(_) => { + return Err(super::invalid( + path, + "Query expressions cannot introduce physical data sources", + )); + } + Expression::Select(select) if select.into.is_some() => { + return Err(super::invalid( + path, + "Query expressions cannot introduce physical data sources", + )); + } + Expression::Function(function) => { + if !allowed_function(&function.name) { + return Err(super::invalid( + path, + format!( + "Function {} is not allowed in query expressions", + function.name + ), + )); + } + } + Expression::AggregateFunction(function) => { + if !allowed_function(&function.name) { + return Err(super::invalid( + path, + format!( + "Function {} is not allowed in query expressions", + function.name + ), + )); + } + } + Expression::MethodCall(_) => { + return Err(super::invalid( + path, + "Qualified functions are not allowed in query expressions", + )); + } + _ => { + let structural = matches!( + kind, + "literal" + | "boolean" + | "null" + | "identifier" + | "column" + | "star" + | "select" + | "union" + | "intersect" + | "except" + | "subquery" + | "values" + | "alias" + | "and" + | "or" + | "xor" + | "add" + | "sub" + | "mul" + | "div" + | "eq" + | "neq" + | "lt" + | "lte" + | "gt" + | "gte" + | "like" + | "i_like" + | "bitwise_and" + | "bitwise_or" + | "bitwise_xor" + | "not" + | "neg" + | "bitwise_not" + | "in" + | "between" + | "is_null" + | "is_true" + | "is_false" + | "is" + | "from" + | "join" + | "where" + | "group_by" + | "having" + | "order_by" + | "ordered" + | "limit" + | "offset" + | "data_type" + | "tuple" + | "paren" + | "var" + | "dot" + | "bracket" + | "at_time_zone" + | "window" + | "window_function" + | "over" + | "within_group" + | "when" + | "whens" + ); + let canonical = match kind { + "if_func" => "IF", + "safe_cast" => "TRY_CAST", + _ => kind, + }; + if !structural && !allowed_function(canonical) { + return Err(super::invalid( + path, + format!("SQL expression {kind} is not allowed in query expressions"), + )); + } + } + } + } + } + match value { + Value::Object(fields) => { + for child in fields.values() { + validate_nodes(child, path)?; + } + } + Value::Array(children) => { + for child in children { + validate_nodes(child, path)?; + } + } + _ => {} + } + Ok(()) +} + +pub(super) fn validate_request_expression( + sql: &str, + dialect: DialectType, + order: bool, +) -> Result<()> { + let (path, prefix) = if order { + ("query.order_by", "SELECT 1 ORDER BY ") + } else { + ("query.filters", "SELECT 1 WHERE ") + }; + let parsed = super::dialects::parse(&format!("{prefix}{sql}"), dialect) + .map_err(|error| super::invalid(path, format!("Invalid query expression: {error}")))?; + let Expression::Select(mut select) = parsed else { + return Err(super::invalid( + path, + "Query expression contains disallowed SQL", + )); + }; + let expression = if order { + let clause = select + .order_by + .take() + .filter(|clause| clause.expressions.len() == 1) + .ok_or_else(|| super::invalid(path, "Expected one ordering expression"))?; + Expression::OrderBy(Box::new(clause)) + } else { + select + .where_clause + .take() + .ok_or_else(|| super::invalid(path, "Expected one query expression"))? + .this + }; + select.expressions.clear(); + select.leading_comments.clear(); + select.post_select_comments.clear(); + if *select != Select::new() { + return Err(super::invalid( + path, + "Query expression contains extra clauses", + )); + } + let value = serde_json::to_value(expression).map_err(|error| super::invalid(path, error))?; + validate_nodes(&value, path) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn caller_filters_reject_reads_functions_and_clause_escapes() { + super::super::with_semantic_stack(|| { + for sql in [ + "EXISTS (SELECT 1 FROM secret_table)", + "events.user_id IN (SELECT user_id FROM events_raw)", + "EXISTS (SELECT 1 FROM read_csv('/tmp/private.csv'))", + "1 = 1 UNION SELECT 1", + "1 = 1 ORDER BY 1", + "1 = 1; SELECT 2", + "(SELECT pg_read_file('/tmp/private')) IS NOT NULL", + "readfile('/tmp/private') IS NOT NULL", + "lo_get(123) IS NOT NULL", + "custom_schema.abs('/tmp/private') IS NOT NULL", + "coalesce(readfile('/tmp/private'), '') != ''", + ] { + assert!( + validate_request_expression(sql, DialectType::DuckDB, false).is_err(), + "{sql}" + ); + } + for sql in [ + "count; SELECT 2", + "count DESC LIMIT 1", + "count, user_id", + "random()", + ] { + assert!( + validate_request_expression(sql, DialectType::DuckDB, true).is_err(), + "{sql}" + ); + } + Ok(()) + }) + .unwrap(); + } + + #[test] + fn caller_filters_keep_scalar_subqueries_literals_and_known_functions() { + super::super::with_semantic_stack(|| { + for sql in [ + "EXISTS (SELECT 1 WHERE 2 > 1)", + "event_type != '; -- FROM secret_table'", + "coalesce(user_id, 0) >= 1", + "CASE WHEN user_id > 0 THEN user_id ELSE 0 END > 0", + "lower(event_type) = 'signup'", + "COUNT(DISTINCT user_id) > 0", + "CAST(user_id AS VARCHAR) != ''", + ] { + validate_request_expression(sql, DialectType::DuckDB, false)?; + } + validate_request_expression("count DESC NULLS LAST", DialectType::DuckDB, true)?; + validate_request_expression( + "CASE WHEN user_id > 0 THEN user_id ELSE 0 END DESC", + DialectType::DuckDB, + true, + )?; + Ok(()) + }) + .unwrap(); + } +} diff --git a/sidemantic-rs/src/semantic_input/inheritance.rs b/sidemantic-rs/src/semantic_input/inheritance.rs new file mode 100644 index 000000000..efb297ffb --- /dev/null +++ b/sidemantic-rs/src/semantic_input/inheritance.rs @@ -0,0 +1,194 @@ +//! Resolve raw declarations before projecting away policy and importer fields. + +use std::collections::{HashMap, HashSet}; + +use serde_json::{Map, Value}; + +use super::{invalid, object, DESCRIPTIVE_FIELDS}; +use crate::error::Result; + +pub(super) fn resolve(models: &[Value]) -> Result> { + let mut declarations = HashMap::new(); + let mut names = Vec::new(); + for (index, model) in models.iter().enumerate() { + let path = format!("models[{index}]"); + let raw = object(model.clone(), &path)?; + let name = raw + .get("name") + .and_then(Value::as_str) + .ok_or_else(|| invalid(&path, "model requires a name"))? + .to_owned(); + if declarations.insert(name.clone(), raw).is_some() { + return Err(invalid(&path, format!("duplicate model '{name}'"))); + } + names.push(name); + } + let mut resolved = HashMap::new(); + let mut visiting = HashSet::new(); + for name in &names { + resolve_model(name, &declarations, &mut resolved, &mut visiting)?; + } + Ok(names + .into_iter() + .map(|name| Value::Object(resolved.remove(&name).unwrap())) + .collect()) +} + +fn resolve_model( + name: &str, + declarations: &HashMap>, + resolved: &mut HashMap>, + visiting: &mut HashSet, +) -> Result> { + if let Some(model) = resolved.get(name) { + return Ok(model.clone()); + } + let path = format!("models.{name}.extends"); + let mut child = declarations + .get(name) + .cloned() + .ok_or_else(|| invalid(&path, format!("unknown parent model '{name}'")))?; + if !visiting.insert(name.to_owned()) { + return Err(invalid(&path, "circular model inheritance")); + } + let model = match child.remove("extends") { + None | Some(Value::Null) => child, + Some(Value::String(parent)) => { + let parent = resolve_model(&parent, declarations, resolved, visiting)?; + merge(child, parent, name)? + } + Some(_) => return Err(invalid(&path, "expected a parent model name")), + }; + visiting.remove(name); + resolved.insert(name.to_owned(), model.clone()); + Ok(model) +} + +fn merge( + mut child: Map, + mut parent: Map, + name: &str, +) -> Result> { + // Match Python merge_model: named collections override by name; invariant + // filters always accumulate, and the parent's security remains authoritative. + for field in [ + "dimensions", + "metrics", + "relationships", + "segments", + "pre_aggregations", + "invariant_filters", + ] { + let path = format!("models.{name}.{field}"); + let mut items = Vec::::new(); + let mut positions = HashMap::::new(); + for value in [parent.remove(field), child.remove(field)] + .into_iter() + .flatten() + { + let entries = value + .as_array() + .ok_or_else(|| invalid(&path, "expected an array"))?; + for entry in entries { + if field == "invariant_filters" { + items.push(entry.clone()); + continue; + } + let item_name = entry + .get("name") + .and_then(Value::as_str) + .ok_or_else(|| invalid(&path, "collection item requires a name"))?; + if let Some(position) = positions.get(item_name) { + items[*position] = entry.clone(); + } else { + positions.insert(item_name.to_owned(), items.len()); + items.push(entry.clone()); + } + } + } + parent.insert(field.to_owned(), Value::Array(items)); + } + for field in [ + "name", + "table", + "sql", + "source_uri", + "description", + "primary_key", + "unique_keys", + "default_time_dimension", + "default_grain", + "freshness", + "metadata", + "auto_dimensions", + "schema_exposure", + "meta", + ] { + if let Some(value) = child.remove(field) { + parent.insert(field.to_owned(), value); + } + } + // Python retains these fields from the parent, including inherited policy. + // Preserve unexpected child fields so the normal decoder still rejects them. + for (field, value) in child { + if !matches!(field.as_str(), "security" | "dax" | "expression_language") + && !DESCRIPTIVE_FIELDS.contains(&field.as_str()) + { + parent.insert(field, value); + } + } + Ok(parent) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn inheritance_is_transitive_and_keeps_parent_policy_and_named_order() { + let models = vec![ + json!({"name":"leaf", "extends":"child", "invariant_filters":["amount < 100"]}), + json!({"name":"base", "table":"sales", "primary_key":"id", + "security":{"row_filters":["tenant = 1"]}, + "invariant_filters":["not deleted"], + "dimensions":[{"name":"id"},{"name":"amount","sql":"raw_amount"}]}), + json!({"name":"child", "extends":"base", "primary_key":null, + "security":null, "invariant_filters":["amount > 10"], + "dimensions":[{"name":"amount","sql":"net_amount"},{"name":"region"}]}), + ]; + let output = resolve(&models).unwrap(); + let leaf = &output[0]; + assert_eq!(leaf["name"], "leaf"); + assert_eq!(leaf["table"], "sales"); + assert!(leaf["primary_key"].is_null()); + assert_eq!(leaf["security"], models[1]["security"]); + assert_eq!( + leaf["invariant_filters"], + json!(["not deleted", "amount > 10", "amount < 100"]) + ); + assert_eq!( + leaf["dimensions"], + json!([{"name":"id"},{"name":"amount","sql":"net_amount"},{"name":"region"}]) + ); + assert!(leaf.get("extends").is_none()); + } + + #[test] + fn invalid_inheritance_fails_with_validation_errors() { + for models in [ + vec![json!({"name":"a", "extends":"missing"})], + vec![ + json!({"name":"a", "extends":"b"}), + json!({"name":"b", "extends":"a"}), + ], + vec![json!({"name":"a", "extends":5})], + vec![json!({"name":"a"}), json!({"name":"a"})], + ] { + assert!(matches!( + resolve(&models), + Err(crate::error::SidemanticError::ValidationIssue { .. }) + )); + } + } +} diff --git a/sidemantic-rs/src/semantic_input/policies.rs b/sidemantic-rs/src/semantic_input/policies.rs index 24ddf89d8..cdbb3d708 100644 --- a/sidemantic-rs/src/semantic_input/policies.rs +++ b/sidemantic-rs/src/semantic_input/policies.rs @@ -156,7 +156,7 @@ fn prepare_with_dialects( population.expression(filter, None)?; } for order in &query.order_by { - for column in order_columns(order)? { + for column in order_columns(order, query)? { population.column(&column, None)?; } } @@ -503,7 +503,7 @@ fn check_visibility( columns.extend(outer_columns(parse_semantic_expression(filter)?)?); } for order in &query.order_by { - columns.extend(order_columns(order)?); + columns.extend(order_columns(order, query)?); } for column in columns { if let Some(model) = column @@ -530,11 +530,30 @@ fn check_visibility( Ok(()) } -fn order_columns(order: &str) -> Result> { +fn order_columns(order: &str, query: &SemanticQuery) -> Result> { + let known: Vec<_> = query + .metrics + .iter() + .chain(&query.dimensions) + .map(String::as_str) + .collect(); + let (field, suffix) = crate::sql::split_order_field(order, &known); + let framed = if known.contains(&field) { + // Selected fields are semantic names, including names with spaces. + // Quote only this parser input, leaving the request's binding keys intact. + let quote = |name: &str| format!("\"{}\"", name.replace('"', "\"\"")); + let field = field.split_once('.').map_or_else( + || quote(field), + |(model, field)| format!("{}.{}", quote(model), quote(field)), + ); + format!("{field} {suffix}") + } else { + order.to_owned() + }; #[cfg(target_arch = "wasm32")] - crate::wasm_sql_guard::check(order, DialectType::DuckDB)?; + crate::wasm_sql_guard::check(&framed, DialectType::DuckDB)?; let expression = - polyglot_sql::parse_one(&format!("SELECT 1 ORDER BY {order}"), DialectType::DuckDB) + polyglot_sql::parse_one(&format!("SELECT 1 ORDER BY {framed}"), DialectType::DuckDB) .map_err(|error| SidemanticError::SqlParse(error.to_string()))?; outer_columns(expression) } diff --git a/sidemantic-rs/src/sql/generator.rs b/sidemantic-rs/src/sql/generator.rs index ff27eee07..e83a5935a 100644 --- a/sidemantic-rs/src/sql/generator.rs +++ b/sidemantic-rs/src/sql/generator.rs @@ -6,6 +6,7 @@ mod cohort; mod conversion; mod fanout_aggregate; mod fanout_complete; +mod imported_totals; mod join_kind; mod options; mod retention; @@ -48,6 +49,8 @@ pub struct SemanticQuery { pub limit: Option, pub offset: Option, pub ungrouped: bool, + /// Explicit compatibility escape hatch: aggregate every snapshot row. + pub allow_non_additive_unsafe: bool, pub use_preaggregations: bool, pub preagg_database: Option, pub preagg_schema: Option, @@ -300,10 +303,9 @@ impl<'a> SqlGenerator<'a> { .invariant_filters .values() .any(|filters| !filters.is_empty()) - && !query.ungrouped && query.table_calculations.is_empty() { - if required_models.len() > 1 { + if required_models.len() > 1 && !query.ungrouped { let anchor = query .consumption_base_model .as_deref() @@ -319,7 +321,11 @@ impl<'a> SqlGenerator<'a> { )? { return Ok(sql); } - } else if let Some(model_name) = required_models.iter().next() { + } else if let Some(model_name) = required_models + .iter() + .next() + .filter(|_| required_models.len() == 1) + { if let Some(preagg_sql) = self.try_use_preaggregation( model_name, &metric_refs, @@ -330,6 +336,7 @@ impl<'a> SqlGenerator<'a> { query.offset, query.preagg_database.as_deref(), query.preagg_schema.as_deref(), + query.ungrouped, )? { return Ok(preagg_sql); } @@ -409,13 +416,13 @@ impl<'a> SqlGenerator<'a> { { continue; } - let window_sql = self.normalize_cte_source_expression(window_expr); + let window_sql = self.normalize_cte_source_expression(window_expr, model)?; raw_model_columns .entry(model_name.clone()) .or_default() .push(format!( "{window_sql} AS {}", - self.quote_identifier(&dimension.name) + self.quote_identifier(&Self::window_dimension_alias(dimension)) )); raw_model_aliases .entry(model_name.clone()) @@ -431,8 +438,10 @@ impl<'a> SqlGenerator<'a> { let metric = self.metric_for_model_with_source(&model_name, &metric_name, graph_metric)?; let raw_alias = self.metric_raw_alias(model, &metric_name, metric); - let mut raw_expr = - self.normalize_cte_source_expression(&self.metric_raw_expression(metric, model)?); + let mut raw_expr = self.normalize_cte_source_expression( + &self.metric_raw_expression(metric, model)?, + model, + )?; if !metric.filters.is_empty() { let metric_filter = self.normalize_metric_filters( &metric.filters, @@ -466,7 +475,9 @@ impl<'a> SqlGenerator<'a> { })?; let raw_expr = model .get_dimension(&column_name) - .map(|dimension| self.normalize_cte_source_expression(dimension.sql_expr())) + .and_then(|dimension| dimension.sql.as_deref()) + .map(|sql| self.normalize_cte_source_expression(sql, model)) + .transpose()? .unwrap_or_else(|| self.quote_identifier(&column_name)); raw_model_columns .entry(model_name.clone()) @@ -577,19 +588,36 @@ impl<'a> SqlGenerator<'a> { } else { identity } + } else if dimension.window.is_some() { + let column = format!( + "{}.{}", + alias, + self.quote_identifier(&Self::window_dimension_alias(dimension)) + ); + if let Some(granularity) = dim_ref + .granularity + .as_deref() + .or(dimension.granularity.as_deref()) + { + self.date_trunc_sql(granularity, &column)? + } else { + column + } } else if let Some(granularity) = dim_ref .granularity .as_deref() .or(dimension.granularity.as_deref()) { + let source = dimension + .sql + .clone() + .unwrap_or_else(|| self.quote_identifier(&dimension.name)); self.normalize_select_expression( - &self.date_trunc_sql(granularity, dimension.sql_expr())?, + &self.date_trunc_sql(granularity, &source)?, &alias, ) - } else if dimension.window.is_some() { - format!("{}.{}", alias, self.quote_identifier(&dimension.name)) } else { - self.dimension_select_expression(dimension, &alias) + self.dimension_select_expression(dimension, &alias)? } } else if Self::is_relationship_foreign_key_dimension(model, &dim_ref.name) { format!("{}.{}", alias, self.quote_identifier(&dim_ref.name)) @@ -720,7 +748,10 @@ impl<'a> SqlGenerator<'a> { }, MetricType::Derived => { // For derived metrics, we need to expand referenced metrics - self.expand_derived_metric(metric.sql_expr(), &metric_ref.model)? + self.expand_derived_metric( + &self.imported_calculation_expression(metric, &metric_ref.model)?, + &metric_ref.model, + )? } MetricType::Ratio => { // For ratio metrics, expand numerator and denominator @@ -761,7 +792,7 @@ impl<'a> SqlGenerator<'a> { sql.push('\n'); // FROM clause - let source_start = sql.len(); + let mut source_start = sql.len(); sql.push_str(&format!( "FROM {}_cte AS {}\n", base_model, @@ -815,11 +846,15 @@ impl<'a> SqlGenerator<'a> { } else if cte_where_filters .get(&step.to_model) .is_some_and(|filters| !filters.is_empty()) - || query - .prepared_policies - .filters_for_model(&step.to_model) - .next() - .is_some() + // Junction policies restrict which links are visible; + // they must not remove source rows with no visible link. + // Explicit query filters above still constrain the domain. + || (!self.graph.is_bridge_instance(&step.to_model) + && query + .prepared_policies + .filters_for_model(&step.to_model) + .next() + .is_some()) { "INNER JOIN" } else { @@ -837,6 +872,26 @@ impl<'a> SqlGenerator<'a> { sql.push_str(&format!("WHERE {}\n", filter_sql.join(" AND "))); } + // BSL all() aggregates the same filtered, joined population before + // grouping. In particular, distinct totals cannot sum group counts. + if select_parts + .iter() + .any(|part| part.to_ascii_lowercase().contains("__bsl_all(")) + { + let source = sql[source_start..].to_string(); + for part in &mut select_parts { + if part.to_ascii_lowercase().contains("__bsl_all(") { + *part = self.expand_imported_totals(part, &source)?; + } + } + sql.truncate(select_start); + sql.push_str("SELECT\n"); + sql.push_str(&select_parts.join(",\n")); + sql.push('\n'); + source_start = sql.len(); + sql.push_str(&source); + } + if !aggregate_ranks.is_empty() { let (ranked, rewritten_having) = self.ranked_aggregate_source( &select_parts, @@ -1351,7 +1406,11 @@ impl<'a> SqlGenerator<'a> { .flatten() .chain(metric.filters.iter().map(String::as_str)) { - for column in semantic_column_references(fragment)? { + let fragment = crate::core::replace_model_placeholder( + fragment, + self.graph.metric_owner(reference), + )?; + for column in semantic_column_references(&fragment)? { if let Some(model) = &column.model { if self.graph.get_model(model).is_none() { return Err(SidemanticError::InvalidConfig(format!( @@ -1394,17 +1453,32 @@ impl<'a> SqlGenerator<'a> { } } } - if owners.len() != 1 { + // A graph-level aggregate can name its source through its public name, + // e.g. events.p95.latency with physical SQL input `latency`. + if owners.is_empty() && metric.agg.is_some() { + if let Some((model, _)) = reference.split_once('.') { + if self.graph.get_model(model).is_some() { + owners.insert(model.to_owned()); + } + } + } + if owners.is_empty() { return Err(SidemanticError::UnsupportedSemanticFeatures { capabilities: vec![format!("metric.graph_scope.{reference}")], }); } - Ok(owners.into_iter().collect()) + // This is dependency discovery, not assignment of a synthetic owner. + // Independent aggregate planning and public alias resolution need the + // full source set; source-local callers check for a single owner. + let mut owners: Vec<_> = owners.into_iter().collect(); + owners.sort(); + Ok(owners) } fn metric_reference_tokens(&self, expression: &str) -> Result> { if self.graph.has_strict_metric_scope() { - return Ok(semantic_column_references(expression)? + let expression = crate::core::replace_model_placeholder(expression, None)?; + return Ok(semantic_column_references(&expression)? .into_iter() .filter(|column| !column.aggregate_input) .map(|column| column.name()) @@ -1670,7 +1744,8 @@ impl<'a> SqlGenerator<'a> { models: &mut HashSet, ) -> Result<()> { if self.graph.has_strict_metric_scope() { - for column in semantic_column_references(expr)? { + let expr = crate::core::replace_model_placeholder(expr, None)?; + for column in semantic_column_references(&expr)? { if let Some(model) = column.model { if self.graph.get_model(&model).is_some() { models.insert(model); @@ -1876,7 +1951,10 @@ impl<'a> SqlGenerator<'a> { let metric = self.metric_for_ref(metric_ref)?; if self.graph.has_strict_metric_scope() && metric.r#type == MetricType::Derived { - for column in semantic_column_references(metric.sql_expr())? { + let expression = self.imported_calculation_expression(metric, &metric_ref.model)?; + let expression = + crate::core::replace_model_placeholder(&expression, Some(&metric_ref.model))?; + for column in semantic_column_references(&expression)? { if column.aggregate_input { deps.insert(( column.model.unwrap_or_else(|| metric_ref.model.clone()), @@ -1964,7 +2042,8 @@ impl<'a> SqlGenerator<'a> { deps: &mut HashSet<(String, String)>, ) -> Result<()> { if self.graph.has_strict_metric_scope() { - for column in semantic_column_references(expr)? { + let expr = crate::core::replace_model_placeholder(expr, Some(default_model))?; + for column in semantic_column_references(&expr)? { let model = column.model.unwrap_or_else(|| default_model.to_string()); if self.graph.get_model(&model).is_some() { deps.insert((model, column.field)); @@ -2125,6 +2204,20 @@ impl<'a> SqlGenerator<'a> { } let names: Vec<_> = known_fields.iter().map(String::as_str).collect(); let (head, suffix) = crate::sql::split_order_field(item, &names); + // Boundary dialect normalization preserves identifier quotes. Resolve + // the parsed column name, while retaining the original SQL if it is + // not a selected semantic field. + let normalized_head = if names.contains(&head) { + None + } else if let Ok(Expression::Column(column)) = parse_semantic_expression(head) { + Some(match column.table { + Some(table) => format!("{}.{}", table.name, column.name.name), + None => column.name.name, + }) + } else { + None + }; + let reference = normalized_head.as_deref().unwrap_or(head); // Resolve the field and NULL placement from the same suffix. Python's // ordinary SQLGlot builder defaults to ascending NULLs first, descending last. let suffix = if suffix.contains("NULLS") { @@ -2138,14 +2231,14 @@ impl<'a> SqlGenerator<'a> { }; for metric_ref in metric_refs { - if metric_ref.name == head || metric_ref.alias == head { + if metric_ref.name == reference || metric_ref.alias == reference { let alias = self.output_alias(&metric_ref.model, &metric_ref.alias, alias_collisions); return format!("{}{}", self.quote_identifier(&alias), suffix); } } - let Ok((model, field, granularity)) = self.graph.parse_reference(head) else { + let Ok((model, field, granularity)) = self.graph.parse_reference(reference) else { return format!("{head}{suffix}"); }; @@ -2308,6 +2401,7 @@ impl<'a> SqlGenerator<'a> { if window_models.is_empty() { non_window_filters.push(filter.clone()); } else { + let filter = self.window_row_filter(filter, &window_models)?; for model_name in window_models { window_filters_by_model .entry(model_name) @@ -2573,7 +2667,11 @@ impl<'a> SqlGenerator<'a> { cohort_metrics.push(metric_ref.clone()); } _ => { - let explicit_ref = format!("{}.{}", metric_ref.model, metric_ref.name); + let explicit_ref = if metric_ref.graph_metric { + metric_ref.name.clone() + } else { + format!("{}.{}", metric_ref.model, metric_ref.name) + }; if seen_metrics.insert(explicit_ref.clone()) { base_metrics.push(explicit_ref); } @@ -3957,6 +4055,16 @@ impl<'a> SqlGenerator<'a> { let mut seen_models = HashSet::new(); for metric_ref in metrics { + // Graph calculations are scalar unless dimensions were requested. + // Only cumulative graph metrics inherit a dependency's time grain + // because their window needs an ordering dimension. + if self + .graph + .get_metric(metric_ref) + .is_some_and(|metric| metric.r#type != MetricType::Cumulative) + { + continue; + } let model_name = if let Some((model_name, _, _)) = self.exact_metric_reference(metric_ref)? { model_name @@ -4269,6 +4377,7 @@ impl<'a> SqlGenerator<'a> { offset: Option, preagg_database: Option<&str>, preagg_schema: Option<&str>, + ungrouped: bool, ) -> Result> { let model = self.graph.get_model(model_name).ok_or_else(|| { let available: Vec<&str> = self.graph.models().map(|m| m.name.as_str()).collect(); @@ -4291,6 +4400,25 @@ impl<'a> SqlGenerator<'a> { return Ok(None); }; query_metric_names.extend(filter_plan.metrics.iter().cloned()); + if ungrouped + && (!filter_plan.aggregates.is_empty() + || metric_refs.iter().any(|reference| { + model.get_metric(&reference.name).is_none_or(|metric| { + metric.r#type != MetricType::Simple + || !matches!( + metric.agg, + Some( + Aggregation::Sum + | Aggregation::Count + | Aggregation::Min + | Aggregation::Max + ) + ) + }) + })) + { + return Ok(None); + } // Ordering is limited to selected semantic outputs on this route. let mut rewritten_order = Vec::new(); @@ -4327,6 +4455,17 @@ impl<'a> SqlGenerator<'a> { let mut best_match: Option<(crate::core::PreAggregation, i32)> = None; for preagg in &model.pre_aggregations { + if ungrouped { + let keys = model.primary_keys(); + if keys.is_empty() + || !preagg + .dimensions + .as_ref() + .is_some_and(|dimensions| keys.iter().all(|key| dimensions.contains(key))) + { + continue; + } + } if !self.preaggregation_can_satisfy_query( model, preagg, @@ -4366,6 +4505,7 @@ impl<'a> SqlGenerator<'a> { offset, preagg_database, preagg_schema, + ungrouped, )?; Ok(Some(format!("{preagg_sql}\n-- used_preagg=true"))) @@ -4982,6 +5122,7 @@ impl<'a> SqlGenerator<'a> { offset: Option, preagg_database: Option<&str>, preagg_schema: Option<&str>, + ungrouped: bool, ) -> Result { let preagg_table = self.preaggregation_source(model, preagg, preagg_database, preagg_schema)?; @@ -5023,14 +5164,17 @@ impl<'a> SqlGenerator<'a> { )); } for metric_ref in metric_refs { - let expression = self - .preaggregation_metric_expression( + let expression = if ungrouped { + self.quote_identifier(&format!("{}_raw", metric_ref.name)) + } else { + self.preaggregation_metric_expression( model, preagg, &metric_ref.name, &mut HashSet::new(), ) - .expect("matcher checked aggregate state"); + .expect("matcher checked aggregate state") + }; select_parts.push(format!( "{expression} AS {}", self.quote_identifier(&metric_ref.alias) @@ -5043,7 +5187,7 @@ impl<'a> SqlGenerator<'a> { if !filters.is_empty() { sql.push_str(&format!("\nWHERE {}", filters.join(" AND "))); } - if !dimension_refs.is_empty() { + if !ungrouped && !dimension_refs.is_empty() { let group_by: Vec<_> = (1..=dimension_refs.len()) .map(|index| index.to_string()) .collect(); @@ -5075,37 +5219,35 @@ impl<'a> SqlGenerator<'a> { ) -> Result<(Vec, Vec)> { let mut where_filters = Vec::new(); let mut having_filters = Vec::new(); - let ref_re = regex::Regex::new(r"\b([A-Za-z_][A-Za-z0-9_]*)\.([A-Za-z_][A-Za-z0-9_]*)\b") - .expect("valid model.field regex"); - for filter in filters { - let mut rewritten = filter.clone(); - let mut uses_metric = false; - - for cap in ref_re.captures_iter(filter) { - let Some(model_match) = cap.get(1) else { - continue; - }; - let Some(field_match) = cap.get(2) else { - continue; - }; - let model_name = model_match.as_str(); - let field_name = field_match.as_str(); - let Some(model) = self.graph.get_model(model_name) else { - continue; + let mut replacements = HashMap::new(); + for column in crate::core::outer_semantic_column_references(filter)? { + let reference = column.name(); + let metric = if self.graph.get_metric(&reference).is_some() { + self.exact_metric_reference(&reference)? + } else if let Some(model) = column.model.as_deref() { + self.graph.get_model(model).and_then(|model| { + model + .get_metric(&column.field) + .map(|_| (model.name.clone(), column.field.clone(), false)) + }) + } else { + self.exact_metric_reference(&reference)? }; - if model.get_metric(field_name).is_none() { - continue; + if let Some((model, name, _)) = metric { + replacements.insert( + (column.model, column.field), + self.quote_identifier(&self.output_alias(&model, &name, collisions)), + ); } - - uses_metric = true; - let replacement = self.output_alias(model_name, field_name, collisions); - let full_ref = format!("{model_name}.{field_name}"); - rewritten = rewritten.replace(&full_ref, &replacement); } - - if uses_metric { - having_filters.push(rewritten); + if !replacements.is_empty() { + having_filters.push(self.emit_expression( + &crate::core::replace_outer_semantic_columns( + parse_semantic_expression(filter)?, + &replacements, + )?, + )?); } else { where_filters.push(filter.clone()); } @@ -5235,6 +5377,62 @@ impl<'a> SqlGenerator<'a> { .is_empty() } + /// A metric beside a window dimension in a row predicate denotes its input, + /// before aggregation. Keep that predicate out of the child's HAVING clause. + fn window_row_filter(&self, filter: &str, owners: &HashSet) -> Result { + let mut replacements = HashMap::new(); + for column in crate::core::outer_semantic_column_references(filter)? { + let Some(owner) = column.model.as_deref() else { + continue; + }; + let Some(model) = self.graph.get_model(owner) else { + continue; + }; + let Some(metric) = model.get_metric(&column.field) else { + continue; + }; + if !owners.contains(owner) + || metric.r#type != MetricType::Simple + || metric.sql_is_complete + { + return Err(SidemanticError::UnsupportedSemanticFeatures { + capabilities: vec!["aggregation.mixed_row_aggregate_filter".into()], + }); + } + let alias = self.model_alias(owner); + let mut raw = self + .metric_raw_expression(metric, model)? + .replace("{model}", owner); + if !metric.filters.is_empty() { + let predicate = self.normalize_metric_filters(&metric.filters, owner, &alias)?; + raw = format!("CASE WHEN {predicate} THEN {raw} END"); + } + let mut inputs = HashMap::new(); + for input in semantic_column_references(&raw)? { + if input.model.as_deref().is_some_and(|source| { + source != owner && source != alias && source != model.table_name() + }) { + return Err(SidemanticError::UnsupportedSemanticFeatures { + capabilities: vec!["aggregation.mixed_row_aggregate_filter".into()], + }); + } + inputs.insert( + (input.model, input.field.clone()), + format!("{alias}.{}", self.quote_identifier(&input.field)), + ); + } + let raw = self.emit_expression(&crate::core::replace_semantic_columns( + parse_semantic_expression(&raw)?, + &inputs, + )?)?; + replacements.insert((column.model, column.field), format!("({raw})")); + } + self.emit_expression(&crate::core::replace_outer_semantic_columns( + parse_semantic_expression(filter)?, + &replacements, + )?) + } + fn filter_window_dimension_models( &self, filter: &str, @@ -5292,10 +5490,21 @@ impl<'a> SqlGenerator<'a> { capabilities: vec!["filter.computed_key_source".into()], }); } - let source = if computed_keys.contains(&column.field) { - self.key_sql(model, &column.field, None)? - } else if let Some(dimension) = model.get_dimension(&column.field) { - let source = dimension.sql_expr().replace("{model}", model_name); + let (field, grain) = column + .field + .rsplit_once("__") + .filter(|(field, _)| model.get_dimension(field).is_some()) + .map_or((column.field.as_str(), None), |(field, grain)| { + (field, Some(grain)) + }); + let mut source = if computed_keys.contains(field) { + self.key_sql(model, field, None)? + } else if let Some(dimension) = model.get_dimension(field) { + let source = dimension + .sql + .clone() + .unwrap_or_else(|| self.quote_identifier(&dimension.name)) + .replace("{model}", model_name); let mut inputs = HashMap::new(); for input in semantic_column_references(&source)? { if input @@ -5319,6 +5528,9 @@ impl<'a> SqlGenerator<'a> { } else { self.quote_identifier(&column.field) }; + if let Some(grain) = grain { + source = self.date_trunc_sql(grain, &source)?; + } let replacement = match polyglot_sql::parse_one(&format!("SELECT {source}"), self.dialect) { Ok(Expression::Select(select)) @@ -5391,8 +5603,25 @@ impl<'a> SqlGenerator<'a> { Ok(rendered.join(" AND ")) } - fn normalize_cte_source_expression(&self, expr: &str) -> String { - expr.replace("{model}.", "").replace("{model}", "") + fn normalize_cte_source_expression(&self, expr: &str, model: &Model) -> Result { + let expr = expr.replace("{model}.", "").replace("{model}", ""); + let mut replacements = HashMap::new(); + for column in semantic_column_references(&expr)? { + if column.model.as_deref().is_some_and(|owner| { + owner == model.name + || owner == self.model_alias(&model.name) + || owner == model.table_name() + }) { + replacements.insert( + (column.model, column.field.clone()), + self.quote_identifier(&column.field), + ); + } + } + self.emit_expression(&crate::core::replace_semantic_columns( + parse_semantic_expression(&expr)?, + &replacements, + )?) } fn normalize_select_expression(&self, expr: &str, alias: &str) -> String { @@ -5403,13 +5632,39 @@ impl<'a> SqlGenerator<'a> { &self, dimension: &crate::core::Dimension, alias: &str, - ) -> String { - let expr = dimension.sql_expr(); - if expr.contains("{model}") { - self.normalize_select_expression(expr, alias) - } else { - format!("{}.{}", alias, expr) + ) -> Result { + if dimension.sql.is_none() { + // A default dimension is a physical identifier, not authored SQL. + // Parsing names such as "Order Date" as expressions loses that + // distinction (and can interpret the second word as an alias). + return Ok(format!( + "{alias}.{}", + self.quote_identifier(&dimension.name) + )); + } + let expr = self.normalize_select_expression(dimension.sql_expr(), alias); + // Qualify the input columns, not the complete expression: prefixing + // CASE, arithmetic or a function call produces invalid SQL. + let mut replacements = HashMap::new(); + for column in semantic_column_references(&expr)? { + if column.model.is_none() + || column.model.as_deref() == alias.strip_suffix("_cte") + || column.model.as_deref().is_some_and(|owner| { + self.graph + .get_model(alias.strip_suffix("_cte").unwrap_or(alias)) + .is_some_and(|model| model.sql.is_none() && owner == model.table_name()) + }) + { + replacements.insert( + (column.model, column.field.clone()), + format!("{alias}.{}", self.quote_identifier(&column.field)), + ); + } } + self.emit_expression(&crate::core::replace_semantic_columns( + parse_semantic_expression(&expr)?, + &replacements, + )?) } fn is_simple_identifier(identifier: &str) -> bool { @@ -5420,14 +5675,23 @@ impl<'a> SqlGenerator<'a> { fn quote_identifier(&self, identifier: &str) -> String { if Self::is_simple_identifier(identifier) { - identifier.to_string() - } else { - // A quoted identifier is a leaf AST node with no unsupported operations. + // The SQL generator owns the dialect's reserved-word table. polyglot_sql::generate( - &Expression::Identifier(Identifier::quoted(identifier)), + &Expression::Identifier(Identifier::new(identifier)), self.dialect, ) - .expect("quoted identifier generation is infallible") + .expect("identifier generation is infallible") + } else { + // Keep the name literal: the pinned generator otherwise splits + // ASC/DESC and index-prefix suffixes even inside quoted identifiers. + let dialect = polyglot_sql::Dialect::get(self.dialect); + let style = &dialect.generator_config().identifier_quote_style; + format!( + "{}{}{}", + style.start, + identifier.replace(style.end, &format!("{}{}", style.end, style.end)), + style.end + ) } } @@ -5554,9 +5818,11 @@ impl<'a> SqlGenerator<'a> { }); } MetricType::Simple => self.simple_metric_reference_sql(metric, &metric_name, &alias)?, - MetricType::Derived => { - self.expand_derived_metric_inner(metric.sql_expr(), &model_name, visited)? - } + MetricType::Derived => self.expand_derived_metric_inner( + &self.imported_calculation_expression(metric, &model_name)?, + &model_name, + visited, + )?, MetricType::Ratio => { let num_ref = metric.numerator.as_deref().unwrap_or("1"); let den_ref = metric.denominator.as_deref().unwrap_or("1"); @@ -5673,8 +5939,9 @@ impl<'a> SqlGenerator<'a> { default_model: &str, visited: &mut HashSet<(String, String, bool)>, ) -> Result { - let parsed = parse_semantic_expression(expression)?; - let columns = semantic_column_references(expression)?; + let expression = crate::core::replace_model_placeholder(expression, Some(default_model))?; + let parsed = parse_semantic_expression(&expression)?; + let columns = semantic_column_references(&expression)?; let has_aggregate = columns.iter().any(|column| column.aggregate_input); let mut replacements = HashMap::new(); for column in columns { @@ -5813,6 +6080,33 @@ impl<'a> SqlGenerator<'a> { fn expand_filter_with_polyglot(&self, filter: &str) -> Result { let parsed = self.parse_where_expr(filter)?; let mut keys = HashMap::new(); + for column in crate::core::outer_semantic_column_references(filter)? { + let matches: Vec<_> = self + .graph + .models() + .filter_map(|model| { + if column.model.as_deref().is_some_and(|owner| { + owner.strip_suffix("_cte").unwrap_or(owner) != model.name + }) { + return None; + } + model + .get_dimension(&column.field) + .filter(|dimension| dimension.window.is_some()) + .map(|dimension| (model, dimension)) + }) + .collect(); + if let [(model, dimension)] = matches.as_slice() { + keys.insert( + (column.model, column.field), + format!( + "{}.{}", + self.model_alias(&model.name), + self.quote_identifier(&Self::window_dimension_alias(dimension)) + ), + ); + } + } if self.has_computed_key_models(&self.find_filter_models(&[filter.to_string()]))? { for column in semantic_column_references(filter)? { if let Some(model) = column @@ -5851,7 +6145,10 @@ impl<'a> SqlGenerator<'a> { sql: format!( "{}.{}", self.model_alias(&model.name), - dimension.sql_expr() + dimension + .sql + .clone() + .unwrap_or_else(|| self.quote_identifier(&dimension.name)) ), })); } @@ -6057,6 +6354,252 @@ mod tests { Relationship, }; + #[test] + fn implicit_spaced_dimensions_remain_identifiers_in_grains_and_filters() { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("Sales", "id") + .with_table("sales") + .with_dimension(Dimension::time("Order Date").with_granularity("month")) + .with_dimension(Dimension::categorical("Order Status")) + .with_metric(Metric::sum("qty", "quantity")), + ) + .unwrap(); + let generator = SqlGenerator::new(&graph); + for (dimension, filter) in [ + ("Order Date", None), + ("Order Status", None), + ("Order Status", Some("Sales.\"Order Status\" = 'shipped'")), + ] { + let mut query = SemanticQuery::new() + .with_metrics(vec!["Sales.qty".into()]) + .with_dimensions(vec![format!("Sales.{dimension}")]); + query.order_by = vec![format!("Sales.{dimension}")]; + query.filters = filter.into_iter().map(str::to_owned).collect(); + let sql = generator.generate(&query).unwrap(); + crate::semantic_input::dialects::parse(&sql, DialectType::DuckDB).unwrap(); + assert!(sql.contains(&format!("\"{dimension}\"")), "{sql}"); + if filter.is_some() { + assert!(sql.contains("\"Order Status\" = 'shipped'"), "{sql}"); + } + } + } + + #[test] + fn source_qualifiers_bind_at_cte_and_dimension_projection_boundaries() { + for source_sql in [None, Some("SELECT * FROM sales")] { + let mut model = Model::new("sales_model", "id") + .with_table("sales") + .with_dimension(Dimension::categorical("sales_id").with_sql("sales.id")) + .with_metric(Metric::sum("amount", "sales_model.amount * 2")); + model.sql = source_sql.map(str::to_string); + let mut graph = SemanticGraph::new(); + graph.add_model(model).unwrap(); + let generator = SqlGenerator::new(&graph); + let model = graph.get_model("sales_model").unwrap(); + let expression = generator + .normalize_cte_source_expression( + "sales_model.amount + CASE WHEN sales.amount > 0 THEN 1 ELSE 0 END", + model, + ) + .unwrap(); + assert!(semantic_column_references(&expression) + .unwrap() + .iter() + .all(|column| column.model.is_none())); + let sql = generator + .generate(&SemanticQuery::new().with_metrics(vec!["sales_model.amount".into()])) + .unwrap(); + assert!(!sql.contains("sales_model.amount * 2"), "{sql}"); + if source_sql.is_none() { + let sql = generator + .generate( + &SemanticQuery::new() + .with_metrics(vec!["sales_model.amount".into()]) + .with_dimensions(vec!["sales_model.sales_id".into()]), + ) + .unwrap(); + assert!(sql.contains("sales_model_cte.id AS sales_id"), "{sql}"); + } + } + } + + #[test] + fn reserved_complete_aggregate_inputs_remain_quoted() { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("orders", "id") + .with_table("orders") + .with_metric(Metric::derived("total", "SUM({model}.\"select\")")), + ) + .unwrap(); + let generator = SqlGenerator::new(&graph); + let sql = generator + .generate(&SemanticQuery::new().with_metrics(vec!["orders.total".into()])) + .unwrap(); + crate::semantic_input::dialects::parse(&sql, DialectType::DuckDB).unwrap(); + assert!(sql.contains("\"select\""), "{sql}"); + } + + #[test] + fn identifier_quoting_uses_reserved_words_and_keeps_literal_suffixes() { + let graph = SemanticGraph::new(); + for (dialect, quote) in [(DialectType::DuckDB, '"'), (DialectType::BigQuery, '`')] { + let generator = SqlGenerator::new(&graph).with_dialect(dialect); + assert_eq!( + generator.quote_identifier("select"), + format!("{quote}select{quote}") + ); + for name in ["Order Date", "value DESC", "value(10)"] { + assert_eq!( + generator.quote_identifier(name), + format!("{quote}{name}{quote}") + ); + } + } + } + + #[test] + fn computed_dimensions_qualify_each_physical_input() { + for expression in [ + "CASE WHEN amount < 20 THEN 'small' ELSE NULL END", + "COALESCE(amount, 0) + adjustment", + "{model}.amount + adjustment", + "orders.amount + adjustment", + ] { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("orders", "id") + .with_table("raw_orders") + .with_dimension(Dimension::categorical("band").with_sql(expression)) + .with_metric(Metric::count("rows")), + ) + .unwrap(); + let generator = SqlGenerator::new(&graph); + let dimension = graph + .get_model("orders") + .unwrap() + .get_dimension("band") + .unwrap(); + let rendered = generator + .dimension_select_expression(dimension, "orders_cte") + .unwrap(); + let columns = semantic_column_references(&rendered).unwrap(); + assert!(!columns.is_empty(), "{rendered}"); + assert!( + columns + .iter() + .all(|column| column.model.as_deref() == Some("orders_cte")), + "{rendered}" + ); + for ungrouped in [false, true] { + let query = SemanticQuery::new() + .with_metrics(vec!["orders.rows".into()]) + .with_dimensions(vec!["orders.band".into()]) + .with_ungrouped(ungrouped); + let sql = generator.generate(&query).unwrap(); + polyglot_sql::parse_one(&sql, DialectType::DuckDB).unwrap(); + assert!(sql.contains(&rendered), "{sql}"); + } + } + } + + #[test] + fn graph_metric_predicate_is_having_and_preserves_string_literals() { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("orders", "id") + .with_table("orders") + .with_dimension(Dimension::categorical("region")), + ) + .unwrap(); + let mut count = Metric::count("one"); + count.filters = vec!["status = 'paid'".into()]; + graph.add_metric_unvalidated(count).unwrap(); + graph + .set_metric_scopes(HashMap::from([("one".into(), "orders".into())])) + .unwrap(); + let query = SemanticQuery::new() + .with_metrics(vec!["one".into()]) + .with_dimensions(vec!["orders.region".into()]) + .with_filters(vec!["one > 0".into(), "orders.region <> 'one'".into()]); + let sql = SqlGenerator::new(&graph).generate(&query).unwrap(); + assert!(sql.contains("HAVING one > 0"), "{sql}"); + assert!(!sql.contains("WHERE one > 0"), "{sql}"); + assert!(sql.contains("region <> 'one'"), "{sql}"); + } + + #[test] + fn grained_filter_binds_physical_expression_without_selected_alias() { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("events", "id") + .with_table("events") + .with_dimension(Dimension::time("created_at").with_sql("occurred_at")) + .with_dimension(Dimension::new("gross").with_sql("unit_price * quantity")) + .with_metric(Metric::sum("revenue", "amount")), + ) + .unwrap(); + for dialect in [DialectType::DuckDB, DialectType::PostgreSQL] { + let sql = SqlGenerator::new(&graph) + .with_dialect(dialect) + .generate( + &SemanticQuery::new() + .with_metrics(vec!["events.revenue".into()]) + .with_filters(vec![ + "events.created_at__month = DATE '2024-02-01'".into(), + "events.gross >= 20".into(), + ]), + ) + .unwrap(); + assert!(!sql.contains("created_at__month"), "{sql}"); + assert!( + sql.to_ascii_uppercase() + .contains("DATE_TRUNC('MONTH', OCCURRED_AT)"), + "{sql}" + ); + assert!(sql.contains("(unit_price * quantity) >= 20"), "{sql}"); + polyglot_sql::parse_one(&sql, dialect).unwrap(); + } + } + + #[test] + fn scalar_graph_metrics_do_not_inherit_model_default_time_dimension() { + let mut graph = SemanticGraph::new(); + let mut model = Model::new("orders", "id") + .with_table("orders") + .with_dimension(Dimension::time("day")) + .with_metric(Metric::sum("revenue", "amount")); + model.default_time_dimension = Some("day".into()); + model.default_grain = Some("day".into()); + graph.add_model(model).unwrap(); + graph + .add_metric_unvalidated(Metric::derived("total_revenue", "orders.revenue")) + .unwrap(); + graph.set_metric_scopes(HashMap::new()).unwrap(); + let generator = SqlGenerator::new(&graph); + assert!(generator + .apply_default_time_dimensions(&["total_revenue".into()], &[]) + .unwrap() + .is_empty()); + assert_eq!( + generator + .apply_default_time_dimensions(&["orders.revenue".into()], &[]) + .unwrap(), + vec!["orders.day__day"] + ); + let sql = generator + .generate(&SemanticQuery::new().with_metrics(vec!["total_revenue".into()])) + .unwrap(); + assert!(!sql.contains("GROUP BY"), "{sql}"); + assert!(!sql.contains("DATE_TRUNC"), "{sql}"); + } + #[test] fn computed_time_key_projection_applies_grain_and_timezone() { let mut dimension = Dimension::time("event_time").with_sql("CAST(raw_time AS TIMESTAMP)"); @@ -6177,6 +6720,57 @@ mod tests { assert!(!sql.contains("orders_preagg_daily"), "{sql}"); } + #[test] + fn ungrouped_rollups_require_complete_primary_key_and_per_row_state() { + let source = serde_json::json!({ + "name":"orders", "table":"raw_orders", "primary_key":"id", + "dimensions":[{"name":"id", "type":"categorical"}, {"name":"status", "type":"categorical"}], + "metrics":[{"name":"revenue", "agg":"sum", "sql":"amount"}, {"name":"average", "agg":"avg", "sql":"amount"}], + "pre_aggregations":[{"name":"detail", "dimensions":["id", "status"], "measures":["revenue", "average"]}] + }); + for (keys, dimensions, metric, filter, expected) in [ + (vec!["id"], vec!["id", "status"], "revenue", None, true), + (vec!["id"], vec!["status"], "revenue", None, false), + (vec![], vec!["id", "status"], "revenue", None, false), + (vec!["id", "status"], vec!["id"], "revenue", None, false), + ( + vec!["id", "status"], + vec!["id", "status"], + "revenue", + None, + true, + ), + (vec!["id"], vec!["id", "status"], "average", None, false), + ( + vec!["id"], + vec!["id", "status"], + "revenue", + Some("orders.revenue > 1"), + false, + ), + ] { + let mut model: Model = serde_json::from_value(source.clone()).unwrap(); + model.primary_key = keys.first().copied().unwrap_or_default().into(); + model.primary_key_columns = keys.iter().map(|key| (*key).into()).collect(); + model.pre_aggregations[0].dimensions = + Some(dimensions.iter().map(|name| (*name).into()).collect()); + let mut graph = SemanticGraph::new(); + graph.add_model(model).unwrap(); + let mut query = SemanticQuery::new() + .with_metrics(vec![format!("orders.{metric}")]) + .with_dimensions(vec!["orders.id".into()]) + .with_ungrouped(true) + .with_use_preaggregations(true); + query.filters = filter.into_iter().map(str::to_owned).collect(); + let sql = SqlGenerator::new(&graph).generate(&query).unwrap(); + assert_eq!(sql.contains("used_preagg=true"), expected, "{sql}"); + if expected { + assert!(!sql.contains("GROUP BY"), "{sql}"); + assert!(sql.contains("revenue_raw AS revenue"), "{sql}"); + } + } + } + #[test] fn distinct_rollup_requires_exact_projected_grain() { let mut model = rollup_model(); @@ -6565,9 +7159,9 @@ mod tests { let sql = generator.generate(&query).unwrap(); - assert!(sql.contains("sales.amount AS revenue_raw"), "{sql}"); + assert!(sql.contains("amount AS revenue_raw"), "{sql}"); + assert!(sql.contains("SUM(sales_cte.revenue_raw)"), "{sql}"); assert!(sql.contains("FROM sales"), "{sql}"); - assert!(!sql.contains("\n amount AS revenue_raw"), "{sql}"); assert!(!sql.contains("FROM orders"), "{sql}"); } @@ -6597,8 +7191,8 @@ mod tests { let query = SemanticQuery::new().with_metrics(vec!["conversion_rate".into()]); let sql = generator.generate(&query).unwrap(); - assert!(sql.contains("orders.signups AS signups_raw"), "{sql}"); - assert!(sql.contains("orders.visitors AS visitors_raw"), "{sql}"); + assert!(sql.contains("signups AS signups_raw"), "{sql}"); + assert!(sql.contains("visitors AS visitors_raw"), "{sql}"); } #[test] @@ -6813,6 +7407,35 @@ models: ); } + #[test] + fn test_quoted_semantic_order_by_rewrites_to_output_aliases() { + let graph = create_test_graph(); + let query = SemanticQuery::new() + .with_metrics(vec!["orders.revenue".into()]) + .with_dimensions(vec!["orders.status".into()]) + .with_order_by(vec![ + "\"orders\".\"revenue\" DESC NULLS FIRST".into(), + "\"orders\".\"status\" ASC NULLS LAST".into(), + ]); + let sql = SqlGenerator::new(&graph).generate(&query).unwrap(); + let statement = polyglot_sql::parse_one(&sql, SOURCE_DIALECT).unwrap(); + let Expression::Select(select) = statement else { + panic!("expected SELECT: {sql}"); + }; + let order = select.order_by.unwrap(); + for (item, expected) in order.expressions.iter().zip(["revenue", "status"]) { + let Expression::Column(column) = &item.this else { + panic!("expected result column: {sql}"); + }; + assert!(column.table.is_none(), "{sql}"); + assert_eq!(column.name.name, expected, "{sql}"); + } + assert!( + sql.contains("revenue DESC NULLS FIRST, status ASC NULLS LAST"), + "{sql}" + ); + } + #[test] fn test_time_comparison_calendar_lookup_partitions_by_non_time_dimensions() { let mut graph = SemanticGraph::new(); diff --git a/sidemantic-rs/src/sql/generator/aggregate_plan.rs b/sidemantic-rs/src/sql/generator/aggregate_plan.rs index 66a984a80..82d19e421 100644 --- a/sidemantic-rs/src/sql/generator/aggregate_plan.rs +++ b/sidemantic-rs/src/sql/generator/aggregate_plan.rs @@ -28,6 +28,7 @@ struct Plan<'a, 'g> { expressions: HashMap, active: HashSet, cross_source_calculation: bool, + inline_aggregates: bool, } fn unsupported(capability: &str) -> SidemanticError { @@ -36,10 +37,158 @@ fn unsupported(capability: &str) -> SidemanticError { } } +fn count_aggregate(expression: &Expression) -> bool { + match expression { + Expression::Count(_) + | Expression::CountIf(_) + | Expression::ApproxDistinct(_) + | Expression::ApproxCountDistinct(_) => true, + Expression::Filter(filter) => count_aggregate(&filter.this), + Expression::WithinGroup(group) => count_aggregate(&group.this), + _ => false, + } +} + impl<'a, 'g> Plan<'a, 'g> { + /// Split authored aggregate calls before expanding scalar metric references. + /// Each call must read one source; arithmetic combines its grouped output. + fn expand_calculation( + &mut self, + node: &mut serde_json::Value, + context: Option<&str>, + bindings: &HashSet, + ) -> Result<()> { + let aggregate_node = crate::core::is_aggregate_ast_node(node); + if let serde_json::Value::Object(fields) = node { + let kind = (fields.len() == 1).then(|| fields.keys().next().unwrap().as_str()); + if kind == Some("lambda") { + let lambda = fields.get_mut("lambda").unwrap(); + let mut bindings = bindings.clone(); + if let Some(parameters) = lambda["parameters"].as_array() { + bindings.extend( + parameters + .iter() + .filter_map(|parameter| parameter["name"].as_str().map(str::to_owned)), + ); + } + return self.expand_calculation(&mut lambda["body"], context, &bindings); + } + if matches!( + kind, + Some("window" | "window_function" | "select" | "subquery" | "raw") + ) { + return Err(unsupported("calculation_shape")); + } + let aggregate = aggregate_node || matches!(kind, Some("filter" | "within_group")); + if aggregate || kind == Some("column") { + let expression: Expression = serde_json::from_value(node.clone()) + .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; + let sql = self.generator.emit_expression(&expression)?; + let replacement = if aggregate { + self.inline_aggregates = true; + let columns = semantic_column_references(&sql)?; + let mut owners = HashSet::new(); + for column in columns { + // Hoisting an aggregate out of a lambda would change + // the binding of its local inputs. + if bindings.contains(column.model.as_deref().unwrap_or(&column.field)) { + return Err(unsupported("calculation_shape")); + } + let owner = column + .model + .as_deref() + .or(context) + .ok_or_else(|| unsupported("unscoped_leaf"))?; + if self.generator.graph.get_model(owner).is_none() { + return Err(unsupported("cross_source_raw_input")); + } + owners.insert(owner.to_string()); + } + if owners.is_empty() { + owners.extend(context.map(str::to_string)); + } + if owners.len() != 1 { + return Err(unsupported("cross_source_raw_input")); + } + let model = owners.into_iter().next().unwrap(); + if !self.models.contains(&model) { + self.models.push(model.clone()); + } + let alias = format!("__sidemantic_metric_{}", self.leaves.len()); + let mut metric = Metric::derived(&alias, sql); + metric.sql_is_complete = true; + self.leaves.push(Leaf { + reference: format!("{model}.{alias}"), + model: model.clone(), + metric, + alias: alias.clone(), + }); + let output = format!( + "{}.{}", + self.generator.quote_identifier(&format!("{model}_preagg")), + self.generator.quote_identifier(&alias), + ); + if count_aggregate(&expression) { + format!("COALESCE({output}, 0)") + } else { + output + } + } else { + let column = semantic_column_references(&sql)?.remove(0); + if bindings.contains(column.model.as_deref().unwrap_or(&column.field)) { + return Ok(()); + } + self.expand(&column.name(), context)? + }; + *node = + serde_json::to_value(parse_semantic_expression(&format!("({replacement})"))?) + .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; + return Ok(()); + } + } + match node { + serde_json::Value::Object(fields) => { + for child in fields.values_mut() { + self.expand_calculation(child, context, bindings)?; + } + } + serde_json::Value::Array(children) => { + for child in children { + self.expand_calculation(child, context, bindings)?; + } + } + _ => {} + } + Ok(()) + } + + fn graph_metric_context(&self, reference: &str, metric: &Metric) -> Result> { + if let Some(owner) = self.generator.graph.metric_owner(reference) { + return Ok(Some(owner.to_string())); + } + // Imported graph aggregates can bind their source in qualified SQL + // without an explicit owner annotation. Scalar calculations stay unowned. + if metric.agg.is_some() && !metric.sql_is_complete { + let owners = self + .generator + .graph_metric_owner_models(reference, metric)?; + if owners.len() == 1 { + return Ok(owners.into_iter().next()); + } + } + Ok(None) + } + fn resolve(&self, reference: &str, context: Option<&str>) -> Result> { let graph = self.generator.graph; if let Some((model_name, name)) = reference.split_once('.') { + if let Some(metric) = graph.get_metric(reference) { + return Ok(Some(ResolvedMetric { + reference: reference.to_string(), + context: self.graph_metric_context(reference, metric)?, + metric: metric.clone(), + })); + } return Ok(graph.get_model(model_name).and_then(|model| { model.get_metric(name).map(|metric| ResolvedMetric { reference: reference.to_string(), @@ -63,7 +212,7 @@ impl<'a, 'g> Plan<'a, 'g> { if let Some(metric) = graph.get_metric(reference) { return Ok(Some(ResolvedMetric { reference: reference.to_string(), - context: graph.metric_owner(reference).map(str::to_string), + context: self.graph_metric_context(reference, metric)?, metric: metric.clone(), })); } @@ -108,8 +257,10 @@ impl<'a, 'g> Plan<'a, 'g> { { return Err(unsupported("calculation_filters")); } - let leaf_type = if metric.sql_is_complete { + let leaf_type = if metric.sql_is_complete && resolved.context.is_some() { MetricType::Simple + } else if metric.sql_is_complete { + MetricType::Derived } else { metric.r#type.clone() }; @@ -196,16 +347,12 @@ impl<'a, 'g> Plan<'a, 'g> { metric.name )) })?; - let mut replacements = HashMap::new(); - for column in semantic_column_references(sql)? { - if column.aggregate_input { - return Err(unsupported("inline_aggregate")); - } - let expanded = self.expand(&column.name(), resolved.context.as_deref())?; - replacements.insert((column.model, column.field), format!("({expanded})")); - } - let expression = - replace_semantic_columns(parse_semantic_expression(sql)?, &replacements)?; + let sql = crate::core::replace_model_placeholder(sql, resolved.context.as_deref())?; + let mut ast = serde_json::to_value(parse_semantic_expression(&sql)?) + .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; + self.expand_calculation(&mut ast, resolved.context.as_deref(), &HashSet::new())?; + let expression = serde_json::from_value(ast) + .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; self.generator.emit_expression(&expression)? } _ => return Err(unsupported("calculation_shape")), @@ -247,6 +394,36 @@ fn dimension_alias(index: usize) -> String { format!("__sidemantic_dimension_{index}") } +fn independent_source_path( + graph: &SemanticGraph, + from: &str, + to: &str, + dimensions: &[DimensionRef], +) -> Result> { + match graph.find_join_path(from, to) { + Ok(path) => Ok(Some(path)), + Err(SidemanticError::AmbiguousJoinPath { .. }) + if dimensions.iter().any(|dimension| { + [from, to].into_iter().all(|source| { + graph + .find_join_path(source, &dimension.model) + .is_ok_and(|path| { + path.steps + .iter() + .all(|step| step.relationship_type != RelationshipType::Cross) + }) + }) + }) => + { + // Both children have a unique keyed route to a requested grouping + // model. Their aggregate outputs meet there; no source-to-source + // row join needs to choose among the alternate graph routes. + Ok(None) + } + Err(error) => Err(error), + } +} + /// Normalize the ordinary compiler's public outputs at the child boundary. /// Each child has its own collision set because it selects different measures. fn child_projection( @@ -307,6 +484,7 @@ pub(super) fn try_generate( expressions: HashMap::new(), active: HashSet::new(), cross_source_calculation: false, + inline_aggregates: false, }; let mut outputs = Vec::new(); for reference in &query.metrics { @@ -340,8 +518,11 @@ pub(super) fn try_generate( } } let mut effective_query; - let query = if plan.models.len() == 1 && !query.skip_default_time_dimensions { + let query = if !query.skip_default_time_dimensions { effective_query = query.clone(); + // Defaults belong to selected model metric references, not every source + // discovered while expanding graph calculations. Resolve them once for + // the whole query before creating independent aggregate children. effective_query.dimensions = generator.apply_default_time_dimensions(&query.metrics, &query.dimensions)?; effective_query.skip_default_time_dimensions = true; @@ -350,13 +531,22 @@ pub(super) fn try_generate( query }; let dimensions = generator.parse_dimension_refs(&query.dimensions)?; + // Inline splitting exists to combine independent sources. The ordinary + // single-source compiler binds semantic dimension inputs and owns authored + // aggregate/window expressions without manufacturing physical columns. + if plan.inline_aggregates && plan.models.len() < 2 { + return Ok(None); + } // Cartesian products require every participating source even in a child // selecting only one source's measure: an empty sibling annihilates rows. // Keyed independent aggregates keep their existing separate populations. let mut population_models = query.required_population_models.clone(); for (index, from) in plan.models.iter().enumerate() { for to in plan.models.iter().skip(index + 1) { - let path = generator.graph.find_join_path(from, to)?; + let Some(path) = independent_source_path(generator.graph, from, to, &dimensions)? + else { + continue; + }; if path .steps .iter() @@ -417,19 +607,9 @@ pub(super) fn try_generate( if query.ungrouped || !query.table_calculations.is_empty() { return Err(unsupported("cross_grain_query_shape")); } - if !query.skip_default_time_dimensions - && plan.models.iter().any(|model| { - generator - .graph - .get_model(model) - .is_some_and(|model| model.default_time_dimension.is_some()) - }) - { - return Err(unsupported("cross_grain_default_time_dimension")); - } // Independent populations still require a declared, supported model graph. for model in plan.models.iter().skip(1) { - generator.graph.find_join_path(&plan.models[0], model)?; + independent_source_path(generator.graph, &plan.models[0], model, &dimensions)?; } for dimension in &dimensions { if generator @@ -467,10 +647,92 @@ pub(super) fn try_generate( }) .collect(); let mut row_filters = Vec::new(); + let mut window_filters: HashMap> = HashMap::new(); let mut aggregate_filters = Vec::new(); for filter in filters { let sql = generator.emit_expression(&filter)?; let columns = crate::core::outer_semantic_column_references(&sql)?; + let window_owners: HashSet<_> = columns + .iter() + .filter_map(|column| { + let owner = column.model.as_deref()?; + generator + .graph + .get_model(owner)? + .get_dimension(&column.field)? + .window + .as_ref() + .map(|_| owner.to_string()) + }) + .collect(); + if !window_owners.is_empty() { + // Window predicates filter source rows after window evaluation and + // before that source's aggregate. Metric references in the same + // predicate therefore refer to their row inputs, not outer totals. + let mut replacements = HashMap::new(); + for column in &columns { + if let Some(metric) = plan.resolve(&column.name(), None)? { + let owner = metric + .context + .as_deref() + .ok_or_else(|| unsupported("mixed_row_aggregate_filter"))?; + if !window_owners.contains(owner) + || metric.metric.r#type != MetricType::Simple + || metric.metric.sql_is_complete + { + return Err(unsupported("mixed_row_aggregate_filter")); + } + let source_model = generator.graph.get_model(owner).unwrap(); + let mut raw = generator.metric_raw_expression(&metric.metric, source_model)?; + if !metric.metric.filters.is_empty() { + let predicate = generator.normalize_metric_filters( + &metric.metric.filters, + owner, + &generator.model_alias(owner), + )?; + raw = format!("CASE WHEN {predicate} THEN {raw} END"); + } + raw = raw.replace("{model}", owner); + let mut inputs = HashMap::new(); + for input in semantic_column_references(&raw)? { + if input.model.as_deref().is_some_and(|qualifier| { + qualifier != owner + && qualifier != generator.model_alias(owner) + && qualifier != source_model.table_name() + }) { + return Err(unsupported("mixed_row_aggregate_filter")); + } + // Bind physical inputs directly to the CTE. Going back + // through semantic names could expand a same-named + // computed dimension instead of the metric's row input. + let qualified = format!( + "{}.{}", + generator.model_alias(owner), + generator.quote_identifier(&input.field) + ); + inputs.insert((input.model, input.field), qualified); + } + let raw = generator.emit_expression(&replace_semantic_columns( + parse_semantic_expression(&raw)?, + &inputs, + )?)?; + replacements.insert( + (column.model.clone(), column.field.clone()), + format!("({raw})"), + ); + } + } + let predicate = generator.emit_expression( + &crate::core::replace_outer_semantic_columns(filter, &replacements)?, + )?; + for owner in window_owners { + window_filters + .entry(owner) + .or_default() + .push(predicate.clone()); + } + continue; + } let mut replacements = HashMap::new(); let mut has_metric = false; let mut has_raw = false; @@ -512,6 +774,9 @@ pub(super) fn try_generate( child.required_population_models = population_models.clone(); child.metrics = leaves.iter().map(|leaf| leaf.reference.clone()).collect(); child.filters = row_filters.clone(); + child + .filters + .extend(window_filters.get(model).into_iter().flatten().cloned()); child.segments.clear(); child.order_by.clear(); child.limit = None; @@ -534,13 +799,22 @@ pub(super) fn try_generate( &dimensions, &metrics, fanout_models.contains(model), + plan.models.len() > 1, )? } else { let mut projection = child_projection(generator, &dimensions, &leaves)?; if query.with_totals && !dimensions.is_empty() { projection.push("__sidemantic_source._is_total AS _is_total".into()); } - let sql = generator.generate_from_model(&child, Some(model))?; + // Independent sources retain unmatched rows. Grouping order must + // not choose which population contributes to the calculation. + let anchor = if plan.models.len() > 1 { + Some(model.clone()) + } else { + generator + .query_base_model(&dimensions, &generator.parse_metric_refs(&child.metrics)?) + }; + let sql = generator.generate_from_model(&child, anchor.as_deref())?; format!( "SELECT {}\nFROM (\n{sql}\n) AS __sidemantic_source", projection.join(", ") @@ -791,6 +1065,282 @@ mod tests { assert_valid_sql(&sql); } + #[test] + fn cross_source_graph_alias_does_not_require_a_single_owner() { + let graph = graph(); + let generator = SqlGenerator::new(&graph); + let mut query = SemanticQuery::new().with_metrics(vec!["ratio".into()]); + query.aliases.insert("ratio".into(), "value".into()); + let sql = generator.generate(&query).unwrap(); + assert!(sql.contains("orders_preagg AS"), "{sql}"); + assert!(sql.contains("customers_preagg AS"), "{sql}"); + assert!(sql.contains("AS \"value\""), "{sql}"); + assert_valid_sql(&sql); + } + + #[test] + fn dotted_graph_calculation_is_not_reinterpreted_as_a_model_path() { + let mut graph = graph(); + graph + .add_metric_unvalidated(Metric::derived("business.ratio", "ratio * 2")) + .unwrap(); + let sql = compile(&graph, &["business.ratio"], &[]).unwrap(); + assert!(sql.contains("orders_preagg AS"), "{sql}"); + assert!(sql.contains("customers_preagg AS"), "{sql}"); + assert!(sql.contains("AS \"business.ratio\""), "{sql}"); + assert_valid_sql(&sql); + } + + #[test] + fn grouped_children_preserve_each_source_domain() { + let graph = graph(); + for dimension in ["customers.region", "orders.region"] { + let query = SemanticQuery::new() + .with_metrics(vec!["ratio".into()]) + .with_dimensions(vec![dimension.into()]); + let sql = SqlGenerator::new(&graph).generate(&query).unwrap(); + for owner in ["orders", "customers"] { + assert!( + sql.contains(&format!("FROM {owner}_cte AS {owner}_cte")), + "{sql}" + ); + } + assert_valid_sql(&sql); + } + } + + #[test] + fn inline_cross_source_aggregates_are_split_before_scalar_arithmetic() { + let mut graph = graph(); + for (name, expression) in [ + ( + "inline_ratio", + "COUNT(orders.id) * 1.0 / NULLIF(COUNT(customers.id), 0)", + ), + ( + "inline_sum", + "SUM(orders.amount) + SUM(CASE WHEN customers.id > 1 THEN customers.id ELSE 0 END)", + ), + ] { + let mut metric = Metric::derived(name, expression); + metric.sql_is_complete = true; + graph.add_metric_unvalidated(metric).unwrap(); + let sql = compile(&graph, &[name], &[]).unwrap(); + assert!(sql.contains("orders_preagg AS"), "{sql}"); + assert!(sql.contains("customers_preagg AS"), "{sql}"); + assert!(sql.contains("CROSS JOIN customers_preagg"), "{sql}"); + assert_valid_sql(&sql); + } + } + + #[test] + fn aggregate_splitting_preserves_lambda_bindings_and_expands_free_metrics() { + let mut graph = graph(); + graph + .add_metric_unvalidated(Metric::derived( + "median_plus_count", + concat!( + "LIST_AGGREGATE(LIST_TRANSFORM(LIST_DISTINCT(", + "LIST(STRUCT_PACK(k := orders.id, v := orders.amount))), ", + "x -> x.v), 'quantile_cont', 0.5) + customers.customer_count", + ), + )) + .unwrap(); + let sql = compile(&graph, &["median_plus_count"], &[]).unwrap(); + assert!(sql.contains("x.v"), "{sql}"); + assert!(sql.contains("customers_preagg"), "{sql}"); + assert_valid_sql(&sql); + } + + #[test] + fn inline_filtered_distinct_count_restores_absent_sources_to_zero() { + let mut graph = graph(); + let mut metric = Metric::derived( + "filtered_count_ratio", + "COUNT(DISTINCT orders.id) FILTER (WHERE orders.amount > 10) / NULLIF(COUNT(customers.id), 0)", + ); + metric.sql_is_complete = true; + graph.add_metric_unvalidated(metric).unwrap(); + let sql = compile(&graph, &["filtered_count_ratio"], &[]).unwrap(); + assert!( + sql.contains("COALESCE(orders_preagg.__sidemantic_metric_"), + "{sql}" + ); + assert!( + sql.contains("FILTER(WHERE") || sql.contains("FILTER (WHERE"), + "{sql}" + ); + assert_valid_sql(&sql); + } + + #[test] + fn inline_cross_source_metric_filters_are_not_discarded() { + let mut graph = graph(); + let mut metric = Metric::derived( + "filtered_inline", + "SUM(orders.amount) + COUNT(customers.id)", + ); + metric.sql_is_complete = true; + metric.filters.push("orders.amount > 10".into()); + graph.add_metric_unvalidated(metric).unwrap(); + let generator = SqlGenerator::new(&graph); + let mut plan = Plan { + generator: &generator, + leaves: Vec::new(), + models: Vec::new(), + expressions: HashMap::new(), + active: HashSet::new(), + cross_source_calculation: false, + inline_aggregates: false, + }; + assert!(matches!(plan.expand("filtered_inline", None), + Err(SidemanticError::UnsupportedSemanticFeatures { capabilities }) + if capabilities == vec!["aggregation.calculation_filters"])); + } + + #[test] + fn inline_single_source_and_window_aggregates_keep_existing_compiler() { + for sql in [ + "COUNT(orders.virtual_row)", + "SUM(orders.amount) / NULLIF(SUM(SUM(orders.amount)) OVER (), 0)", + ] { + let mut graph = graph(); + let mut orders = graph.get_model("orders").unwrap().clone(); + orders + .dimensions + .push(Dimension::categorical("virtual_row").with_sql("1")); + graph.replace_model(orders).unwrap(); + let mut metric = Metric::derived("inline", sql); + metric.sql_is_complete = true; + graph.add_metric_unvalidated(metric).unwrap(); + let query = SemanticQuery::new().with_metrics(vec!["inline".into()]); + assert!(try_generate(&SqlGenerator::new(&graph), &query) + .unwrap() + .is_none()); + } + } + + #[test] + fn imported_aggregate_leaves_use_qualified_inputs_and_requested_group_routes() { + let mut graph = SemanticGraph::new(); + for name in ["clicks", "impressions"] { + let mut model = Model::new(name, "id").with_table(name); + for target in ["campaigns", "publishers"] { + let mut relationship = Relationship::many_to_one(target); + relationship.foreign_key = Some(format!("{target}_id")); + model.relationships.push(relationship); + } + graph.add_model(model).unwrap(); + } + for name in ["campaigns", "publishers"] { + graph + .add_model( + Model::new(name, "id") + .with_table(name) + .with_dimension(Dimension::categorical("name")), + ) + .unwrap(); + } + let mut click_count = Metric::count("click_count"); + click_count.sql = Some("clicks.id".into()); + graph.add_metric_unvalidated(click_count).unwrap(); + let mut ratio = Metric::derived( + "ctr", + "COUNT(clicks.id) * 1.0 / NULLIF(COUNT(impressions.id), 0)", + ); + ratio.sql_is_complete = true; + graph.add_metric_unvalidated(ratio).unwrap(); + graph.set_metric_scopes(HashMap::new()).unwrap(); + let query = SemanticQuery::new() + .with_metrics(vec!["click_count".into(), "ctr".into()]) + .with_dimensions(vec!["campaigns.name".into()]); + let sql = SqlGenerator::new(&graph).generate(&query).unwrap(); + assert!(sql.contains("clicks_preagg AS"), "{sql}"); + assert!(sql.contains("impressions_preagg AS"), "{sql}"); + assert!(!sql.contains("publishers_cte"), "{sql}"); + assert_valid_sql(&sql); + let mut ungrouped = query; + ungrouped.dimensions.clear(); + assert!(matches!( + SqlGenerator::new(&graph).generate(&ungrouped), + Err(SidemanticError::AmbiguousJoinPath { .. }) + )); + } + + #[test] + fn legacy_window_metric_predicate_is_applied_before_aggregation() { + let mut legacy = SemanticGraph::new(); + for model in graph().models() { + legacy.add_model(model.clone()).unwrap(); + } + let mut orders = legacy.get_model("orders").unwrap().clone(); + let mut next_status = Dimension::categorical("next_status").with_sql("status"); + next_status.window = Some("LEAD(status) OVER (ORDER BY id)".into()); + orders.dimensions.push(next_status); + legacy.replace_model(orders).unwrap(); + let query = SemanticQuery::new() + .with_metrics(vec![ + "orders.revenue".into(), + "customers.customer_count".into(), + ]) + .with_dimensions(vec!["orders.region".into()]) + .with_filters(vec![ + "orders.next_status = 'complete' OR orders.revenue > 100".into(), + ]); + let sql = SqlGenerator::new(&legacy).generate(&query).unwrap(); + let (orders_sql, customers_sql) = sql.split_once("customers_preagg AS (").unwrap(); + assert!(orders_sql.contains("'complete'"), "{sql}"); + assert!(!orders_sql.contains("HAVING"), "{sql}"); + assert!(!customers_sql.contains("'complete'"), "{sql}"); + assert_valid_sql(&sql); + } + + #[test] + fn complete_count_without_inputs_retains_owner_join_and_dimension_domain() { + let mut graph = graph(); + let mut orders = graph.get_model("orders").unwrap().clone(); + let mut count = Metric::derived("opaque_count", "COUNT(*)"); + count.sql_is_complete = true; + orders.metrics.push(count); + graph.replace_model(orders).unwrap(); + let query = SemanticQuery::new() + .with_metrics(vec!["orders.opaque_count".into()]) + .with_dimensions(vec!["customers.region".into()]); + let sql = SqlGenerator::new(&graph).generate(&query).unwrap(); + assert!(sql.contains("COUNT(*) AS __sidemantic_metric_0"), "{sql}"); + assert!(sql.contains("FROM customers_cte AS customers_cte"), "{sql}"); + assert!(sql.contains("LEFT JOIN orders_cte AS orders_cte"), "{sql}"); + assert_valid_sql(&sql); + } + + #[test] + fn mixed_window_predicate_filters_only_its_source_before_aggregation() { + let mut graph = graph(); + let mut orders = graph.get_model("orders").unwrap().clone(); + let mut next_status = crate::core::Dimension::categorical("next_status").with_sql("status"); + next_status.window = Some("LEAD(status) OVER (ORDER BY id)".into()); + orders.dimensions.push(next_status); + graph.replace_model(orders).unwrap(); + let query = SemanticQuery::new() + .with_metrics(vec![ + "orders.revenue".into(), + "customers.customer_count".into(), + ]) + .with_dimensions(vec!["orders.region".into()]) + .with_filters(vec![ + "orders.next_status = 'complete' OR orders.revenue > 100".into(), + ]); + let sql = SqlGenerator::new(&graph).generate(&query).unwrap(); + let (orders_sql, rest) = sql.split_once("customers_preagg AS (").unwrap(); + assert!(orders_sql.contains("'complete'"), "{sql}"); + assert!( + orders_sql.contains("amount) > 100") || orders_sql.contains("amount > 100"), + "{sql}" + ); + assert!(!rest.contains("'complete'"), "{sql}"); + assert_valid_sql(&sql); + } + #[test] fn proxies_and_local_names_expand_to_aggregate_owners() { let graph = graph(); @@ -984,4 +1534,52 @@ mod tests { assert!(!sql.contains("orders_preagg AS")); assert_valid_sql(&sql); } + + #[test] + fn default_time_dimensions_follow_selected_metrics_across_sources() { + let mut graph = graph(); + for (owner, field, grain) in [ + ("orders", "ordered_at", "month"), + ("customers", "signed_up_at", "week"), + ] { + let mut model = graph.get_model(owner).unwrap().clone(); + model.dimensions.push(Dimension::time(field)); + model.default_time_dimension = Some(field.into()); + model.default_grain = Some(grain.into()); + graph.replace_model(model).unwrap(); + } + for (metrics, dimensions, skip, expected_month, expected_week) in [ + (vec!["ratio"], vec![], false, false, false), + ( + vec!["orders.revenue", "customers.customer_count"], + vec![], + false, + true, + true, + ), + ( + vec!["orders.revenue", "customers.customer_count"], + vec!["orders.ordered_at__day"], + false, + false, + true, + ), + ( + vec!["orders.revenue", "customers.customer_count"], + vec![], + true, + false, + false, + ), + ] { + let query = SemanticQuery::new() + .with_metrics(metrics.into_iter().map(str::to_string).collect()) + .with_dimensions(dimensions.into_iter().map(str::to_string).collect()) + .with_skip_default_time_dimensions(skip); + let sql = SqlGenerator::new(&graph).generate(&query).unwrap(); + assert_eq!(sql.contains("ordered_at__month"), expected_month, "{sql}"); + assert_eq!(sql.contains("signed_up_at__week"), expected_week, "{sql}"); + assert_valid_sql(&sql); + } + } } diff --git a/sidemantic-rs/src/sql/generator/cohort.rs b/sidemantic-rs/src/sql/generator/cohort.rs index 366482c82..f77d25c73 100644 --- a/sidemantic-rs/src/sql/generator/cohort.rs +++ b/sidemantic-rs/src/sql/generator/cohort.rs @@ -326,13 +326,13 @@ impl SqlGenerator<'_> { )); } let mut order = Vec::new(); + let names: Vec<_> = output_dimensions + .iter() + .map(|dimension| dimension.alias.as_str()) + .chain(std::iter::once(metric.name.as_str())) + .collect(); for item in &query.order_by { - let (field, direction) = item - .rsplit_once(' ') - .filter(|(_, direction)| { - direction.eq_ignore_ascii_case("asc") || direction.eq_ignore_ascii_case("desc") - }) - .unwrap_or((item, "")); + let (field, direction) = crate::sql::split_order_field(item, &names); let name = field .strip_prefix(&format!("{}.", model.name)) .unwrap_or(field); @@ -476,4 +476,35 @@ mod tests { } } } + + #[test] + fn owned_graph_cohort_approximate_aggregate_uses_qualified_inner_results() { + let input = json!({ + "version": 1, "input_dialect": "duckdb", + "models": [{ + "name": "events", "table": "events", "primary_key": "id", + "dimensions": [{"name": "person", "type": "categorical"}] + }], + "metrics": [{ + "name": "qualified", "type": "cohort", "entity": "person", + "agg": "approx_count_distinct", "sql": "amount", + "inner_metrics": [{"name": "amount", "agg": "sum", "sql": "raw_amount"}], + "having": "amount > 10" + }], + "metric_owners": {"qualified": "events"} + }); + let sql = compile_with_semantic_input( + &input.to_string(), + &json!({"metrics": ["qualified"]}).to_string(), + ) + .unwrap(); + // The graph metric's SQL names an inner result, not a source column. + assert!( + sql.contains("APPROX_COUNT_DISTINCT(cohort_sub.\"amount\")"), + "{sql}" + ); + assert!(sql.contains("SUM((\"raw_amount\"))"), "{sql}"); + assert!(sql.contains("HAVING"), "{sql}"); + polyglot_sql::parse_one(&sql, DialectType::DuckDB).unwrap(); + } } diff --git a/sidemantic-rs/src/sql/generator/conversion.rs b/sidemantic-rs/src/sql/generator/conversion.rs index 85c96deb1..8f7db5e85 100644 --- a/sidemantic-rs/src/sql/generator/conversion.rs +++ b/sidemantic-rs/src/sql/generator/conversion.rs @@ -117,15 +117,17 @@ impl SqlGenerator<'_> { if !output_names.insert(dimension.alias.to_ascii_lowercase()) { return Err(unsupported("output_alias")); } - let mut expression = self.conversion_source_expression(model, &dimension.name)?; - if let Some(grain) = &dimension.granularity { - expression = self.date_trunc_sql(grain, &expression)?; - } + let expression = self.conversion_source_expression(model, &dimension.name)?; let internal = format!("__funnel_group_{index}"); projection.push(format!("{expression} AS {internal}")); - secured - .dimensions - .push(crate::core::Dimension::categorical(&internal)); + let mut group_dimension = crate::core::Dimension::categorical(&internal); + if let Some(grain) = &dimension.granularity { + // Bucket in the first-step grouping, matching the sequential + // planner's source grain. Pre-bucketing the repeated source can + // give DuckDB incorrect NULL ordering statistics through joins. + group_dimension.sql = Some(self.date_trunc_sql(grain, &internal)?); + } + secured.dimensions.push(group_dimension); inner_dimensions.push(DimensionRef { model: model.name.clone(), name: internal.clone(), @@ -206,13 +208,9 @@ impl SqlGenerator<'_> { output.join(", ") ); let mut order = Vec::new(); + let names: Vec<_> = output_names.iter().map(String::as_str).collect(); for item in &query.order_by { - let (field, direction) = item - .rsplit_once(' ') - .filter(|(_, direction)| { - direction.eq_ignore_ascii_case("asc") || direction.eq_ignore_ascii_case("desc") - }) - .unwrap_or((item, "")); + let (field, direction) = crate::sql::split_order_field(item, &names); let name = field .strip_prefix(&format!("{}.", model.name)) .unwrap_or(field); @@ -328,13 +326,13 @@ impl SqlGenerator<'_> { } } let mut ordering = Vec::new(); + let names: Vec<_> = dimensions + .iter() + .map(|dimension| dimension.alias.as_str()) + .chain(std::iter::once(metric.name.as_str())) + .collect(); for item in &query.order_by { - let (field, direction) = item - .rsplit_once(' ') - .filter(|(_, direction)| { - direction.eq_ignore_ascii_case("asc") || direction.eq_ignore_ascii_case("desc") - }) - .unwrap_or((item, "")); + let (field, direction) = crate::sql::split_order_field(item, &names); let alias = if field == metric.name || field == format!("{}.{}", model.name, metric.name) { &metric.name @@ -451,6 +449,70 @@ impl SqlGenerator<'_> { mod tests { use super::*; + #[test] + fn both_conversion_algorithms_accept_null_ordering_but_reject_nonoutputs() { + crate::semantic_input::with_semantic_stack(|| { + for multistep in [false, true] { + let mut metric = Metric::new("funnel"); + metric.r#type = MetricType::Conversion; + metric.entity = Some("user_id".into()); + if multistep { + metric.steps = Some(vec![ + "event_type = 'signup'".into(), + "event_type = 'buy'".into(), + ]); + } else { + metric.base_event = Some("signup".into()); + metric.conversion_event = Some("buy".into()); + } + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("events", "id") + .with_table("events") + .with_dimension(crate::core::Dimension::time("timestamp")) + .with_dimension(crate::core::Dimension::categorical("event_type")) + .with_metric(metric), + ) + .unwrap(); + let generator = SqlGenerator::new(&graph); + let reference = MetricRef { + model: "events".into(), + name: "funnel".into(), + alias: "funnel".into(), + graph_metric: false, + }; + for suffix in ["ASC NULLS FIRST", "DESC NULLS LAST", "NULLS LAST"] { + let query = + SemanticQuery::new().with_order_by(vec![format!("events.funnel {suffix}")]); + let sql = generator + .generate_scoped_conversion(&query, &reference, &[]) + .unwrap(); + assert!( + sql.contains(&format!("ORDER BY \"funnel\" {suffix}")), + "{sql}" + ); + polyglot_sql::parse_one(&sql, DialectType::DuckDB).unwrap(); + } + for field in [ + "funnel; SELECT 2", + "funnel DESC LIMIT 1", + "random()", + "missing", + ] { + let query = SemanticQuery::new().with_order_by(vec![field.into()]); + assert!(matches!( + generator.generate_scoped_conversion(&query, &reference, &[]), + Err(SidemanticError::UnsupportedSemanticFeatures { capabilities }) + if capabilities == vec!["metric.conversion_order_by"] + )); + } + } + Ok(()) + }) + .unwrap(); + } + #[test] fn multistep_requires_a_time_dimension() { let mut metric = Metric::new("funnel"); diff --git a/sidemantic-rs/src/sql/generator/fanout_complete.rs b/sidemantic-rs/src/sql/generator/fanout_complete.rs index a91c9bf95..24fcb4b5b 100644 --- a/sidemantic-rs/src/sql/generator/fanout_complete.rs +++ b/sidemantic-rs/src/sql/generator/fanout_complete.rs @@ -14,6 +14,7 @@ fn unsupported(shape: &str) -> SidemanticError { struct Inputs<'a> { generator: &'a SqlGenerator<'a>, + owner: &'a str, model: Model, names: HashSet, references: Vec, @@ -32,8 +33,7 @@ impl Inputs<'_> { let mut metric = Metric::sum(&name, sql); metric.filters = filters.to_vec(); self.model.metrics.push(metric); - self.references - .push(format!("{}.{}", self.model.name, name)); + self.references.push(format!("{}.{}", self.owner, name)); self.generator.quote_identifier(&name) } } @@ -45,6 +45,7 @@ pub(super) fn generate_entity_aggregates( dimensions: &[DimensionRef], metrics: &[(&Metric, &str)], deduplicate: bool, + independent_source: bool, ) -> Result { let model = generator.graph.get_model(owner).unwrap(); if deduplicate && model.primary_keys().is_empty() { @@ -71,6 +72,7 @@ pub(super) fn generate_entity_aggregates( .collect(); let mut inputs = Inputs { generator, + owner, model: model.clone(), names, references: Vec::new(), @@ -147,9 +149,9 @@ pub(super) fn generate_entity_aggregates( }; selections.push(format!("{sql} AS {}", generator.quote_identifier(alias))); } - // COUNT(*) can be the only output, but the row compiler still needs one - // projected input when no keys or dimensions were required. - if inputs.references.is_empty() && dimensions.is_empty() { + // COUNT(*) can be the only output. Retain its source in the row query even + // when every grouping dimension belongs to a different model. + if inputs.references.is_empty() { inputs.add("1".into(), &[]); } let mut graph = generator.graph.clone(); @@ -167,7 +169,15 @@ pub(super) fn generate_entity_aggregates( rows.ungrouped = true; rows.with_totals = false; rows.use_preaggregations = false; - let row_sql = row_generator.generate_from_model(&rows, Some(owner))?; + // Independent children retain their own population before their grouped + // outputs are joined. A single complete expression retains the ordinary + // dimension-domain contract, including COUNT(*) on null-extended rows. + let source = if independent_source { + Some(owner.to_string()) + } else { + row_generator.query_base_model(dimensions, &row_generator.parse_metric_refs(&rows.metrics)?) + }; + let row_sql = row_generator.generate_from_model(&rows, source.as_deref())?; let mut collisions = HashMap::new(); for dimension in dimensions { *collisions.entry(dimension.alias.clone()).or_insert(0usize) += 1; diff --git a/sidemantic-rs/src/sql/generator/imported_totals.rs b/sidemantic-rs/src/sql/generator/imported_totals.rs new file mode 100644 index 000000000..fa2d23658 --- /dev/null +++ b/sidemantic-rs/src/sql/generator/imported_totals.rs @@ -0,0 +1,253 @@ +//! Imported calculations retain their source-measure and population semantics. +use super::*; + +impl SqlGenerator<'_> { + /// LookML post-SQL calculations encode base measures as aggregate inputs. + /// Resolve only this explicit import contract; ordinary complete aggregates + /// continue to address physical columns even when a metric shadows a name. + pub(super) fn imported_calculation_expression( + &self, + metric: &Metric, + model_name: &str, + ) -> Result { + let calculation = metric + .meta + .as_ref() + .and_then(|meta| meta.get("table_calculation")) + .and_then(serde_json::Value::as_str); + if !matches!( + calculation, + Some("percent_of_total" | "percent_of_previous") + ) { + return Ok(metric.sql_expr().to_string()); + } + let sql = crate::core::replace_model_placeholder(metric.sql_expr(), Some(model_name))?; + let mut replacements = HashMap::new(); + for column in semantic_column_references(&sql)? { + if !column.aggregate_input { + continue; + } + let owner = column.model.as_deref().unwrap_or(model_name); + let Some(model) = self.graph.get_model(owner) else { + continue; + }; + let Some(base) = model.get_metric(&column.field) else { + continue; + }; + if base.r#type != MetricType::Simple { + continue; + } + let mut input = self.metric_raw_expression(base, model)?; + if !base.filters.is_empty() { + let filters = base + .filters + .iter() + .map(|filter| format!("({filter})")) + .collect::>(); + input = format!("CASE WHEN {} THEN {input} END", filters.join(" AND ")); + } + let input = crate::core::replace_model_placeholder(&input, Some(owner))?; + let qualified = semantic_column_references(&input)? + .into_iter() + .map(|reference| { + let name = format!( + "{}.{}", + self.quote_identifier(reference.model.as_deref().unwrap_or(owner)), + self.quote_identifier(&reference.field) + ); + ((reference.model, reference.field), name) + }) + .collect(); + let input = crate::core::replace_semantic_columns( + parse_semantic_expression(&input)?, + &qualified, + )?; + replacements.insert( + (column.model, column.field), + format!("({})", self.emit_expression(&input)?), + ); + } + self.emit_expression(&crate::core::replace_semantic_columns( + parse_semantic_expression(&sql)?, + &replacements, + )?) + } + + /// Reaggregate the expanded measure over the containing query's exact source + /// before GROUP BY/HAVING/pagination. CTE-level policies and metric filters + /// are retained, as are join predicates and residual WHERE filters. + pub(super) fn expand_imported_totals(&self, projection: &str, source: &str) -> Result { + fn rewrite( + generator: &SqlGenerator<'_>, + value: &mut serde_json::Value, + source: &str, + ) -> Result<()> { + if let Some(function) = value.get("function") { + if function + .get("name") + .and_then(serde_json::Value::as_str) + .is_some_and(|name| name.eq_ignore_ascii_case("__bsl_all")) + { + let expression: Expression = serde_json::from_value(value.clone()) + .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; + let Expression::Function(function) = expression else { + unreachable!() + }; + if function.args.len() != 1 { + return Err(SidemanticError::Validation( + "BSL all() requires one measure".into(), + )); + } + let aggregate = generator.emit_expression(&function.args[0])?; + *value = serde_json::to_value(Expression::Raw(Raw { + sql: format!("(SELECT {aggregate} {source})"), + })) + .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; + return Ok(()); + } + } + match value { + serde_json::Value::Object(fields) => { + for child in fields.values_mut() { + rewrite(generator, child, source)?; + } + } + serde_json::Value::Array(children) => { + for child in children { + rewrite(generator, child, source)?; + } + } + _ => {} + } + Ok(()) + } + let statement = + crate::semantic_input::dialects::parse(&format!("SELECT {projection}"), self.dialect)?; + let Expression::Select(mut select) = statement else { + unreachable!() + }; + let expression = select.expressions.remove(0); + let mut value = serde_json::to_value(expression) + .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; + rewrite(self, &mut value, source)?; + let expression = serde_json::from_value(value) + .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; + self.emit_expression(&expression) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::{Dimension, Relationship}; + + #[test] + fn bsl_distinct_total_reaggregates_joined_filtered_population() { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("orders", "id") + .with_table("orders") + .with_dimension(Dimension::categorical("customer_id")) + .with_metric(Metric::count_distinct("users", "user_id")) + .with_metric(Metric::derived("share", "users / __bsl_all(users)")) + .with_relationship( + Relationship::many_to_one("customers").with_keys("customer_id", "id"), + ), + ) + .unwrap(); + graph + .add_model( + Model::new("customers", "id") + .with_table("customers") + .with_dimension(Dimension::categorical("region")), + ) + .unwrap(); + graph.set_metric_scopes(HashMap::new()).unwrap(); + let query = SemanticQuery::new() + .with_metrics(vec!["orders.share".into()]) + .with_dimensions(vec!["customers.region".into()]) + .with_filters(vec!["customers.region = 'EU'".into()]); + let sql = SqlGenerator::new(&graph).generate(&query).unwrap(); + assert!(!sql.to_ascii_lowercase().contains("__bsl_all"), "{sql}"); + assert!( + sql.contains("SELECT (COUNT(DISTINCT orders_cte.users_raw))"), + "{sql}" + ); + // Both aggregates read the same CTEs and join edge. The region filter + // may be pushed into the shared customers CTE, so it need not repeat. + assert_eq!(sql.matches("JOIN orders_cte").count(), 2, "{sql}"); + assert!(sql.contains("'EU'"), "{sql}"); + polyglot_sql::parse_one(&sql, DialectType::DuckDB).unwrap(); + } + + #[test] + fn lookml_post_sql_count_distinct_binds_base_measure_input() { + let mut graph = SemanticGraph::new(); + let mut percentage = Metric::derived( + "percentage", + "COUNT(DISTINCT {model}.users) / NULLIF(SUM(COUNT(DISTINCT {model}.users)) OVER (), 0)", + ); + percentage.meta = Some(serde_json::json!({"table_calculation":"percent_of_total"})); + graph + .add_model( + Model::new("visits", "id") + .with_table("visits") + .with_dimension(Dimension::categorical("country")) + .with_metric(Metric::count_distinct("users", "user_id")) + .with_metric(percentage), + ) + .unwrap(); + graph.set_metric_scopes(HashMap::new()).unwrap(); + let sql = SqlGenerator::new(&graph) + .generate( + &SemanticQuery::new() + .with_metrics(vec!["visits.percentage".into()]) + .with_dimensions(vec!["visits.country".into()]), + ) + .unwrap(); + assert!( + sql.contains("COUNT(DISTINCT (visits_cte.user_id))"), + "{sql}" + ); + assert!(!sql.contains("visits_cte.users"), "{sql}"); + assert!(!sql.contains("users AS users"), "{sql}"); + polyglot_sql::parse_one(&sql, DialectType::DuckDB).unwrap(); + } + + #[test] + fn lookml_post_sql_base_filters_preserve_conjunction_grouping() { + let mut graph = SemanticGraph::new(); + let mut users = Metric::count_distinct("users", "user_id"); + users.filters = vec![ + "country = 'US' OR country = 'CA'".into(), + "active = 1".into(), + ]; + graph + .add_model( + Model::new("visits", "id") + .with_table("visits") + .with_metric(users), + ) + .unwrap(); + let mut percentage = Metric::derived("percentage", "COUNT(DISTINCT {model}.users)"); + percentage.meta = Some(serde_json::json!({"table_calculation":"percent_of_total"})); + let expression = SqlGenerator::new(&graph) + .imported_calculation_expression(&percentage, "visits") + .unwrap(); + assert!( + expression.contains( + "(visits.country = 'US' OR visits.country = 'CA') AND (visits.active = 1)" + ), + "{expression}" + ); + // The same unmarked expression remains a physical aggregate input. + percentage.meta = None; + assert_eq!( + SqlGenerator::new(&graph) + .imported_calculation_expression(&percentage, "visits") + .unwrap(), + "COUNT(DISTINCT {model}.users)" + ); + } +} diff --git a/sidemantic-rs/src/sql/generator/options.rs b/sidemantic-rs/src/sql/generator/options.rs index 874fa4f20..4ce4f33fc 100644 --- a/sidemantic-rs/src/sql/generator/options.rs +++ b/sidemantic-rs/src/sql/generator/options.rs @@ -152,8 +152,18 @@ impl SqlGenerator<'_> { let mut projections = Vec::new(); let mut replacements = HashMap::new(); let mut renamed = false; - let quote = - |name: &str| self.emit_expression(&Expression::Identifier(Identifier::quoted(name))); + // polyglot 0.1.15 incorrectly splits ASC/DESC suffixes even inside + // quoted identifiers. Keep output names literal at this boundary. + let quote = |name: &str| { + let dialect = polyglot_sql::Dialect::get(self.dialect); + let style = &dialect.generator_config().identifier_quote_style; + format!( + "{}{}{}", + style.start, + name.replace(style.end, &format!("{}{}", style.end, style.end)), + style.end + ) + }; for expression in &select.expressions { let name = match expression { Expression::Alias(alias) => &alias.alias.name, @@ -168,10 +178,10 @@ impl SqlGenerator<'_> { renamed |= alias != name; projections.push(format!( "__sidemantic_result.{} AS {}", - quote(name)?, - quote(alias)? + quote(name), + quote(alias) )); - replacements.insert((None, name.clone()), quote(alias)?); + replacements.insert((None, name.clone()), quote(alias)); let source = match expression { Expression::Alias(alias) => &alias.this, other => other, @@ -182,32 +192,40 @@ impl SqlGenerator<'_> { column.table.as_ref().map(|table| table.name.clone()), column.name.name.clone(), ), - quote(alias)?, + quote(alias), ); } } if !renamed { return Ok(sql); } + let (inner_sql, used_preaggregation) = sql + .strip_suffix("\n-- used_preagg=true") + .map_or((sql.as_str(), false), |sql| (sql, true)); let wrapper = format!( - "SELECT {} FROM ({sql}) AS __sidemantic_result", + "SELECT {} FROM ({inner_sql}\n) AS __sidemantic_result", projections.join(", ") ); #[cfg(target_arch = "wasm32")] crate::wasm_sql_guard::check(&wrapper, self.dialect)?; - let Expression::Select(mut outer) = - crate::semantic_input::dialects::parse(&wrapper, self.dialect) - .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))? - else { - unreachable!() - }; - outer.order_by = select.order_by; - if let Some(order) = &mut outer.order_by { + let mut order_by = select.order_by; + if let Some(order) = &mut order_by { for item in &mut order.expressions { item.this = replace_semantic_columns(item.this.clone(), &replacements)?; } } - self.emit_expression(&Expression::Select(outer)) + let mut result = if let Some(order) = order_by { + let order = self.emit_expression(&Expression::OrderBy(Box::new(order)))?; + format!("{wrapper} {order}") + } else { + wrapper + }; + // Routing metadata belongs to the final statement, not a nested SQL + // comment that the next parser can discard or attach to another node. + if used_preaggregation { + result.push_str("\n-- used_preagg=true"); + } + Ok(result) } } @@ -231,6 +249,21 @@ mod tests { graph } + #[test] + fn alias_wrapper_preserves_trailing_rollup_marker() { + let graph = graph(); + let generator = SqlGenerator::new(&graph); + let sql = generator + .alias_result( + "SELECT 42 AS revenue\n-- used_preagg=true".into(), + &HashMap::from([("revenue".into(), "total".into())]), + ) + .unwrap(); + assert!(sql.ends_with("\n-- used_preagg=true"), "{sql}"); + crate::semantic_input::dialects::parse(&sql, DialectType::DuckDB).unwrap(); + assert!(sql.contains("AS \"total\""), "{sql}"); + } + #[test] fn aliases_and_totals_match_result_schema() { let graph = graph(); @@ -259,6 +292,26 @@ mod tests { ); } + #[test] + fn semantic_boundary_binds_spaced_order_aliases_before_policy_parsing() { + let source = json!({"version": 1, "input_dialect": "duckdb", "models": [{"name": "orders", "table": "orders", "primary_key": "id", "dimensions": [{"name": "category", "type": "categorical"}], "metrics": [{"name": "revenue", "agg": "sum", "sql": "amount"}]}]}).to_string(); + for alias in ["Category label", "Category DESC", "Category NULLS FIRST"] { + for suffix in ["", " DESC", " ASC NULLS FIRST", "\tDESC\tNULLS\tLAST"] { + for dialect in ["duckdb", "postgres"] { + let query = json!({ + "metrics": ["orders.revenue"], "dimensions": ["orders.category"], + "aliases": {"orders.category": alias}, + "order_by": [format!("{alias}{suffix}")], "limit": 2, + "query_dialect": dialect, "dialect": dialect + }); + let sql = compile_with_semantic_input(&source, &query.to_string()).unwrap(); + assert!(sql.contains(&format!("ORDER BY \"{alias}\"")), "{sql}"); + assert!(sql.contains("LIMIT 2"), "{sql}"); + } + } + } + } + #[test] fn malformed_options_and_totals_controls_are_invalid_on_both_boundaries() { let source = json!({"version": 1, "input_dialect": "duckdb", "models": [{"name": "orders", "table": "orders", "primary_key": "id", "dimensions": [{"name": "category", "type": "categorical"}], "metrics": [{"name": "revenue", "agg": "sum", "sql": "amount"}]}]}).to_string(); diff --git a/sidemantic-rs/src/sql/generator/retention.rs b/sidemantic-rs/src/sql/generator/retention.rs index e14e0e728..23c7836a6 100644 --- a/sidemantic-rs/src/sql/generator/retention.rs +++ b/sidemantic-rs/src/sql/generator/retention.rs @@ -269,12 +269,7 @@ JOIN cohort_sizes c USING (cohort_date)"# ]; let mut ordering = Vec::new(); for item in &query.order_by { - let (field, direction) = item - .rsplit_once(' ') - .filter(|(_, suffix)| { - suffix.eq_ignore_ascii_case("asc") || suffix.eq_ignore_ascii_case("desc") - }) - .unwrap_or((item, "")); + let (field, direction) = crate::sql::split_order_field(item, &outputs); if !outputs.contains(&field) { return Err(unsupported("order_by")); } @@ -362,7 +357,6 @@ mod tests { for (field, value) in [ ("dimensions", json!(["events.person"])), ("filters", json!(["COUNT(*) > 1"])), - ("filters", json!(["ROW_NUMBER() OVER () > 1"])), ("filters", json!(["other.id = 1"])), ] { let mut query = query(); @@ -374,6 +368,22 @@ mod tests { } } + #[test] + fn retention_window_filter_is_rejected_at_query_boundary() { + let mut query = query(); + query["filters"] = json!(["ROW_NUMBER() OVER () > 1"]); + let error = + compile_with_semantic_input(&input().to_string(), &query.to_string()).unwrap_err(); + assert!( + matches!( + error, + SidemanticError::ValidationIssue { ref code, ref field, .. } + if code == "invalid_semantic_input" && field == "query.filters" + ), + "{error}" + ); + } + #[test] fn retention_requires_valid_period_and_granularity() { for (field, value) in [ diff --git a/sidemantic-rs/src/sql/generator/snapshots.rs b/sidemantic-rs/src/sql/generator/snapshots.rs index 87f7fef29..fea597b55 100644 --- a/sidemantic-rs/src/sql/generator/snapshots.rs +++ b/sidemantic-rs/src/sql/generator/snapshots.rs @@ -82,6 +82,9 @@ pub(super) fn try_generate( generator: &SqlGenerator<'_>, query: &SemanticQuery, ) -> Result> { + if query.allow_non_additive_unsafe { + return Ok(None); + } if !generator .graph .metrics() @@ -418,11 +421,17 @@ pub(super) fn try_generate( )); } let mut ordering = Vec::new(); + let names: Vec<_> = metrics + .iter() + .map(|metric| metric.alias.as_str()) + .chain( + output_dimensions + .iter() + .map(|dimension| dimension.alias.as_str()), + ) + .collect(); for item in &query.order_by { - let (field, direction) = item - .rsplit_once(' ') - .filter(|(_, dir)| dir.eq_ignore_ascii_case("asc") || dir.eq_ignore_ascii_case("desc")) - .unwrap_or((item, "")); + let (field, direction) = crate::sql::split_order_field(item, &names); let alias = query .metrics .iter() @@ -472,6 +481,27 @@ mod tests { graph } + #[test] + fn explicit_unsafe_option_bypasses_snapshot_selection_without_mutating_graph() { + crate::semantic_input::with_semantic_stack(|| { + let graph = graph(); + let mut query = SemanticQuery::new().with_metrics(vec![ + "snapshots.balance".into(), + "snapshots.activity".into(), + ]); + query.allow_non_additive_unsafe = true; + let generator = SqlGenerator::new(&graph); + assert!(try_generate(&generator, &query)?.is_none()); + let sql = generator.generate(&query)?; + assert!(sql.contains("SUM("), "{sql}"); + assert!(!sql.contains("MAX(day)"), "{sql}"); + query.allow_non_additive_unsafe = false; + assert!(try_generate(&generator, &query)?.is_some()); + Ok(()) + }) + .unwrap(); + } + #[test] fn simple_model_snapshot_is_not_bypassed_by_graph_metric_fast_path() { crate::semantic_input::with_semantic_stack(|| { diff --git a/sidemantic-rs/src/sql/generator/temporal.rs b/sidemantic-rs/src/sql/generator/temporal.rs index 2fe39265f..0e7fe13eb 100644 --- a/sidemantic-rs/src/sql/generator/temporal.rs +++ b/sidemantic-rs/src/sql/generator/temporal.rs @@ -29,7 +29,11 @@ pub(crate) fn validate_metric(metric: &Metric) -> Result<()> { } parse_output_frame(frame)?; } - if let Some(window) = &metric.window { + if let Some(window) = metric + .window + .as_deref() + .filter(|window| *window != "unbounded") + { period_interval(window)?; } } @@ -164,6 +168,10 @@ fn period_interval(value: &str) -> Result<(u32, String)> { } impl SqlGenerator<'_> { + pub(super) fn window_dimension_alias(dimension: &crate::core::Dimension) -> String { + format!("__sidemantic_window_{}", dimension.name) + } + pub(super) fn offset_window_lag_rows( offset: Option<&str>, granularity: Option<&str>, @@ -293,7 +301,11 @@ impl SqlGenerator<'_> { let frame = if let Some(frame) = &metric.window_frame { frame.clone() } else if metric.grain_to_date.is_none() { - if let Some(window) = &metric.window { + if let Some(window) = metric + .window + .as_deref() + .filter(|window| *window != "unbounded") + { let (amount, unit) = period_interval(window)?; format!("RANGE BETWEEN INTERVAL '{amount} {unit}' PRECEDING AND CURRENT ROW") } else { @@ -359,6 +371,57 @@ impl SqlGenerator<'_> { mod tests { use super::*; + #[test] + fn window_dimension_uses_separate_source_alias_in_grouping_and_filter() { + let input = serde_json::json!({ + "version": 1, "input_dialect": "duckdb", + "models": [{"name": "events", "table": "events", "primary_key": "id", + "dimensions": [{"name": "day", "type": "time", "granularity": "day", "window": "MIN(day) OVER ()"}], + "metrics": [{"name": "revenue", "agg": "sum", "sql": "amount"}]}] + }); + let sql = crate::semantic_input::compile_with_semantic_input( + &input.to_string(), + &serde_json::json!({ + "metrics": ["events.revenue"], "dimensions": ["events.day"], + "filters": ["COALESCE(events.day, '2024-01-01') > '2024-01-02'"] + }) + .to_string(), + ) + .unwrap(); + assert!(sql.contains("AS __sidemantic_window_day"), "{sql}"); + assert!( + sql.contains("DATE_TRUNC('day', events_cte.__sidemantic_window_day)"), + "{sql}" + ); + assert!( + sql.contains("COALESCE(events_cte.__sidemantic_window_day"), + "{sql}" + ); + polyglot_sql::parse_one(&sql, DialectType::DuckDB).unwrap(); + } + + #[test] + fn graph_calculation_keeps_graph_reference_in_cumulative_base_query() { + let input = serde_json::json!({ + "version": 1, "input_dialect": "duckdb", + "models": [{"name": "events", "table": "events", "primary_key": "id", + "dimensions": [{"name": "day", "type": "time", "granularity": "day"}], + "metrics": [{"name": "revenue", "agg": "sum", "sql": "amount"}]}], + "metrics": [ + {"name": "total", "type": "derived", "sql": "events.revenue"}, + {"name": "running", "type": "cumulative", "sql": "events.revenue"} + ] + }); + let sql = crate::semantic_input::compile_with_semantic_input( + &input.to_string(), + r#"{"metrics":["total","running"],"dimensions":["events.day"]}"#, + ) + .unwrap(); + assert!(sql.contains("base.total"), "{sql}"); + assert!(sql.contains("SUM(base.revenue) OVER"), "{sql}"); + polyglot_sql::parse_one(&sql, DialectType::DuckDB).unwrap(); + } + #[test] fn explicit_window_requires_a_root_window_capable_function() { for sql in [ @@ -510,6 +573,27 @@ mod tests { (graph, dimensions) } + #[test] + fn explicit_unbounded_window_preserves_running_total_frame() { + let (graph, dimensions) = grouped_graph(); + let generator = SqlGenerator::new(&graph); + let mut metric = Metric::sum("running", "revenue"); + metric.r#type = MetricType::Cumulative; + metric.window = Some("unbounded".into()); + validate_metric(&metric).unwrap(); + let explicit = generator + .cumulative_window_sql(&metric, &dimensions, "day__month") + .unwrap(); + metric.window = None; + let implicit = generator + .cumulative_window_sql(&metric, &dimensions, "day__month") + .unwrap(); + assert_eq!(explicit, implicit); + assert!(explicit.contains("ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW")); + metric.window = Some("invalid".into()); + assert!(validate_metric(&metric).is_err()); + } + #[test] fn calendar_comparison_uses_exact_period_and_group_partition() { let (graph, dimensions) = grouped_graph(); diff --git a/sidemantic-rs/src/sql/rewriter.rs b/sidemantic-rs/src/sql/rewriter.rs index 40e73a4ab..51bc9296d 100644 --- a/sidemantic-rs/src/sql/rewriter.rs +++ b/sidemantic-rs/src/sql/rewriter.rs @@ -2,7 +2,7 @@ use std::collections::{HashMap, HashSet}; -use polyglot_sql::parse as polyglot_parse; +use crate::semantic_input::dialects::parse_many as polyglot_parse; use polyglot_sql::{ expressions::{Identifier, Join, JoinKind, Select, TableRef, With}, generate as polyglot_generate, DialectType, Expression, @@ -26,6 +26,7 @@ pub struct QueryRewriter<'a> { rename_only: bool, security_controls: bool, warnings: std::cell::RefCell>, + used_preaggregation: std::cell::Cell, } impl<'a> QueryRewriter<'a> { @@ -37,6 +38,7 @@ impl<'a> QueryRewriter<'a> { rename_only: false, security_controls: false, warnings: std::cell::RefCell::new(Vec::new()), + used_preaggregation: std::cell::Cell::new(false), } } @@ -73,6 +75,7 @@ impl<'a> QueryRewriter<'a> { input_dialect: DialectType, output_dialect: DialectType, ) -> Result { + self.used_preaggregation.set(false); if let Some(rewritten) = self.rewrite_yardstick(sql, input_dialect, output_dialect)? { return Ok(rewritten); } @@ -97,7 +100,14 @@ impl<'a> QueryRewriter<'a> { )?); } - Ok(rewritten_statements.join(";\n")) + let mut sql = rewritten_statements.join(";\n"); + // Parsing semantic leaves into relational wrappers drops their trailing + // comments. Preserve routing independently so missing-rollup fallback + // and strict mode observe the compiler's actual selection. + if self.used_preaggregation.get() { + sql.push_str("\n-- used_preagg=true"); + } + Ok(sql) } fn rewrite_statement(&self, statement: Expression) -> Result { @@ -114,7 +124,7 @@ fn parse_sql_with_dialect(sql: &str, dialect: DialectType) -> Result Result, next_metric: usize, strict_fields: bool, + secured_subquery: bool, } impl Bindings { + fn unknown_field(&self, reference: &str) -> SidemanticError { + if self.secured_subquery { + SidemanticError::Security(format!( + "Cannot authorize semantic subquery: field '{reference}' is not declared" + )) + } else { + SidemanticError::Validation(format!("Field '{reference}' not found")) + } + } + fn resolve(&self, column: &Column) -> Result { let name = &column.name.name; if let Some(table) = &column.table { @@ -161,9 +172,7 @@ impl Bindings { } else if model.get_dimension(split_granularity(field).0).is_some() { false } else { - return Err(SidemanticError::Validation(format!( - "Field '{reference}' not found" - ))); + return Err(self.unknown_field(&reference)); } } else if self.graph.get_metric(&reference).is_some() { true @@ -282,9 +291,7 @@ impl Bindings { if model.get_metric(field).is_none() && model.get_dimension(split_granularity(field).0).is_none() { - return Err(SidemanticError::Validation(format!( - "Field '{reference}' not found" - ))); + return Err(self.unknown_field(&reference)); } } Ok(Some(Expression::qualified_column(owner, field))) @@ -409,7 +416,11 @@ impl Bindings { } impl QueryRewriter<'_> { - pub(super) fn compile_semantic_select(&self, mut select: Select) -> Result { let mut remainder = select.clone(); remainder.expressions.clear(); remainder.from = None; @@ -453,6 +464,7 @@ impl QueryRewriter<'_> { aggregate_inputs: Vec::new(), next_metric: 0, strict_fields: self.security_controls, + secured_subquery: self.security_controls && nested, }; let mut projections = Vec::new(); let mut aliases = HashSet::new(); @@ -548,8 +560,16 @@ impl QueryRewriter<'_> { bindings.query.order_by.clear(); } let sql = SqlGenerator::new(&bindings.graph).generate(&bindings.query)?; + // Preserve routing metadata outside the parsed subquery: trailing SQL + // comments are discarded by the parser and can consume its closing ')'. + let sql = if let Some(inner) = sql.strip_suffix("\n-- used_preagg=true") { + self.used_preaggregation.set(true); + inner + } else { + &sql + }; let mut wrapper = parse_sql_with_dialect( - &format!("SELECT * FROM ({sql}) AS __semantic_query"), + &format!("SELECT * FROM ({sql}\n) AS __semantic_query"), DialectType::DuckDB, )?; let Expression::Select(mut outer) = wrapper.remove(0) else { diff --git a/sidemantic-rs/src/sql/rewriter/policy.rs b/sidemantic-rs/src/sql/rewriter/policy.rs index ace759ca3..62eeac2f0 100644 --- a/sidemantic-rs/src/sql/rewriter/policy.rs +++ b/sidemantic-rs/src/sql/rewriter/policy.rs @@ -124,13 +124,14 @@ impl QueryRewriter<'_> { rename_only: true, security_controls: false, warnings: std::cell::RefCell::new(Vec::new()), + used_preaggregation: std::cell::Cell::new(false), }; rewriter.rewrite_policy_statement(statement) } pub(super) fn rewrite_policy_statement(&self, statement: Expression) -> Result { let mut names = CteNames::new(&statement, self.graph, self.policy_definitions)?; - self.rewrite_policy_query(statement, &HashMap::new(), &mut names) + self.rewrite_policy_query(statement, &HashMap::new(), &mut names, false) } fn rewrite_policy_query( @@ -138,34 +139,36 @@ impl QueryRewriter<'_> { statement: Expression, inherited_ctes: &HashMap, names: &mut CteNames, + nested: bool, ) -> Result { // Set operations own their WITH/ORDER/LIMIT clauses. Rewrite their // operands without moving those clauses onto an individual SELECT. match statement { Expression::Union(mut set) => { let ctes = self.rewrite_ctes(&mut set.with, inherited_ctes, names)?; - set.left = self.rewrite_policy_query(set.left.clone(), &ctes, names)?; - set.right = self.rewrite_policy_query(set.right.clone(), &ctes, names)?; + set.left = self.rewrite_policy_query(set.left.clone(), &ctes, names, nested)?; + set.right = self.rewrite_policy_query(set.right.clone(), &ctes, names, nested)?; return self.rewrite_set_clauses(Expression::Union(set), &ctes, names); } Expression::Intersect(mut set) => { let ctes = self.rewrite_ctes(&mut set.with, inherited_ctes, names)?; - set.left = self.rewrite_policy_query(set.left.clone(), &ctes, names)?; - set.right = self.rewrite_policy_query(set.right.clone(), &ctes, names)?; + set.left = self.rewrite_policy_query(set.left.clone(), &ctes, names, nested)?; + set.right = self.rewrite_policy_query(set.right.clone(), &ctes, names, nested)?; return self.rewrite_set_clauses(Expression::Intersect(set), &ctes, names); } Expression::Except(mut set) => { let ctes = self.rewrite_ctes(&mut set.with, inherited_ctes, names)?; - set.left = self.rewrite_policy_query(set.left.clone(), &ctes, names)?; - set.right = self.rewrite_policy_query(set.right.clone(), &ctes, names)?; + set.left = self.rewrite_policy_query(set.left.clone(), &ctes, names, nested)?; + set.right = self.rewrite_policy_query(set.right.clone(), &ctes, names, nested)?; return self.rewrite_set_clauses(Expression::Except(set), &ctes, names); } Expression::Subquery(mut query) => { - query.this = self.rewrite_policy_query(query.this, inherited_ctes, names)?; + query.this = self.rewrite_policy_query(query.this, inherited_ctes, names, true)?; return Ok(Expression::Subquery(query)); } Expression::Paren(mut paren) => { - paren.this = self.rewrite_policy_query(paren.this, inherited_ctes, names)?; + paren.this = + self.rewrite_policy_query(paren.this, inherited_ctes, names, nested)?; return Ok(Expression::Paren(paren)); } Expression::Select(_) => {} @@ -197,8 +200,8 @@ impl QueryRewriter<'_> { && select.from.as_ref().is_some_and(|from| { from.expressions.first().is_some_and(|source| { matches!(source, Expression::Table(table) - if table.schema.is_none() && table.catalog.is_none() - && !ctes.contains_key(&table.name.name.to_ascii_lowercase()) + if (table.schema.is_some() || table.catalog.is_some() + || !ctes.contains_key(&table.name.name.to_ascii_lowercase())) && (table.name.name.eq_ignore_ascii_case("metrics") || self.graph.get_model(&table.name.name).is_some())) }) @@ -217,13 +220,22 @@ impl QueryRewriter<'_> { if semantic_leaf { if let Some(from) = &select.from { if let Some(Expression::Table(table)) = from.expressions.first() { + // `metrics` is a virtual relation, not a physical table in + // an arbitrary catalog/schema. Qualified model names still + // resolve through the policy preparer below. + if self.query_preparer.is_some() + && table.name.name.eq_ignore_ascii_case("metrics") + && (table.schema.is_some() || table.catalog.is_some()) + { + return Err(unsupported()); + } validate_table(table)?; if !table.column_aliases.is_empty() { return Err(unsupported()); } } } - let mut compiled = self.compile_semantic_select(*select)?; + let mut compiled = self.compile_semantic_select(*select, nested)?; // Keep the existing semantic-root error contract even though the // renamed input CTE would no longer capture the generated source. // Only names actually emitted for this query are conflicts. @@ -271,7 +283,7 @@ impl QueryRewriter<'_> { if with.recursive { ctes.insert(original.clone(), alias.clone()); } - cte.this = self.rewrite_policy_query(cte.this.clone(), &ctes, names)?; + cte.this = self.rewrite_policy_query(cte.this.clone(), &ctes, names, true)?; cte.alias = alias.clone(); ctes.insert(original, alias); } @@ -295,7 +307,7 @@ impl QueryRewriter<'_> { | Expression::Except(_) ) { return self - .rewrite_policy_query(node.clone(), ctes, names) + .rewrite_policy_query(node.clone(), ctes, names, true) .map(Some); } if self.query_preparer.is_some() @@ -389,7 +401,7 @@ impl QueryRewriter<'_> { Ok(Expression::Table(table)) } Expression::Subquery(mut subquery) => { - subquery.this = self.rewrite_policy_query(subquery.this, ctes, names)?; + subquery.this = self.rewrite_policy_query(subquery.this, ctes, names, true)?; Ok(Expression::Subquery(subquery)) } Expression::Alias(mut alias) => { @@ -424,6 +436,11 @@ impl QueryRewriter<'_> { fn validate_table(table: &TableRef) -> Result<()> { let mut remainder = table.clone(); remainder.name = Identifier::new(""); + // Semantic model lookup uses the terminal relation name, including the + // schemas advertised by the PostgreSQL server. Qualifiers do not bypass + // semantic compilation or its policy preparer. + remainder.schema = None; + remainder.catalog = None; remainder.alias = None; remainder.alias_explicit_as = false; remainder.column_aliases.clear(); @@ -440,6 +457,102 @@ mod tests { use super::*; use crate::core::{Dimension, Metric, Model}; + #[test] + fn undeclared_nested_fields_are_security_errors_only_with_controls() { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("orders", "id") + .with_table("physical_orders") + .with_metric(Metric::sum("revenue", "amount")), + ) + .unwrap(); + let prepare = |_: &SemanticGraph, _: &mut SemanticQuery| Ok(()); + for controls in [false, true] { + let rewriter = QueryRewriter::new(&graph).with_query_preparer(&prepare, "", controls); + let nested = rewriter + .rewrite("SELECT * FROM (SELECT amount FROM orders) AS scoped") + .unwrap_err(); + if controls { + assert!(matches!(nested, SidemanticError::Security(ref message) + if message.contains("semantic subquery"))); + } else { + assert!(matches!(nested, SidemanticError::Validation(_))); + } + let root = rewriter.rewrite("SELECT amount FROM orders").unwrap_err(); + assert!(matches!(root, SidemanticError::Validation(_))); + } + } + + #[test] + fn qualified_semantic_sources_still_prepare_policy() { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("orders", "id") + .with_table("private_orders") + .with_dimension(Dimension::new("tenant")) + .with_metric(Metric::sum("revenue", "amount")), + ) + .unwrap(); + let prepared = std::cell::Cell::new(0); + let prepare = |_: &SemanticGraph, query: &mut SemanticQuery| { + prepared.set(prepared.get() + 1); + assert_eq!(query.metrics, vec!["orders.revenue"]); + query.filters.push("orders.tenant = 1".into()); + Ok(()) + }; + let rewriter = QueryRewriter::new(&graph).with_query_preparer(&prepare, "", true); + for source in [ + "main.orders", + "\"main\".\"orders\"", + "semantic_layer.orders", + ] { + // A same-named CTE only shadows unqualified relations. + let sql = + format!("WITH orders AS (SELECT 0 AS revenue) SELECT o.revenue FROM {source} AS o"); + let rewritten = rewriter.rewrite(&sql).unwrap(); + assert!(rewritten.contains("private_orders"), "{rewritten}"); + assert!(rewritten.contains("tenant = 1"), "{rewritten}"); + assert!(!rewritten.contains(source), "{rewritten}"); + } + assert_eq!(prepared.get(), 3); + for source in [ + "main.metrics", + "\"main\".\"metrics\"", + "catalog.main.metrics", + ] { + for sql in [ + format!("SELECT orders.revenue FROM {source}"), + format!("SELECT * FROM (SELECT orders.revenue FROM {source}) AS q"), + ] { + assert!( + matches!( + rewriter.rewrite(&sql), + Err(SidemanticError::UnsupportedSemanticFeatures { .. }) + ), + "{sql}" + ); + } + } + assert_eq!(prepared.get(), 3); + + let deny = |_: &SemanticGraph, _: &mut SemanticQuery| { + Err(SidemanticError::Validation("access denied".into())) + }; + let rewriter = QueryRewriter::new(&graph).with_query_preparer(&deny, "", true); + assert!(rewriter + .rewrite("SELECT revenue FROM main.orders") + .unwrap_err() + .to_string() + .contains("access denied")); + assert!(rewriter + .rewrite("SELECT * FROM main.private_orders") + .unwrap_err() + .to_string() + .contains("rewrite.policy_select_shape")); + } + #[test] fn generated_cte_conflicts_preserve_semantic_root_contract() { let mut graph = SemanticGraph::new(); diff --git a/sidemantic-rs/src/sql/rewriter/yardstick.rs b/sidemantic-rs/src/sql/rewriter/yardstick.rs index 7f720a0fb..3c287b0b4 100644 --- a/sidemantic-rs/src/sql/rewriter/yardstick.rs +++ b/sidemantic-rs/src/sql/rewriter/yardstick.rs @@ -208,7 +208,8 @@ impl Lowerer<'_, '_> { .authored_expression(candidate) .ok() .filter(|expr| reference(expr).is_some()); - visible = true; + // Curly references have the plain measure evaluation context; + // only AGGREGATE(...) inherits the visible row predicate. } else { while stream .get(end + 1) @@ -290,6 +291,9 @@ impl Lowerer<'_, '_> { map_columns(&mut value, &mut |node| { if let Expression::Column(mut column) = node { column.table = None; + // Quoting is syntax, not a different context dimension. The + // declared dimension expansion may have removed source quotes. + column.name.quoted = false; return Ok(Expression::Column(column)); } Ok(node) @@ -894,14 +898,23 @@ impl Lowerer<'_, '_> { let inner = self.remap(group.clone(), &aliases, "_inner", single)?; let mut outer = self.remap(group, &[model_name], alias, single)?; if let Some(output_alias) = context_aliases.get(&signature) { - let unsafe_alias = model.dimensions.iter().any(|dimension| { - dimension.name.eq_ignore_ascii_case(output_alias) - && self - .expression(dimension.sql_expr()) - .ok() - .and_then(|expression| reference(&expression)) - .is_some_and(|(_, name)| name.eq_ignore_ascii_case(output_alias)) - }); + // SELECT * imports can expose physical grouping columns + // without declaring dimensions. An unqualified output + // alias with that same name would bind to the inner source + // and turn correlation into a tautology. + let unsafe_alias = columns + .iter() + .any(|(_, name)| name.eq_ignore_ascii_case(output_alias)) + || model.dimensions.iter().any(|dimension| { + dimension.name.eq_ignore_ascii_case(output_alias) + && self + .expression(dimension.sql_expr()) + .ok() + .and_then(|expression| reference(&expression)) + .is_some_and(|(_, name)| { + name.eq_ignore_ascii_case(output_alias) + }) + }); if !unsafe_alias { outer = Expression::Identifier(Identifier::new(output_alias)); } @@ -1376,6 +1389,19 @@ fn has_aggregate_semantics(value: &Value) -> bool { } fn expand_groups(expression: &Expression, output: &mut Vec) -> Result<()> { + // The pinned parser represents inline ROLLUP/CUBE as ordinary functions, + // but WITH ROLLUP/CUBE as the typed variants handled below. + if let Expression::Function(function) = expression { + if ["ROLLUP", "CUBE", "GROUPING SETS"] + .iter() + .any(|name| function.name.eq_ignore_ascii_case(name)) + { + for child in &function.args { + expand_groups(child, output)?; + } + return Ok(()); + } + } let value = encode(expression)?; let Some(fields) = value.as_object() else { return Ok(()); @@ -1474,3 +1500,32 @@ impl QueryRewriter<'_> { dialects::emit(decode(value)?, DialectType::DuckDB, output).map(Some) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::{Metric, Model, SemanticGraph}; + + #[test] + fn wildcard_import_grouping_retains_outer_column_qualification() { + let mut model = Model::new("orders_v", "id") + .with_table("raw_orders") + .with_metric(Metric::sum("revenue", "amount")); + model.metadata = Some(serde_json::json!({"yardstick": {}})); + let mut graph = SemanticGraph::new(); + graph.add_model(model).unwrap(); + for grouping in [ + "o.product", + "ROLLUP(o.product)", + "CUBE(o.product)", + "GROUPING SETS ((o.product), ())", + ] { + let sql = QueryRewriter::new(&graph).rewrite(&format!("SELECT o.product, AGGREGATE(o.revenue) AS total FROM orders_v AS o GROUP BY {grouping}")).unwrap(); + assert!( + sql.contains("(_inner.product) IS NOT DISTINCT FROM (o.product)"), + "{sql}" + ); + assert!(!sql.contains("IS NOT DISTINCT FROM (product)"), "{sql}"); + } + } +} diff --git a/sidemantic-rs/tests/many_role_contracts.rs b/sidemantic-rs/tests/many_role_contracts.rs new file mode 100644 index 000000000..2c1f55766 --- /dev/null +++ b/sidemantic-rs/tests/many_role_contracts.rs @@ -0,0 +1,65 @@ +use serde_json::json; +use sidemantic::semantic_input::compile_with_semantic_input; + +fn source() -> serde_json::Value { + json!({ + "version": 1, + "input_dialect": "duckdb", + "models": [ + { + "name": "orders", "table": "orders", "primary_key": "id", + "dimensions": [{"name": "id", "type": "numeric"}], + "metrics": [{"name": "revenue", "agg": "sum", "sql": "amount"}], + "relationships": [ + {"name": "primary_tags", "target_model": "tags", "type": "many_to_many", + "through": "links", "through_foreign_key": "order_id", + "related_foreign_key": "primary_tag", "edge_id": "primary_tags"}, + {"name": "secondary_tags", "target_model": "tags", "type": "many_to_many", + "through": "links", "through_foreign_key": "order_id", + "related_foreign_key": "secondary_tag", "edge_id": "secondary_tags"} + ] + }, + { + "name": "tags", "table": "tags", "primary_key": "id", + "dimensions": [{"name": "name", "type": "categorical"}], + "metrics": [{"name": "sum_ids", "agg": "sum", "sql": "id"}] + }, + {"name": "links", "table": "links", "primary_key": "id"} + ] + }) +} + +#[test] +fn target_metric_inputs_keep_the_requested_role() { + for role in ["primary_tags", "secondary_tags"] { + let query = json!({"metrics": [format!("{role}.sum_ids")], "dimensions": ["orders.id"]}); + let sql = compile_with_semantic_input(&source().to_string(), &query.to_string()).unwrap(); + assert!(sql.contains(&format!("{role}_cte")), "{sql}"); + assert!(sql.contains(&format!("{role}$through_cte")), "{sql}"); + assert!(sql.contains("SELECT DISTINCT"), "{sql}"); + } +} + +#[test] +fn junction_policies_filter_links_without_changing_source_preservation() { + let mut source = source(); + source["models"][2]["security"] = json!({"row_filters": ["tenant = {{ user.tenant }}"]}); + source["models"][2]["invariant_filters"] = json!(["enabled"]); + let query = json!({ + "metrics": ["orders.revenue"], + "dimensions": ["primary_tags.name", "secondary_tags.name"], + "user_attributes": {"tenant": "a"} + }); + let sql = compile_with_semantic_input(&source.to_string(), &query.to_string()).unwrap(); + for role in ["primary_tags", "secondary_tags"] { + assert!( + sql.contains(&format!("LEFT JOIN {role}$through_cte")), + "{sql}" + ); + assert!( + !sql.contains(&format!("INNER JOIN {role}$through_cte")), + "{sql}" + ); + } + assert_eq!(sql.matches("tenant = 'a' AND enabled").count(), 2, "{sql}"); +} diff --git a/sidemantic-rs/tests/package_metadata.rs b/sidemantic-rs/tests/package_metadata.rs index 634970f43..229190270 100644 --- a/sidemantic-rs/tests/package_metadata.rs +++ b/sidemantic-rs/tests/package_metadata.rs @@ -7,12 +7,14 @@ fn toml_string_value(contents: &str, key: &str) -> Option { } #[test] -fn rust_crate_and_python_extension_versions_match() { +fn python_distributions_versions_match() { let pyproject = include_str!("../pyproject.toml"); let pyproject_version = toml_string_value(pyproject, "version").expect("pyproject.toml project.version"); - assert_eq!(env!("CARGO_PKG_VERSION"), pyproject_version); + let parent_version = toml_string_value(include_str!("../../pyproject.toml"), "version") + .expect("parent pyproject.toml project.version"); + assert_eq!(parent_version, pyproject_version); } #[test] @@ -21,7 +23,7 @@ fn python_extension_metadata_targets_the_expected_module_and_feature() { assert!(pyproject.contains("name = \"sidemantic-rs\"")); assert!(pyproject.contains("module-name = \"sidemantic_rs\"")); - assert!(pyproject.contains("features = [\"python-adbc\"]")); + assert!(pyproject.contains("features = [\"python\"]")); assert!(pyproject.contains("license = \"AGPL-3.0-only\"")); } diff --git a/sidemantic-rs/tests/python_wheel_python_smoke.py b/sidemantic-rs/tests/python_wheel_python_smoke.py index 817ca03f2..796153c59 100644 --- a/sidemantic-rs/tests/python_wheel_python_smoke.py +++ b/sidemantic-rs/tests/python_wheel_python_smoke.py @@ -4,6 +4,8 @@ import importlib.metadata import importlib.util +import tomllib +from pathlib import Path import sidemantic_rs @@ -11,7 +13,10 @@ if root_python_package is not None: raise AssertionError("isolated sidemantic_rs python-feature wheel unexpectedly found root sidemantic package") -if importlib.metadata.version("sidemantic-rs") != "0.1.0": +expected_version = tomllib.loads((Path(__file__).resolve().parents[1] / "pyproject.toml").read_text())["project"][ + "version" +] +if importlib.metadata.version("sidemantic-rs") != expected_version: raise AssertionError("unexpected sidemantic-rs wheel version") models_yaml = """ diff --git a/sidemantic-rs/tests/python_wheel_smoke.py b/sidemantic-rs/tests/python_wheel_smoke.py index 74eeeda4d..84e6e3625 100644 --- a/sidemantic-rs/tests/python_wheel_smoke.py +++ b/sidemantic-rs/tests/python_wheel_smoke.py @@ -5,6 +5,8 @@ import importlib.metadata import importlib.util import json +import tomllib +from pathlib import Path import sidemantic_rs @@ -26,7 +28,10 @@ def expect_raises(exc_type: type[BaseException], func, *args) -> str: if root_python_package is not None: raise AssertionError("isolated sidemantic_rs wheel smoke unexpectedly found root sidemantic package") -if importlib.metadata.version("sidemantic-rs") != "0.1.0": +expected_version = tomllib.loads((Path(__file__).resolve().parents[1] / "pyproject.toml").read_text())["project"][ + "version" +] +if importlib.metadata.version("sidemantic-rs") != expected_version: raise AssertionError("unexpected sidemantic-rs wheel version") models_yaml = """ diff --git a/sidemantic/cli.py b/sidemantic/cli.py index 6147cb572..bef5fd5d1 100644 --- a/sidemantic/cli.py +++ b/sidemantic/cli.py @@ -184,11 +184,14 @@ def _resolve_engine_options(engine: str | None, fallback: bool | None) -> tuple[ if resolved_fallback is None: resolved_fallback = _loaded_config.runtime.fallback + if resolved_engine is None: + from sidemantic.runtime import default_engine + + resolved_engine = default_engine() + if resolved_fallback is None: resolved_fallback = resolved_engine == "auto" - if resolved_engine is None and fallback is not None: - raise typer.BadParameter("--fallback/--no-fallback requires --engine or runtime.engine in config") if resolved_engine == "python" and resolved_fallback: raise typer.BadParameter("--fallback is only meaningful with the rust or auto engine") diff --git a/sidemantic/config.py b/sidemantic/config.py index e6130a222..126cfa2fa 100644 --- a/sidemantic/config.py +++ b/sidemantic/config.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, ConfigDict, Field +from sidemantic.runtime import default_engine from sidemantic.yaml_compat import safe_load as _yaml_safe_load @@ -155,7 +156,7 @@ class APIServerConfig(BaseModel): class RuntimeConfig(BaseModel): """Runtime engine selection.""" - engine: Literal["python", "rust", "auto"] = Field(default="python", description="Runtime engine") + engine: Literal["python", "rust", "auto"] = Field(default_factory=default_engine, description="Runtime engine") fallback: bool = Field(default=False, description="Allow Rust runtime fallback to Python") diff --git a/sidemantic/core/semantic_layer.py b/sidemantic/core/semantic_layer.py index 01a4039b4..28a3b05e8 100644 --- a/sidemantic/core/semantic_layer.py +++ b/sidemantic/core/semantic_layer.py @@ -18,6 +18,7 @@ from sidemantic.core.metric import Metric from sidemantic.core.model import Model from sidemantic.core.semantic_graph import SemanticGraph +from sidemantic.runtime import default_engine from sidemantic.rust_bridge import get_rust_module from sidemantic.rust_parity import is_strict_for from sidemantic.semantic_handoff import RustBackendUnavailableError, UnsupportedSemanticFeaturesError @@ -114,7 +115,8 @@ def __init__( loading extensions, attaching catalogs, creating secrets) engine: Runtime engine for native query validation/compilation. Supported values are "python", "rust", and "auto". If omitted, - legacy SIDEMANTIC_RS_* environment flags are honored. + defaults to Rust (Python in Pyodide). SIDEMANTIC_ENGINE overrides + the platform default; legacy SIDEMANTIC_RS_* flags remain supported. fallback: Whether an unavailable Rust backend or a known unsupported capability may fall back to Python. Invalid input and unexpected compiler failures propagate. Defaults to False for engine="rust" and True for engine="auto". @@ -136,6 +138,10 @@ def __init__( """ from sidemantic.db.base import BaseDatabaseAdapter + # Preserve explicitly configured legacy parity runs. Ordinary installs + # select the same native default as the CLI and project configuration. + if engine is None and not any(key.startswith("SIDEMANTIC_RS_") for key in os.environ): + engine = default_engine() if engine is not None: engine = engine.lower() if engine not in {"python", "rust", "auto"}: @@ -1361,6 +1367,7 @@ def _validate_query( dimensions, input_dialect="duckdb" if self.dialect == "postgres" else self.dialect, rust_module=self._rust_module, + **({"allow_non_additive_unsafe": True} if self.allow_non_additive_unsafe else {}), ) except (RustBackendUnavailableError, UnsupportedSemanticFeaturesError) as exc: if self._strict_rust_query_validation or self._rust_no_fallback: @@ -1568,6 +1575,7 @@ def _compile_with_rust( "preagg_schema": self.preagg_schema, "user_attributes": user_attributes, "enforce_visibility": self.enforce_visibility, + **({"allow_non_additive_unsafe": True} if self.allow_non_additive_unsafe else {}), } try: @@ -1629,7 +1637,10 @@ def _apply_post_process(self, inner_sql: str, post_process: str | None) -> str: # subquery position. CTEs inside subqueries are valid SQL in # all target databases and naturally scoped, avoiding name # collisions with CTEs in the post_process SQL. - return post_process.replace("{inner}", stripped) + # Native compilation can leave a routing marker (or source SQL can + # end in a line comment). Terminate it before the wrapper's closing + # parenthesis, preserving the SQL and its comments verbatim. + return post_process.replace("{inner}", stripped + "\n") return inner_sql @@ -2223,6 +2234,7 @@ def sql(self, query: str, *, user_attributes: dict | None = None): self.dialect, self.use_preaggregations, self.enforce_visibility, + self.allow_non_additive_unsafe, self._explicit_engine, self._use_rust_sql_generator, self._rust_no_fallback, @@ -2284,6 +2296,7 @@ def explain_sql(self, query: str, strict: bool = True): enforce_visibility=self.enforce_visibility, use_rust_rewriter=self._use_rust_sql_generator if self._explicit_engine else None, rust_no_fallback=self._rust_no_fallback, + allow_non_additive_unsafe=self.allow_non_additive_unsafe, ) explanation = rewriter.explain(query, strict=strict) self.last_engine_selection = rewriter.last_engine_selection diff --git a/sidemantic/core/transport_security.py b/sidemantic/core/transport_security.py index 952043398..6f1daa319 100644 --- a/sidemantic/core/transport_security.py +++ b/sidemantic/core/transport_security.py @@ -179,6 +179,7 @@ def rewrite_transport_sql( getattr(layer, "_use_rust_sql_generator", None) if getattr(layer, "_explicit_engine", False) else None ), rust_no_fallback=getattr(layer, "_rust_no_fallback", None), + allow_non_additive_unsafe=getattr(layer, "allow_non_additive_unsafe", False), ) # Yardstick's explicit and implicit measure paths expand directly against # physical model tables. They do not currently route those reads through diff --git a/sidemantic/runtime.py b/sidemantic/runtime.py new file mode 100644 index 000000000..2c5b92ceb --- /dev/null +++ b/sidemantic/runtime.py @@ -0,0 +1,13 @@ +"""Shared runtime defaults without importing the native extension.""" + +import os +import sys +from typing import Literal, cast + + +def default_engine() -> Literal["python", "rust", "auto"]: + """Use Rust on native hosts and Python in Pyodide; allow a process override.""" + engine = os.environ.get("SIDEMANTIC_ENGINE", "python" if sys.platform == "emscripten" else "rust").lower() + if engine not in {"python", "rust", "auto"}: + raise ValueError("SIDEMANTIC_ENGINE must be one of: python, rust, auto") + return cast(Literal["python", "rust", "auto"], engine) diff --git a/sidemantic/rust_bridge.py b/sidemantic/rust_bridge.py index b8bba1397..c57f2ce4e 100644 --- a/sidemantic/rust_bridge.py +++ b/sidemantic/rust_bridge.py @@ -40,8 +40,7 @@ def get_rust_module() -> object: raise raise RustBackendUnavailableError( "Rust backend requires the sidemantic_rs Python extension. " - "Build it with: uv run --with maturin maturin develop " - "--manifest-path sidemantic-rs/Cargo.toml --features python-adbc" + "Install it with: uv pip install sidemantic-rs, or select --engine python." ) from e return sidemantic_rs @@ -173,6 +172,7 @@ def validate_semantic_input( dimensions: list[str], *, input_dialect: str = "duckdb", + allow_non_additive_unsafe: bool = False, rust_module=None, ) -> list[str]: """Validate references through the same input contract used by compilation.""" @@ -181,7 +181,14 @@ def validate_semantic_input( module, "validate_with_semantic_input", graph_to_semantic_json(graph, input_dialect=input_dialect), - json.dumps({"metrics": metrics, "dimensions": dimensions}, allow_nan=False), + json.dumps( + { + "metrics": metrics, + "dimensions": dimensions, + **({"allow_non_additive_unsafe": True} if allow_non_additive_unsafe else {}), + }, + allow_nan=False, + ), ) if not isinstance(errors, list) or not all(isinstance(error, str) for error in errors): raise TypeError("Rust validator returned an invalid errors payload") @@ -197,6 +204,8 @@ def rewrite_semantic_input( output_dialect: str | None = None, user_attributes: dict | None = None, enforce_visibility: bool = False, + use_preaggregations: bool = False, + allow_non_additive_unsafe: bool = False, rust_module=None, ) -> str: """Rewrite SQL with caller context through the versioned graph contract.""" @@ -211,6 +220,8 @@ def rewrite_semantic_input( or sql_dialect is not None or user_attributes is not None or enforce_visibility + or use_preaggregations + or allow_non_additive_unsafe or any(model.security is not None or model.invariant_filters for model in graph.models.values()) ) args = [graph_to_semantic_json(graph, input_dialect=input_dialect), sql] @@ -226,6 +237,8 @@ def rewrite_semantic_input( { "user_attributes": user_attributes, "enforce_visibility": enforce_visibility, + **({"use_preaggregations": True} if use_preaggregations else {}), + **({"allow_non_additive_unsafe": True} if allow_non_additive_unsafe else {}), **({"output_dialect": output_dialect} if output_dialect is not None else {}), **({"sql_dialect": sql_dialect} if sql_dialect is not None else {}), }, @@ -417,6 +430,9 @@ def _normalize_metric_type(metric_payload: dict, *, empty_filters_to_none: bool normalized["type"] = None elif metric_type == "timecomparison": normalized["type"] = "time_comparison" + # Rust serializes an unset window as null; Python uses its default instead. + if normalized.get("non_additive_window") is None: + normalized.pop("non_additive_window", None) if empty_filters_to_none and normalized.get("filters") == []: normalized["filters"] = None return normalized diff --git a/sidemantic/semantic_handoff.py b/sidemantic/semantic_handoff.py index e35b69f9a..0163092eb 100644 --- a/sidemantic/semantic_handoff.py +++ b/sidemantic/semantic_handoff.py @@ -7,6 +7,8 @@ from pydantic import BaseModel +from sidemantic.core.model import Model +from sidemantic.core.relationship import Relationship from sidemantic.core.semantic_graph import SemanticGraph SEMANTIC_INPUT_VERSION = 1 @@ -31,7 +33,18 @@ def _definition(value: BaseModel) -> dict[str, Any]: field is absent. Source identity/type fields excluded by authoring exports remain part of this compiler input. """ - data = value.model_dump(mode="json", exclude_none=True, exclude_defaults=True) + if isinstance(value, Model) and value.extends: + # Unresolved children distinguish omitted fields from explicit clears + # and default-valued overrides. The receiver applies inheritance. + data = value.model_dump(mode="json", exclude_unset=True) + else: + data = value.model_dump(mode="json", exclude_none=True, exclude_defaults=True) + if isinstance(value, Relationship) and value.type in ("one_to_one", "one_to_many") and value.primary_key is None: + # TMDL retains the declared source endpoint separately from the model's + # primary key. Snapshot that existing endpoint without resolving SQL. + source_column = getattr(value, "_tmdl_from_column", None) + if isinstance(source_column, str) and source_column.strip(): + data["primary_key"] = source_column for name in ("logical_data_type", "declared_is_time", "edge_id"): field_value = getattr(value, name, None) if field_value is not None: @@ -46,7 +59,7 @@ def _definition(value: BaseModel) -> dict[str, Any]: def graph_to_semantic_input(graph: SemanticGraph, *, input_dialect: str = "duckdb") -> dict[str, Any]: - """Create an inert versioned snapshot of a resolved semantic graph. + """Create an inert versioned snapshot of a semantic graph. Graph metrics stay at graph scope. Only owners explicitly recorded by the graph are transmitted. SQL is copied verbatim; reference binding, dialect @@ -57,13 +70,24 @@ def graph_to_semantic_input(graph: SemanticGraph, *, input_dialect: str = "duckd models = [] for model in graph.models.values(): definition = _definition(model) - definition["primary_key"] = list(model.primary_key_columns) or None + if not model.extends or "primary_key" in model.model_fields_set: + definition["primary_key"] = list(model.primary_key_columns) or None models.append(definition) return { "version": SEMANTIC_INPUT_VERSION, "input_dialect": input_dialect, "models": models, - "metrics": [_definition(metric) for metric in graph.metrics.values()], + "metrics": [ + _definition(metric) + for metric in graph.metrics.values() + # add_model exposes these same objects through graph.metrics for + # unqualified lookup. The native model index already does that. + if not ( + metric.type in ("time_comparison", "conversion") + and metric.name not in graph.metric_owners + and any(metric is owned for model in graph.models.values() for owned in model.metrics) + ) + ], "metric_owners": dict(graph.metric_owners), "parameters": [_definition(parameter) for parameter in graph.parameters.values()], "table_calculations": [_definition(calculation) for calculation in graph.table_calculations.values()], diff --git a/sidemantic/sql/generator.py b/sidemantic/sql/generator.py index f8217d891..887bac0c8 100644 --- a/sidemantic/sql/generator.py +++ b/sidemantic/sql/generator.py @@ -3021,6 +3021,12 @@ def _generate_with_preaggregation( calculations: dict[str, str] = {} leaf_refs: list[str] = [] + # Children need stable, distinct output names even when public fields + # share a basename or a calculation hides a colliding aggregate leaf. + child_aliases = { + f"{reference}__{grain}" if grain else reference: f"__sidemantic_dimension_{index}" + for index, (reference, grain) in enumerate(parsed_dims) + } def expand_metric(reference: str, context: str | None = None, stack: tuple[str, ...] = ()) -> str: if "." not in reference and context and self.graph.get_model(context).get_metric(reference): @@ -3032,8 +3038,18 @@ def expand_metric(reference: str, context: str | None = None, stack: tuple[str, if model_name is not None and aggregate_models == {model_name}: if reference not in leaf_refs: leaf_refs.append(reference) - source_name = aliases.get(reference) or metric.name - calculations[reference] = f"{model_name}_preagg.{self._quote_identifier(source_name)}" + child_aliases[reference] = f"__sidemantic_metric_{len(leaf_refs) - 1}" + source_name = child_aliases[reference] + expression = f"{model_name}_preagg.{self._quote_identifier(source_name)}" + if metric.agg in ("count", "count_distinct", "approx_count_distinct"): + # A missing group has an empty count population. Restore + # zero before evaluating formulas or their outer defaults. + expression = f"COALESCE({expression}, 0)" + else: + # A source can be absent from a sibling's group entirely, + # so its child-level default has no row on which to run. + expression = self._wrap_with_fill_nulls(expression, metric) + calculations[reference] = expression return calculations[reference] stack = (*stack, reference) if metric.type == "ratio": @@ -3166,8 +3182,13 @@ def expand_metric(reference: str, context: str | None = None, stack: tuple[str, # Generate sub-query for this model's metrics at the dimension grain # We call generate() recursively but it won't trigger pre-aggregation - # again because each sub-query has metrics from only one model - sub_query = self.generate( + # again because each sub-query has metrics from only one model. + # Preserve that source's unmatched rows regardless of dimension + # order. An explicit Explore scope still controls the population. + child_generator = copy(self) + child_generator.base_model = self.base_model or model_name + child_generator._generate_cache = {} + sub_query = child_generator.generate( metrics=model_metrics, dimensions=dimensions, _resolved_filters=tuple(model_filters), @@ -3175,7 +3196,7 @@ def expand_metric(reference: str, context: str | None = None, stack: tuple[str, order_by=None, limit=None, offset=None, - aliases=aliases, + aliases=child_aliases, use_preaggregations=use_preaggregations, user_attributes=user_attributes, ) @@ -3191,6 +3212,13 @@ def expand_metric(reference: str, context: str | None = None, stack: tuple[str, # Build the final SELECT that joins all pre-aggregated CTEs select_exprs = [] output_names: dict[str, str] = {} + public_name_counts: dict[str, int] = {} + for dim_ref, gran in parsed_dims: + name = dim_ref.split(".", 1)[-1] + (f"__{gran}" if gran else "") + public_name_counts[name] = public_name_counts.get(name, 0) + 1 + for reference in metrics: + _, metric = self.graph.resolve_metric_reference(reference) + public_name_counts[metric.name] = public_name_counts.get(metric.name, 0) + 1 def register_output_name(output_name: str, *refs: str) -> None: for ref in refs: @@ -3205,11 +3233,9 @@ def dimension_output_name(dim_ref: str, dim_name: str, gran: str | None) -> str: full_ref = dim_ref default = dim_name - canonical_ref = full_ref - return aliases.get(full_ref) or aliases.get(canonical_ref) or default - - def metric_source_name(metric_ref: str, metric_name: str) -> str: - return aliases.get(metric_ref) or aliases.get(f"{metric_ref.split('.')[0]}.{metric_name}") or metric_name + if public_name_counts[default] > 1: + default = f"{dim_ref.split('.', 1)[0]}_{default}" + return aliases.get(full_ref) or default # Add dimensions - use COALESCE across all CTEs for dim_ref, gran in parsed_dims: @@ -3221,44 +3247,34 @@ def metric_source_name(metric_ref: str, metric_name: str) -> str: register_output_name(output_name, full_ref, canonical_ref, dim_name, col_name, output_name) # Build COALESCE expression - quoted_source = self._quote_identifier(output_name) + quoted_source = self._quote_identifier(child_aliases[full_ref]) coalesce_parts = [f"{cte}.{quoted_source}" for cte in cte_names] select_exprs.append(f"COALESCE({', '.join(coalesce_parts)}) AS {self._quote_alias(output_name)}") - # Check for metric name collisions across models - metric_name_counts: dict[str, int] = {} - for metric_ref in metrics: - _, metric = self.graph.resolve_metric_reference(metric_ref) - metric_name_counts[metric.name] = metric_name_counts.get(metric.name, 0) + 1 - # Add metrics in requested order, including calculations over child CTEs. metric_selects: dict[str, str] = {} for model_name, model_metrics in metrics_by_model.items(): - cte_name = f"{model_name}_preagg" for metric_ref in model_metrics: if metric_ref not in metrics: continue metric_name = metric_ref.split(".", 1)[1] if "." in metric_ref else metric_ref - source_name = metric_source_name(metric_ref, metric_name) # Check for custom alias first if metric_ref in aliases: alias = aliases[metric_ref] - elif metric_name_counts.get(metric_name, 1) > 1: + elif public_name_counts.get(metric_name, 1) > 1: # Collision - prefix with model name alias = f"{model_name}_{metric_name}" else: alias = metric_name register_output_name(alias, metric_ref, f"{model_name}.{metric_name}", metric_name, alias) - metric_selects[metric_ref] = ( - f"{cte_name}.{self._quote_identifier(source_name)} AS {self._quote_alias(alias)}" - ) + metric_selects[metric_ref] = f"{calculations[metric_ref]} AS {self._quote_alias(alias)}" for metric_ref in metrics: if metric_ref in leaf_refs: continue model_name, metric = self.graph.resolve_metric_reference(metric_ref) default_alias = ( - f"{model_name}_{metric.name}" if model_name and metric_name_counts[metric.name] > 1 else metric.name + f"{model_name}_{metric.name}" if model_name and public_name_counts[metric.name] > 1 else metric.name ) alias = aliases.get(metric_ref) or default_alias register_output_name(alias, metric_ref, metric.name, alias) @@ -3271,7 +3287,7 @@ def metric_source_name(metric_ref: str, metric_name: str) -> str: # Join remaining CTEs join_clauses = [] - for cte_name in cte_names[1:]: + for index, cte_name in enumerate(cte_names[1:], start=1): if not parsed_dims: # No dimensions - use CROSS JOIN (each CTE returns single row) join_clauses.append(f"CROSS JOIN {cte_name}") @@ -3279,10 +3295,14 @@ def metric_source_name(metric_ref: str, metric_name: str) -> str: # Build join condition on all dimension columns join_conditions = [] for dim_ref, gran in parsed_dims: - dim_name = dim_ref.split(".")[1] if "." in dim_ref else dim_ref - col_name = dimension_output_name(dim_ref, dim_name, gran) + full_ref = f"{dim_ref}__{gran}" if gran else dim_ref + col_name = child_aliases[full_ref] # NULL-safe equality that works for all column types - lhs = exp.Column(this=col_name, table=cte_names[0]) + # and groups introduced by any preceding source. + previous = [exp.column(col_name, table=previous_cte) for previous_cte in cte_names[:index]] + lhs = ( + previous[0] if len(previous) == 1 else exp.Coalesce(this=previous[0], expressions=previous[1:]) + ) rhs = exp.Column(this=col_name, table=cte_name) join_conditions.append(exp.NullSafeEQ(this=lhs, expression=rhs).sql(dialect=self._dialect_instance)) @@ -3302,9 +3322,8 @@ def metric_source_name(metric_ref: str, metric_name: str) -> str: if shared_filters: filter_expressions = dict(calculations) for dim_ref, gran in parsed_dims: - dim_name = dim_ref.split(".", 1)[-1] - source_name = self._quote_identifier(dimension_output_name(dim_ref, dim_name, gran)) full_ref = f"{dim_ref}__{gran}" if gran else dim_ref + source_name = self._quote_identifier(child_aliases[full_ref]) filter_expressions[full_ref] = f"COALESCE({', '.join(f'{cte}.{source_name}' for cte in cte_names)})" preagg_table_map = {} for model_name in metrics_by_model: @@ -5381,6 +5400,7 @@ def _replace_model_placeholder(expr: str) -> str: outer_select_cols = [] outer_group_cols = [] + output_names = [*entity_dim_aliases, metric.name] # Add entity_dimensions to outer SELECT/GROUP BY for alias in entity_dim_aliases: @@ -5419,6 +5439,7 @@ def _replace_model_placeholder(expr: str) -> str: inner_group_cols.append(dim_sql) outer_select_cols.append(quoted_alias) outer_group_cols.append(quoted_alias) + output_names.append(alias) # Join inner select/group after dimensions are added inner_select = ",\n ".join(inner_select_cols + inner_metric_selects) @@ -5437,18 +5458,7 @@ def _replace_model_placeholder(expr: str) -> str: group_by = f"\nGROUP BY {', '.join(outer_group_cols)}" # Order/limit/offset - order_clause = "" - if order_by: - order_fields = [] - for field in order_by: - field_name = field.split(".", 1)[1] if "." in field else field - # Handle "desc"/"asc" suffix - parts = field_name.rsplit(" ", 1) - if len(parts) == 2 and parts[1].upper() in ("ASC", "DESC"): - order_fields.append(f"{quote_alias(parts[0])} {parts[1].upper()}") - else: - order_fields.append(quote_alias(field_name)) - order_clause = f"\nORDER BY {', '.join(order_fields)}" + order_clause = self._specialized_order_clause(order_by, output_names) limit_clause = f"\nLIMIT {limit}" if limit is not None else "" offset_clause = f"\nOFFSET {offset}" if offset is not None else "" diff --git a/sidemantic/sql/query_rewriter.py b/sidemantic/sql/query_rewriter.py index 24195ba66..b04414236 100644 --- a/sidemantic/sql/query_rewriter.py +++ b/sidemantic/sql/query_rewriter.py @@ -143,6 +143,7 @@ def __init__( enforce_visibility: bool = False, use_rust_rewriter: bool | None = None, rust_no_fallback: bool | None = None, + allow_non_additive_unsafe: bool = False, ): """Initialize query rewriter. @@ -153,11 +154,18 @@ def __init__( enforce_visibility: Reject semantic references to fields declared ``public: false`` use_rust_rewriter: Override the environment-controlled Rust rewrite path rust_no_fallback: Reject unsupported Rust requirements instead of falling back + allow_non_additive_unsafe: Aggregate all snapshots without semi-additive protection """ self.graph = graph self.dialect = dialect self.use_preaggregations = use_preaggregations - self.generator = SQLGenerator(graph, dialect=dialect, enforce_visibility=enforce_visibility) + self.allow_non_additive_unsafe = allow_non_additive_unsafe + self.generator = SQLGenerator( + graph, + dialect=dialect, + enforce_visibility=enforce_visibility, + allow_non_additive_unsafe=allow_non_additive_unsafe, + ) self.enforce_visibility = enforce_visibility self._dialect_instance = self.generator._dialect_instance self._rewrite_cache: dict[tuple[object, ...], str] = {} @@ -226,6 +234,8 @@ def _route_rust(self, sql: str, user_attributes: dict | None, strict: bool) -> s return sql contextual_rust = ( user_attributes is not None + or self.use_preaggregations + or self.allow_non_additive_unsafe or self.enforce_visibility or any(model.security is not None or model.invariant_filters for model in self.graph.models.values()) ) @@ -235,17 +245,14 @@ def _route_rust(self, sql: str, user_attributes: dict | None, strict: bool) -> s self._raise_on_user_cte_name_collision(parsed) self.last_engine_selection = {"engine": "rust", "reason": "Rust engine selected"} try: - capabilities = [] - if self.use_preaggregations: - capabilities.append("query.preaggregations") - if capabilities: - raise UnsupportedSemanticFeaturesError(capabilities) rewritten = rewrite_semantic_input( self.graph, sql, input_dialect="duckdb" if self.dialect == "postgres" else self.dialect, user_attributes=user_attributes, enforce_visibility=self.enforce_visibility, + **({"use_preaggregations": True} if self.use_preaggregations else {}), + **({"allow_non_additive_unsafe": True} if self.allow_non_additive_unsafe else {}), **({"sql_dialect": self.dialect, "output_dialect": self.dialect} if self.dialect != "duckdb" else {}), ) self.last_engine_selection = {"engine": "rust", "reason": None} @@ -3130,9 +3137,9 @@ def _generate_from_plan(self, plan: SemanticQueryPlan, query: exp.Select | None ) references = [*plan.metrics, *plan.dimensions] repeated_projection = len(references) != len(set(references)) - restore_projection = query is not None and (temporal_projection or repeated_projection) + restore_projection = query is not None order_by = plan.order_by - if restore_projection and order_by: + if restore_projection and (temporal_projection or repeated_projection) and order_by: alias_references = {} for expression in query.expressions: if isinstance(expression, exp.Alias) and isinstance(expression.this, exp.Column): @@ -3156,19 +3163,40 @@ def _generate_from_plan(self, plan: SemanticQueryPlan, query: exp.Select | None user_attributes=getattr(self, "_rewrite_user_attributes", None), _query_ctes=self._query_ctes_in_scope(query), ) - # Time-comparison generators also expose their base measures for structured - # queries. A SQL SELECT keeps only the columns its projection requests. - # Repeated references likewise need separate projection aliases because - # the structured query carries only one alias per semantic reference. + # Structured queries emit dimensions first and some temporal generators + # expose extra base measures. SQL SELECT owns its output order and fields, + # including separate aliases for repeated semantic references. if restore_projection: - requested = [expression.alias_or_name for expression in query.expressions] - if all(requested) and not any(isinstance(expression, exp.Star) for expression in query.expressions): + projection_query = query + # Flattened SELECT * wrappers inherit their source's column order. + # SQLGlot scopes resolve both named CTEs and derived tables without + # confusing physical source columns with semantic projections. + from sqlglot.optimizer.scope import Scope, build_scope + + scope = build_scope(query) + while ( + scope is not None + and len(projection_query.expressions) == 1 + and isinstance(projection_query.expressions[0], exp.Star) + and len(scope.selected_sources) == 1 + ): + _, source_scope = next(iter(scope.selected_sources.values())) + if not isinstance(source_scope, Scope) or not isinstance(source_scope.expression, exp.Select): + break + scope = source_scope + projection_query = scope.expression + requested = [expression.alias_or_name for expression in projection_query.expressions] + if all(requested) and not any( + isinstance(expression, exp.Star) for expression in projection_query.expressions + ): generated = parse_fragment(generated_sql, self.dialect) if isinstance(generated, exp.Select): + if requested == [expression.alias_or_name for expression in generated.expressions]: + return generated_sql outputs = {expression.alias_or_name: expression for expression in generated.expressions} projections = [] restored_names = {} - for expression, name in zip(query.expressions, requested): + for expression, name in zip(projection_query.expressions, requested): source = expression.this if isinstance(expression, exp.Alias) else expression if not isinstance(source, exp.Column): break @@ -3184,12 +3212,31 @@ def _generate_from_plan(self, plan: SemanticQueryPlan, query: exp.Select | None output = output.this if isinstance(output, exp.Alias) else output projections.append(output.copy().as_(name)) if len(projections) == len(requested): + # GROUP BY positions refer to the generated projection, + # not the user's reordered SELECT. Bind them before + # replacing the output list. + if group := generated.args.get("group"): + for position in list(group.expressions): + if isinstance(position, exp.Literal) and position.is_int: + index = int(position.this) - 1 + if 0 <= index < len(generated.expressions): + field = generated.expressions[index] + field = field.this if isinstance(field, exp.Alias) else field + position.replace(field.copy()) generated.set("expressions", projections) if order := generated.args.get("order"): for column in order.find_all(exp.Column): if not column.table and column.name in restored_names: column.set("this", exp.to_identifier(restored_names[column.name])) - return generated.sql(dialect=self.dialect, pretty=True) + rewritten = generated.sql(dialect=self.dialect) + # Replacing the final projection can discard the SQL + # parser's attached routing/instrumentation comments. + # Missing-rollup fallback depends on this metadata. + for line in generated_sql.splitlines(): + if line.startswith("-- sidemantic:") or line == "-- used_preagg=true": + if line not in rewritten: + rewritten += "\n" + line + return rewritten return generated_sql def _dedupe(self, values: list[str]) -> list[str]: diff --git a/sidemantic/validation.py b/sidemantic/validation.py index f498ab8ed..42481b0c6 100644 --- a/sidemantic/validation.py +++ b/sidemantic/validation.py @@ -897,11 +897,19 @@ def _add_untranslated_dax_model_error(model_ref: str, model) -> None: else: _add_untranslated_dax_model_error(model_name, model) dimension = model.get_dimension(dim_name) - if not dimension: + # The generator projects local many-to-one foreign keys when + # explicitly selected, even without a Dimension declaration. + relationship_key = any( + relationship.type == "many_to_one" + and not (relationship.sql and ("{from}" in relationship.sql or "{to}" in relationship.sql)) + and dim_name in relationship.foreign_key_columns + for relationship in model.relationships + ) + if not dimension and not relationship_key: errors.append( f"Dimension '{dim_name}' not found in model '{model_name}' (referenced in '{dim_ref}')" ) - else: + elif dimension: _add_untranslated_dax_dimension_error(dim_ref, dimension) else: errors.append(f"Dimension reference '{dim_ref}' must be in 'model.dimension' format") diff --git a/tests/adapters/bsl/test_parsing.py b/tests/adapters/bsl/test_parsing.py index 12b26a01e..53435d596 100644 --- a/tests/adapters/bsl/test_parsing.py +++ b/tests/adapters/bsl/test_parsing.py @@ -1300,6 +1300,7 @@ def test_reused_join_aliases_scope_to_source_model(self): import tempfile from pathlib import Path + import duckdb import yaml from sidemantic.sql.generator import SQLGenerator @@ -1394,7 +1395,7 @@ def test_reused_join_aliases_scope_to_source_model(self): dimensions=["orders.order_id"], skip_default_time_dimensions=True, ) - assert "JOIN orders_user_cte" in orders_sql + assert "orders_user_cte" in orders_sql assert "events_user_cte" not in orders_sql assert "FROM customers" in orders_sql @@ -1403,10 +1404,22 @@ def test_reused_join_aliases_scope_to_source_model(self): dimensions=["events.event_id"], skip_default_time_dimensions=True, ) - assert "JOIN events_user_cte" in events_sql + assert "events_user_cte" in events_sql assert "orders_user_cte" not in events_sql assert "FROM accounts" in events_sql + with duckdb.connect() as conn: + conn.execute("CREATE TABLE orders (order_id INTEGER, user_id INTEGER)") + conn.execute("INSERT INTO orders VALUES (1, 10), (2, 10)") + conn.execute("CREATE TABLE customers (customer_id INTEGER, name VARCHAR)") + conn.execute("INSERT INTO customers VALUES (10, 'customer')") + conn.execute("CREATE TABLE events (event_id INTEGER, account_id INTEGER)") + conn.execute("INSERT INTO events VALUES (3, 20)") + conn.execute("CREATE TABLE accounts (account_id INTEGER, name VARCHAR)") + conn.execute("INSERT INTO accounts VALUES (20, 'account')") + assert sorted(conn.execute(orders_sql).fetchall()) == [(1, 1.0), (2, 1.0)] + assert conn.execute(events_sql).fetchall() == [(3, 1.0)] + adapter.export(graph, export_path) with open(export_path) as f: exported = yaml.safe_load(f) diff --git a/tests/adapters/cube/test_correctness_fixes.py b/tests/adapters/cube/test_correctness_fixes.py index 67a4aa7c0..6c53f4c05 100644 --- a/tests/adapters/cube/test_correctness_fixes.py +++ b/tests/adapters/cube/test_correctness_fixes.py @@ -1442,7 +1442,11 @@ def test_cross_cube_trailing_column_ref_translated_to_member(): layer.graph = graph compiled = layer.compile(metrics=["line_items.derived_x"]) assert "${orders}" not in compiled and "{'orders'" not in compiled - assert "SUM(__sidemantic_dedup." in compiled + layer.conn.execute("CREATE TABLE orders (id INTEGER, amt INTEGER)") + layer.conn.execute("INSERT INTO orders VALUES (1, 10), (2, 20)") + layer.conn.execute("CREATE TABLE line_items (order_id INTEGER)") + layer.conn.execute("INSERT INTO line_items VALUES (1), (1), (2)") + assert layer.conn.execute(compiled).fetchall() == [(60,)] def test_rollup_with_only_unmaterializable_measures_is_rejected(): diff --git a/tests/adapters/lookml/test_edge_cases.py b/tests/adapters/lookml/test_edge_cases.py index fc828e7c8..fb7011c4f 100644 --- a/tests/adapters/lookml/test_edge_cases.py +++ b/tests/adapters/lookml/test_edge_cases.py @@ -5813,8 +5813,12 @@ def test_lookml_implicit_dimension_group_compiles_against_group_column(): layer = SemanticLayer() layer.graph = graph sql = layer.compile(dimensions=["orders.created_date"], metrics=["orders.cnt"]) - assert "DATE_TRUNC('day', created)" in sql, sql - assert "'day', created_date)" not in sql, sql # never the generated field name + layer.conn.execute("CREATE TABLE orders (id INTEGER, created TIMESTAMP)") + layer.conn.execute("INSERT INTO orders VALUES (1, '2024-01-02 10:00:00'), (2, '2024-01-02 15:00:00')") + rows = layer.conn.execute(sql).fetchall() + assert len(rows) == 1 + day, count = rows[0] + assert (day.year, day.month, day.day, count) == (2024, 1, 2, 2) # Export still round-trips the group with no invented sql. out = tempfile.mktemp(suffix=".lkml") diff --git a/tests/adapters/metricflow/test_query.py b/tests/adapters/metricflow/test_query.py index a51780602..50a2cdeff 100644 --- a/tests/adapters/metricflow/test_query.py +++ b/tests/adapters/metricflow/test_query.py @@ -155,12 +155,13 @@ def test_inline_simple_metric_filter_is_applied(): def test_inline_metric_filter_qualified_in_join(): - """A filtered inline metric's columns are qualified to its owning model CTE. + """A filtered inline metric's columns bind to its owning model. Regression: the metric filter was rendered with unqualified columns. When the metric is queried with a joined dimension whose CTE also exposes a same-named column, the unqualified filter column was ambiguous and the query failed to - bind. The filter columns must be qualified with the owning model's CTE. + bind. Qualifying the columns or evaluating the filter before the join both + preserve the owning model's values. """ import tempfile import textwrap @@ -233,9 +234,8 @@ def test_inline_metric_filter_qualified_in_join(): layer.conn = conn layer.graph = graph - # The filter column is qualified to the orders CTE (not the ambiguous bare name). + # Compiling and executing the join must preserve the owning filter scope. sql = layer.compile(metrics=["completed_revenue"], dimensions=["customer.status"]) - assert "orders_cte.status" in sql assert "JOIN" in sql.upper() assert "customers" in sql.lower() diff --git a/tests/adapters/sidemantic_adapter/test_parsing.py b/tests/adapters/sidemantic_adapter/test_parsing.py index 6a2386f65..2dde1fa21 100644 --- a/tests/adapters/sidemantic_adapter/test_parsing.py +++ b/tests/adapters/sidemantic_adapter/test_parsing.py @@ -852,8 +852,16 @@ def test_parse_native_yaml_explicit_key_columns(tmp_path): layer.add_model(model) sql = layer.compile(metrics=["order_items.count"], dimensions=["shipments.carrier"]) - assert "shipments_cte.order_id = order_items_cte.order_id" in sql - assert "shipments_cte.item_id = order_items_cte.item_id" in sql + import sqlglot + from sqlglot import exp + + comparisons = { + frozenset((comparison.left.sql(), comparison.right.sql())) + for join in sqlglot.parse_one(sql).find_all(exp.Join) + for comparison in join.find_all(exp.EQ) + } + assert frozenset(("shipments_cte.order_id", "order_items_cte.order_id")) in comparisons + assert frozenset(("shipments_cte.item_id", "order_items_cte.item_id")) in comparisons def test_parse_native_yaml_resolves_model_and_metric_inheritance(tmp_path): diff --git a/tests/adapters/tmdl/test_parsing.py b/tests/adapters/tmdl/test_parsing.py index 3bc361684..56aaf2e0d 100644 --- a/tests/adapters/tmdl/test_parsing.py +++ b/tests/adapters/tmdl/test_parsing.py @@ -21,6 +21,7 @@ from sidemantic.core.relationship import Relationship from sidemantic.core.semantic_graph import SemanticGraph from sidemantic.loaders import load_from_directory +from sidemantic.semantic_handoff import UnsupportedSemanticFeaturesError from sidemantic.sql.generator import SQLGenerator from sidemantic.validation import QueryValidationError @@ -119,8 +120,12 @@ def test_tmdl_untranslated_dax_metric_is_not_compiled_as_sql(): layer = SemanticLayer() load_from_directory(layer, "tests/fixtures/tmdl") - with pytest.raises(QueryValidationError, match="DAX expression but has no SQL translation"): + with pytest.raises((QueryValidationError, UnsupportedSemanticFeaturesError)) as exc_info: layer.compile(metrics=["Sales.Sales LY"]) + if isinstance(exc_info.value, UnsupportedSemanticFeaturesError): + assert "metric.dax" in exc_info.value.capabilities + else: + assert "DAX expression but has no SQL translation" in str(exc_info.value) def test_tmdl_untranslated_dax_dimension_is_not_compiled_as_sql(): @@ -131,8 +136,12 @@ def test_tmdl_untranslated_dax_dimension_is_not_compiled_as_sql(): assert amount_x2.sql is None assert amount_x2.has_untranslated_dax - with pytest.raises(QueryValidationError, match="DAX expression but has no SQL translation"): + with pytest.raises((QueryValidationError, UnsupportedSemanticFeaturesError)) as exc_info: layer.compile(metrics=["Sales.Total Sales"], dimensions=["Sales.Amount x2"]) + if isinstance(exc_info.value, UnsupportedSemanticFeaturesError): + assert "dimension.dax" in exc_info.value.capabilities + else: + assert "DAX expression but has no SQL translation" in str(exc_info.value) with pytest.raises(ValueError, match="DAX expression but has no SQL translation"): SQLGenerator(layer.graph).generate(metrics=["Sales.Total Sales"], dimensions=["Sales.Amount x2"]) @@ -161,8 +170,12 @@ def test_tmdl_dax_only_calculated_table_is_not_compiled_as_sql(): layer = SemanticLayer() layer.graph = graph - with pytest.raises(QueryValidationError, match="DAX table expression but has no SQL/table translation"): + with pytest.raises((QueryValidationError, UnsupportedSemanticFeaturesError)) as exc_info: layer.compile(metrics=["SalesByCategory.Revenue"], dimensions=["SalesByCategory.Category"]) + if isinstance(exc_info.value, UnsupportedSemanticFeaturesError): + assert "model.dax" in exc_info.value.capabilities + else: + assert "DAX table expression but has no SQL/table translation" in str(exc_info.value) with pytest.raises(ValueError, match="DAX table expression but has no SQL/table translation"): SQLGenerator(graph).generate(metrics=["SalesByCategory.Revenue"], dimensions=["SalesByCategory.Category"]) @@ -3026,8 +3039,16 @@ def test_tmdl_keyless_many_to_many_joins_without_keying_off_endpoints(): sql = layer.compile(dimensions=["Authors.region", "Books.genre"]) assert "Authors_cte.author_id" in sql assert "Books_cte.book_author_id" in sql - assert "author_id AS author_id" in sql - assert "book_author_id AS book_author_id" in sql + layer.conn.execute("CREATE TABLE Authors (author_id VARCHAR, region VARCHAR)") + layer.conn.execute("CREATE TABLE Books (book_author_id VARCHAR, genre VARCHAR)") + layer.conn.execute("INSERT INTO Authors VALUES ('a', 'US'), ('a', 'EU')") + layer.conn.execute("INSERT INTO Books VALUES ('a', 'fiction'), ('a', 'history')") + assert set(layer.conn.execute(sql).fetchall()) == { + ("US", "fiction"), + ("US", "history"), + ("EU", "fiction"), + ("EU", "history"), + } def test_tmdl_keyless_many_to_many_metric_raises_clear_error(): @@ -3134,8 +3155,11 @@ def test_tmdl_many_to_many_on_non_primary_key_column(): assert re.search(r"ON\s+B_cte\.b_alt\s*=\s*A_cte\.a_alt", sql) or re.search( r"ON\s+A_cte\.a_alt\s*=\s*B_cte\.b_alt", sql ), sql - assert "a_alt AS a_alt" in sql - assert "b_alt AS b_alt" in sql + layer.adapter.execute("create table A(id varchar, a_alt varchar)") + layer.adapter.execute("create table B(id varchar, b_alt varchar, b_label varchar)") + layer.adapter.execute("insert into A values ('a1', 'shared'), ('a2', 'shared')") + layer.adapter.execute("insert into B values ('b1', 'shared', 'matched')") + assert layer.adapter.execute(sql).fetchall() == [("matched", 2)] def test_tmdl_one_to_one_recovers_keyless_source_key(): @@ -3186,8 +3210,12 @@ def test_tmdl_one_to_one_recovers_keyless_source_key(): layer = SemanticLayer() layer.graph = graph sql = layer.compile(metrics=["A.a_count"], dimensions=["B.b_label"]) - assert "a_key AS a_key" in sql assert "A_cte.a_key" in sql + layer.adapter.execute("create table A(a_key varchar)") + layer.adapter.execute("create table B(b_key varchar, b_label varchar)") + layer.adapter.execute("insert into A values ('joined')") + layer.adapter.execute("insert into B values ('joined', 'matched')") + assert layer.adapter.execute(sql).fetchall() == [("matched", 1)] def test_tmdl_one_to_one_alternate_key_does_not_shadow_real_key(): @@ -3252,7 +3280,11 @@ def test_tmdl_one_to_one_alternate_key_does_not_shadow_real_key(): layer.graph = graph sql = layer.compile(metrics=["Orders.cnt"], dimensions=["OrderMeta.channel"]) assert "Orders_cte.alt_key" in sql - assert "alt_key AS alt_key" in sql + layer.adapter.execute("create table Orders(order_id integer, alt_key varchar)") + layer.adapter.execute("create table OrderMeta(meta_key varchar, channel varchar)") + layer.adapter.execute("insert into Orders values (1, 'alternate')") + layer.adapter.execute("insert into OrderMeta values ('alternate', 'web')") + assert layer.adapter.execute(sql).fetchall() == [("web", 1)] def test_tmdl_ambiguous_one_side_target_not_recovered_from_endpoint(): diff --git a/tests/conftest.py b/tests/conftest.py index f66c4d4a0..8135031cf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,6 +18,21 @@ def isolate_cli_color_environment(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(typer_rich_utils, "FORCE_TERMINAL", None) +def pytest_addoption(parser): + parser.addoption( + "--test-engine", + choices=("python", "rust"), + default="python", + help="Default engine for the shared suite; explicit engine contract tests retain their overrides", + ) + + +@pytest.fixture(autouse=True) +def selected_test_engine(monkeypatch, request): + """Run shared contracts with the selected engine without enabling fallback.""" + monkeypatch.setenv("SIDEMANTIC_ENGINE", request.config.getoption("--test-engine")) + + @pytest.fixture(autouse=True) def reset_registry(): """Clear the global registry before and after each test. diff --git a/tests/core/test_consumption_compiler.py b/tests/core/test_consumption_compiler.py index 19250453e..94c379f09 100644 --- a/tests/core/test_consumption_compiler.py +++ b/tests/core/test_consumption_compiler.py @@ -102,8 +102,10 @@ def test_explore_qualifies_relative_filter_and_order_expressions(): assert "status <> 'deleted'" in sql assert "status = 'paid'" in sql - assert "status AS status" in sql assert "revenue DESC" in sql + layer.conn.execute("CREATE TABLE orders (order_id INTEGER, status VARCHAR, amount INTEGER)") + layer.conn.execute("INSERT INTO orders VALUES (1, 'paid', 10), (2, 'deleted', 20), (3, 'pending', 30)") + assert layer.conn.execute(sql).fetchall() == [(10,)] def test_explore_filter_qualification_skips_subquery_columns(): diff --git a/tests/core/test_cross_surface_equivalence.py b/tests/core/test_cross_surface_equivalence.py index c61da73f6..59c48cdce 100644 --- a/tests/core/test_cross_surface_equivalence.py +++ b/tests/core/test_cross_surface_equivalence.py @@ -8,17 +8,9 @@ Surfaces covered here (the ones that compile/rewrite SQL from the same graph): SQL-first family (semantic SQL string -> rewritten SQL): - 1. Direct ``QueryRewriter`` -- the exact construction used by - ``sidemantic rewrite`` and ``sidemantic query --dry-run`` (see - ``sidemantic/cli.py``: ``QueryRewriter(layer.graph, - dialect=layer.adapter.dialect, use_preaggregations=layer.use_preaggregations)``). - 2. ``SemanticLayer.sql()`` -- the ``sidemantic query`` execution path. It - rewrites through the identical ``QueryRewriter(self.graph, - dialect=self.dialect, use_preaggregations=self.use_preaggregations)`` call - before executing (``sidemantic/core/semantic_layer.py``). Covered as a - regression tripwire so a future divergence in that construction is caught. - 3. HTTP API ``POST /sql/compile`` -- ``sidemantic/api_server.py`` calls - ``QueryRewriter(current_layer.graph, dialect=current_layer.dialect).rewrite(query)``. + 1. Shared policy-aware transport rewrite, used by CLI rewrite and dry-run. + 2. ``SemanticLayer.sql()`` uses the same transport rewrite before execution. + 3. HTTP ``POST /sql/compile`` uses the same transport rewrite for preview. Structured family (dimensions/metrics -> compiled SQL): 4. ``SemanticLayer.compile(...)`` -- the library/CLI compile entry point. @@ -46,7 +38,7 @@ from sidemantic import Dimension, Metric, Model, Relationship, SemanticLayer from sidemantic.api_server import create_app -from sidemantic.sql.query_rewriter import QueryRewriter +from sidemantic.core.transport_security import rewrite_transport_sql # Representative semantic-SQL queries exercised across the SQL-first surfaces. # Chosen to hit: simple metric aggregation; metric + categorical dimension + @@ -199,32 +191,13 @@ def client(layer: SemanticLayer) -> TestClient: def _rewrite_cli(layer: SemanticLayer, sql: str) -> str: - """Reproduce the exact QueryRewriter construction used by the CLI. - - Mirrors ``sidemantic rewrite`` / ``sidemantic query --dry-run`` in - ``sidemantic/cli.py``. - """ - return QueryRewriter( - layer.graph, - dialect=layer.adapter.dialect, - use_preaggregations=layer.use_preaggregations, - ).rewrite(sql) + """Use the CLI's actual policy-aware, engine-aware rewrite entry point.""" + return rewrite_transport_sql(layer, sql, user_attributes=None, transport="CLI rewrite") def _rewrite_layer_sql_path(layer: SemanticLayer, sql: str) -> str: - """Reproduce the rewrite step inside ``SemanticLayer.sql()``. - - ``SemanticLayer.sql()`` executes rather than returning SQL, so it cannot be - asserted on directly. This mirrors the exact construction it uses so a future - divergence between the ``sql()`` construction and the CLI construction fails - here (regression tripwire). The end-to-end ``sql()`` execution is separately - checked for result equality in ``test_execution_results_match_across_surfaces``. - """ - return QueryRewriter( - layer.graph, - dialect=layer.dialect, - use_preaggregations=layer.use_preaggregations, - ).rewrite(sql) + """Use the rewrite entry point called by SemanticLayer.sql().""" + return rewrite_transport_sql(layer, sql, user_attributes=None, transport="SemanticLayer.sql()") def _fetch_sorted(layer: SemanticLayer, sql: str) -> list[tuple]: @@ -248,7 +221,7 @@ def test_sql_first_rewrite_is_byte_identical(name: str, layer: SemanticLayer, cl assert response.status_code == 200, response.text api_sql = response.json()["sql"] - # Direct QueryRewriter (CLI) == SemanticLayer.sql() rewrite step == HTTP /sql/compile. + # CLI transport rewrite == SemanticLayer.sql() rewrite step == HTTP /sql/compile. assert cli_sql == layer_sql_path_sql, f"{name}: CLI rewrite diverged from SemanticLayer.sql() rewrite" assert cli_sql == api_sql, f"{name}: CLI rewrite diverged from HTTP /sql/compile" diff --git a/tests/core/test_invariant_filters.py b/tests/core/test_invariant_filters.py index 00723207d..494c2108a 100644 --- a/tests/core/test_invariant_filters.py +++ b/tests/core/test_invariant_filters.py @@ -51,9 +51,11 @@ def test_structured_compile_and_query_apply_invariant_before_aggregation(): def test_semantic_sql_applies_invariant(): layer = _layer() - assert sorted(layer.sql("SELECT orders.revenue, orders.status FROM orders").fetchall()) == [ - ("new", 10), - ("paid", 20), + result = layer.sql("SELECT orders.revenue, orders.status FROM orders") + assert [column[0] for column in result.description] == ["revenue", "status"] + assert sorted(result.fetchall()) == [ + (10, "new"), + (20, "paid"), ] diff --git a/tests/core/test_runtime_defaults.py b/tests/core/test_runtime_defaults.py new file mode 100644 index 000000000..42680af6b --- /dev/null +++ b/tests/core/test_runtime_defaults.py @@ -0,0 +1,89 @@ +"""Default runtime dispatch, with the native boundary supplied by a test double.""" + +import pytest + +import sidemantic.core.semantic_layer as semantic_layer_module +import sidemantic.runtime as runtime +import sidemantic.rust_bridge as rust_bridge +from sidemantic import Metric, Model, SemanticLayer +from sidemantic.config import RuntimeConfig +from sidemantic.semantic_handoff import RustBackendUnavailableError + + +@pytest.fixture(autouse=True) +def normal_runtime_environment(monkeypatch): + monkeypatch.delenv("SIDEMANTIC_ENGINE", raising=False) + for key in list(runtime.os.environ): + if key.startswith("SIDEMANTIC_RS_"): + monkeypatch.delenv(key) + + +class Runtime: + def validate_with_semantic_input(self, *_args): + return [] + + def compile_with_semantic_input(self, *_args): + return "SELECT 42 AS count" + + +def test_default_compiles_with_rust(monkeypatch): + native = Runtime() + monkeypatch.setattr(semantic_layer_module, "get_rust_module", lambda: native) + monkeypatch.setattr(rust_bridge, "get_rust_module", lambda: native) + layer = SemanticLayer() + layer.add_model(Model(name="orders", table="orders", metrics=[Metric(name="count", agg="count")])) + assert "42" in layer.compile(metrics=["orders.count"]) + assert layer.last_engine_selection == {"engine": "rust", "reason": None} + assert RuntimeConfig().engine == "rust" + + +def test_missing_default_runtime_does_not_silently_fall_back(monkeypatch): + def unavailable(): + raise RustBackendUnavailableError("missing native package") + + monkeypatch.setattr(semantic_layer_module, "get_rust_module", unavailable) + with pytest.raises(RustBackendUnavailableError, match="missing native package"): + SemanticLayer() + assert SemanticLayer(engine="python").engine == "python" + assert SemanticLayer(engine="auto")._rust_unavailable_reason == "missing native package" + + +def test_pyodide_defaults_to_python_without_native_import(monkeypatch): + monkeypatch.setattr(runtime.sys, "platform", "emscripten") + assert RuntimeConfig().engine == "python" + assert SemanticLayer().engine == "python" + + +def test_process_override_and_explicit_selection(monkeypatch): + monkeypatch.setenv("SIDEMANTIC_ENGINE", "python") + assert SemanticLayer().engine == "python" + monkeypatch.setattr(semantic_layer_module, "get_rust_module", Runtime) + assert SemanticLayer(engine="rust").engine == "rust" + + +def test_cli_default_and_fallback_override(monkeypatch): + import sidemantic.cli as cli + + monkeypatch.setattr(cli, "_loaded_config", None) + assert cli._resolve_engine_options(None, None) == ("rust", False) + assert cli._resolve_engine_options(None, True) == ("rust", True) + assert cli._resolve_engine_options("auto", None) == ("auto", True) + monkeypatch.setattr(runtime.sys, "platform", "emscripten") + assert cli._resolve_engine_options(None, None) == ("python", False) + + +def test_default_sql_rewrite_uses_rust(monkeypatch): + class RewritingRuntime(Runtime): + def rewrite_with_semantic_input(self, *_args): + return "SELECT 42 AS count" + + def rewrite_with_semantic_input_context(self, *_args): + return "SELECT 42 AS count" + + native = RewritingRuntime() + monkeypatch.setattr(semantic_layer_module, "get_rust_module", lambda: native) + monkeypatch.setattr(rust_bridge, "get_rust_module", lambda: native) + layer = SemanticLayer() + layer.add_model(Model(name="orders", table="orders", metrics=[Metric(name="count", agg="count")])) + assert layer.sql("select count from orders").fetchall() == [(42,)] + assert layer.last_engine_selection == {"engine": "rust", "reason": None} diff --git a/tests/core/test_rust_bridge_yaml_serialization.py b/tests/core/test_rust_bridge_yaml_serialization.py index ad35f7332..e6f181ff2 100644 --- a/tests/core/test_rust_bridge_yaml_serialization.py +++ b/tests/core/test_rust_bridge_yaml_serialization.py @@ -1,5 +1,6 @@ """Regression coverage for Python->Rust YAML bridge serialization fidelity.""" +import pytest import yaml from sidemantic.core.dimension import Dimension @@ -12,6 +13,20 @@ from tests.rust_layer_adapter import _dimension_to_rust_dict, _metric_to_rust_dict, _relationship_to_rust_dict +@pytest.mark.parametrize("window,expected", [(None, "max"), ("min", "min"), ("max", "max")]) +def test_loaded_native_metric_restores_unset_non_additive_window(window, expected): + from sidemantic.rust_bridge import _graph_from_loaded_payload + + graph = _graph_from_loaded_payload( + { + "top_level_metrics": [ + {"name": "revenue", "type": "simple", "agg": "sum", "sql": "amount", "non_additive_window": window} + ] + } + ) + assert graph.metrics["revenue"].non_additive_window == expected + + def test_models_to_rust_yaml_preserves_extended_core_metadata(): model = Model( name="orders", diff --git a/tests/core/test_security_advisor_regressions.py b/tests/core/test_security_advisor_regressions.py index 27fb3bc8a..fc9a53232 100644 --- a/tests/core/test_security_advisor_regressions.py +++ b/tests/core/test_security_advisor_regressions.py @@ -7,6 +7,8 @@ """ import pytest +import sqlglot +from sqlglot import exp from sidemantic import Dimension, Metric, Model, SemanticLayer from sidemantic.core.security import SecurityPolicy, render_row_filter @@ -106,7 +108,10 @@ def test_semi_additive_month_grain_uses_last_snapshot(): assert "CASE WHEN" in normalized_sql assert " = MAX(" in normalized_sql assert " OVER (PARTITION BY " in normalized_sql - assert " ELSE NULL END" in normalized_sql + snapshot_case = sqlglot.parse_one(sql, dialect="duckdb").find(exp.Case) + assert snapshot_case is not None + # An omitted ELSE has the same SQL NULL semantics as an explicit ELSE NULL. + assert snapshot_case.args.get("default") is None or isinstance(snapshot_case.args["default"], exp.Null) # Correct: last day-of-month per account, summed = 110 + 210 = 320 (NOT naive 620). rows = layer.query(metrics=["bal.total_balance"], dimensions=["bal.day__month"]).fetchall() assert len(rows) == 1 diff --git a/tests/core/test_semantic_handoff.py b/tests/core/test_semantic_handoff.py index fa8d3cb50..dd7eca8d9 100644 --- a/tests/core/test_semantic_handoff.py +++ b/tests/core/test_semantic_handoff.py @@ -6,6 +6,7 @@ import pytest from sidemantic import Dimension, Metric, Model, PreAggregation, Relationship, SecurityPolicy +from sidemantic.core.inheritance import merge_model from sidemantic.core.semantic_graph import SemanticGraph from sidemantic.core.semantic_layer import SecurityError from sidemantic.rust_bridge import ( @@ -22,6 +23,80 @@ from sidemantic.validation import QueryValidationError +@pytest.mark.parametrize( + "overrides", + [ + {}, + {"primary_key": None, "default_grain": None, "auto_dimensions": False, "metadata": {}, "meta": None}, + {"primary_key": "alternate_id", "unique_keys": []}, + ], +) +def test_unresolved_child_snapshot_preserves_inheritance_override_semantics(overrides): + parent = Model( + name="base", + table="orders", + primary_key="id", + default_grain="month", + auto_dimensions=True, + metadata={"source": "parent"}, + meta={"label": "parent"}, + unique_keys=[["alternate_id"]], + ) + child = Model(name="child", extends="base", **overrides) + graph = SemanticGraph() + graph.add_model(parent) + graph.add_model(child) + snapshot = graph_to_semantic_input(graph)["models"][1] + assert ("primary_key" in snapshot) == ("primary_key" in overrides) + transported = Model.model_validate(snapshot) + expected = merge_model(child, parent) + actual = merge_model(transported, parent) + assert actual.primary_key_columns == expected.primary_key_columns + assert actual.model_dump(exclude={"primary_key"}) == expected.model_dump(exclude={"primary_key"}) + assert child.model_fields_set == {"name", "extends", *overrides} + + +@pytest.mark.parametrize("kind", ["one_to_one", "one_to_many", "many_to_one"]) +@pytest.mark.parametrize("explicit", [None, "explicit_key"]) +def test_snapshot_preserves_tmdl_local_endpoint_without_changing_model_key(kind, explicit): + graph = SemanticGraph() + relationship = Relationship(name="metadata", type=kind, foreign_key="meta_key", primary_key=explicit) + relationship._tmdl_from_column = "alt_key" + graph.add_model(Model(name="orders", table="orders", primary_key="id", relationships=[relationship])) + payload = graph_to_semantic_input(graph) + expected = explicit or ("alt_key" if kind in ("one_to_one", "one_to_many") else None) + assert payload["models"][0]["relationships"][0].get("primary_key") == expected + assert payload["models"][0]["primary_key"] == ["id"] + assert relationship.primary_key == explicit + + +@pytest.mark.parametrize( + "definition", + [ + {"type": "conversion", "entity": "id", "base_event": "signup", "conversion_event": "purchase"}, + {"type": "time_comparison", "base_metric": "events.count"}, + ], +) +def test_snapshot_does_not_duplicate_automatically_indexed_model_metrics(definition): + graph = SemanticGraph() + metric = Metric(name="special", **definition) + graph.add_model(Model(name="events", table="events", metrics=[metric])) + payload = graph_to_semantic_input(graph) + assert payload["metrics"] == [] + assert [item["name"] for item in payload["models"][0]["metrics"]] == ["special"] + assert graph.metrics["special"] is metric + + +def test_snapshot_keeps_explicit_graph_metric_even_when_model_uses_same_object(): + graph = SemanticGraph() + metric = Metric(name="revenue", agg="sum", sql="amount") + graph.add_model(Model(name="orders", table="orders", metrics=[metric])) + graph.add_metric(metric, model_name="orders") + payload = graph_to_semantic_input(graph) + assert [item["name"] for item in payload["metrics"]] == ["revenue"] + assert payload["metric_owners"] == {"revenue": "orders"} + + @pytest.fixture def source_graph(): graph = SemanticGraph() diff --git a/tests/metrics/test_filters.py b/tests/metrics/test_filters.py index 273006aa9..b35a62fb2 100644 --- a/tests/metrics/test_filters.py +++ b/tests/metrics/test_filters.py @@ -521,7 +521,7 @@ def test_structured_filters_resolve_grained_and_computed_dimensions_before_where where_sql = postgres_sql.split("WHERE", 1)[1] assert "created_at__month" not in where_sql assert "events.gross" not in where_sql - assert "DATE_TRUNC('MONTH', occurred_at)" in where_sql + assert "DATE_TRUNC('MONTH', OCCURRED_AT)" in where_sql.upper() assert "(unit_price * quantity) >= 20" in where_sql assert df_rows( diff --git a/tests/metrics/test_non_additive_guard.py b/tests/metrics/test_non_additive_guard.py index d24c85f57..03f1b6877 100644 --- a/tests/metrics/test_non_additive_guard.py +++ b/tests/metrics/test_non_additive_guard.py @@ -18,6 +18,7 @@ from sidemantic import Dimension, Metric, Model, Relationship, SemanticLayer from sidemantic.adapters.sidemantic import SidemanticAdapter from sidemantic.core.semantic_layer import UnsupportedMetricError +from sidemantic.semantic_handoff import UnsupportedSemanticFeaturesError from sidemantic.sql.generator import SQLGenerator @@ -92,7 +93,7 @@ def test_semi_additive_value_is_last_snapshot(): # Semi-additive: sum of the last snapshot per account (global last-date window # collapses to the single latest snapshot when no other grouping is requested). sql = layer.compile(metrics=["accounts.balance"]) - assert "__sidemantic_snapshot_field" in sql + assert "MAX(" in sql assert "OVER (" in sql # Grouped by account: last balance per account, summed -> 150 + 70 + 33 = 253. @@ -242,11 +243,15 @@ def test_semi_additive_plus_fanout_symmetric_aggregate_raises(): relationships=[Relationship(name="accounts", type="many_to_one", foreign_key="account_id")], ) ) - with pytest.raises(UnsupportedMetricError) as exc: + expected_error = UnsupportedSemanticFeaturesError if layer.engine == "rust" else UnsupportedMetricError + with pytest.raises(expected_error) as exc: layer.compile(metrics=["accounts.balance", "transactions.amount"], dimensions=["accounts.region"]) - msg = str(exc.value).lower() - assert "symmetric" in msg or "fan-out" in msg - assert "compose" in msg + if layer.engine == "rust": + assert exc.value.capabilities == ["metric.non_additive_metric_shape"] + else: + msg = str(exc.value).lower() + assert "symmetric" in msg or "fan-out" in msg + assert "compose" in msg def test_adapter_round_trips_non_additive_dimension(): @@ -362,7 +367,7 @@ def test_graph_metric_wrapping_semi_additive_measure_is_planned(): ) layer.add_metric(Metric(name="wrapped_balance", sql="bal.total_balance")) sql = layer.compile(metrics=["wrapped_balance"], dimensions=["bal.account"]) - assert "__sidemantic_snapshot_field" in sql + assert "MAX(" in sql assert dict(layer.query(metrics=["wrapped_balance"], dimensions=["bal.account"]).fetchall()) == {"A": 110, "B": 210} diff --git a/tests/optimizations/test_pre_aggregations.py b/tests/optimizations/test_pre_aggregations.py index 06d3a6d81..c2564a7dd 100644 --- a/tests/optimizations/test_pre_aggregations.py +++ b/tests/optimizations/test_pre_aggregations.py @@ -4,6 +4,8 @@ import duckdb import pytest +import sqlglot +from sqlglot import exp from sidemantic import Dimension, Metric, Model from sidemantic.core.pre_aggregation import Index, PreAggregation, RefreshKey, RefreshResult @@ -1208,7 +1210,10 @@ def test_avg_preaggregation_rolls_up_with_sum_count_state(layer): preagg_rows = layer.adapter.execute(preagg_sql).fetchall() assert "products_preagg_by_category" in preagg_sql - assert "SUM(avg_price_raw) / NULLIF(SUM(count_raw), 0)" in preagg_sql + expression = sqlglot.parse_one(preagg_sql) + assert expression.find(exp.Div) is not None + assert expression.find(exp.Nullif) is not None + assert {aggregate.this.name for aggregate in expression.find_all(exp.Sum)} == {"avg_price_raw", "count_raw"} assert preagg_rows == baseline_rows @@ -1296,7 +1301,11 @@ def test_ratio_metric_preaggregation_rebuilds_from_additive_leaves(layer): preagg_rows = layer.adapter.execute(preagg_sql).fetchall() assert "orders_preagg_by_status" in preagg_sql - assert "SUM(revenue_raw) / NULLIF(COALESCE(SUM(count_raw), 0), 0)" in preagg_sql + expression = sqlglot.parse_one(preagg_sql) + assert expression.find(exp.Div) is not None + assert expression.find(exp.Nullif) is not None + assert expression.find(exp.Coalesce) is not None + assert {aggregate.this.name for aggregate in expression.find_all(exp.Sum)} == {"revenue_raw", "count_raw"} assert preagg_rows == baseline_rows @@ -1426,7 +1435,8 @@ def test_lambda_preaggregation_unions_with_granularity_rollup(layer): preagg_rows = layer.adapter.execute(preagg_sql).fetchall() assert "UNION ALL" in preagg_sql - assert "DATE_TRUNC('MONTH', created_at_day)" in preagg_sql + month_bucket = sqlglot.parse_one("DATE_TRUNC('month', created_at_day)", read="duckdb") + assert any(expression == month_bucket for expression in sqlglot.parse_one(preagg_sql, read="duckdb").walk()) assert preagg_rows == baseline_rows @@ -1511,7 +1521,9 @@ def test_derived_metric_preaggregation_rebuilds_from_additive_leaves(layer): preagg_rows = layer.adapter.execute(preagg_sql).fetchall() assert "orders_preagg_by_status" in preagg_sql - assert "SUM(revenue_raw) - SUM(discounts_raw)" in preagg_sql + expression = sqlglot.parse_one(preagg_sql) + assert expression.find(exp.Sub) is not None + assert {aggregate.this.name for aggregate in expression.find_all(exp.Sum)} == {"revenue_raw", "discounts_raw"} assert preagg_rows == baseline_rows @@ -2148,16 +2160,23 @@ def test_preagg_strict_raises_when_table_missing(): ) -def test_sql_path_falls_back_to_raw_when_rollup_missing(): +@pytest.mark.parametrize( + "projection,columns,rows", + [ + ("orders.revenue, orders.status", ["revenue", "status"], {(120, 0), (90, 1)}), + ("orders.status, orders.revenue", ["status", "revenue"], {(0, 120), (1, 90)}), + ], +) +def test_sql_path_falls_back_to_raw_when_rollup_missing(projection, columns, rows): """layer.sql() (the SQL/CLI path) also falls back to raw when the rollup table is missing.""" layer = _layer_with_unbuilt_rollup() layer.use_preaggregations = True - result = layer.sql("SELECT orders.revenue, orders.status FROM orders") + result = layer.sql(f"SELECT {projection} FROM orders") # The rollup table is absent, so rows come back only if it fell back to raw. - # The rewriter orders columns dimensions-first, so each row is (status, revenue). - assert set(result.fetchall()) == {(0, 120), (1, 90)} + assert [column[0] for column in result.description] == columns + assert set(result.fetchall()) == rows def test_sql_path_strict_raises_when_rollup_missing(): diff --git a/tests/optimizations/test_predicate_pushdown.py b/tests/optimizations/test_predicate_pushdown.py index d6e144f67..135d8f3f0 100644 --- a/tests/optimizations/test_predicate_pushdown.py +++ b/tests/optimizations/test_predicate_pushdown.py @@ -740,6 +740,18 @@ def test_mixed_metric_and_window_dim_filter_pushed_into_model_subquery_in_preagg layer.add_model(order_items) # Filter references BOTH a window dim (next_status) and a metric (revenue) + layer.conn.execute(""" + create table orders_table ( + order_id integer, customer_id integer, status varchar, + created_at timestamp, order_date date, amount double + ); + insert into orders_table values + (1, 1, 'pending', '2025-01-01 09:00:00', '2025-01-01', 50), + (2, 1, 'complete', '2025-01-01 10:00:00', '2025-01-01', 200), + (3, 2, 'pending', '2025-01-01 11:00:00', '2025-01-01', 5); + create table order_items_table (item_id integer, order_id integer, qty integer); + insert into order_items_table values (1, 1, 1), (2, 2, 2), (3, 3, 3); + """) sql = layer.compile( metrics=["orders.revenue", "order_items.quantity"], dimensions=["orders.order_date"], @@ -759,3 +771,12 @@ def test_mixed_metric_and_window_dim_filter_pushed_into_model_subquery_in_preagg # The outer query should NOT have the window dim filter outer_query = sql[sql.rindex("SELECT") :] assert "next_status" not in outer_query, "Window dim filter should NOT be in outer WHERE" + + # Order 1 qualifies through its next status; order 2 qualifies through its + # own amount. Order 3 is excluded from revenue, while all item rows still + # contribute to the independent quantity population. + cursor = layer.conn.execute(sql) + assert [column[0] for column in cursor.description] == ["order_date", "revenue", "quantity"] + rows = cursor.fetchall() + assert len(rows) == 1 + assert rows[0][1:] == (250.0, 6) diff --git a/tests/queries/test_basic.py b/tests/queries/test_basic.py index 0dd26cb8f..cb6a5675d 100644 --- a/tests/queries/test_basic.py +++ b/tests/queries/test_basic.py @@ -511,7 +511,6 @@ def test_custom_join_sql_projects_extra_predicate_columns(): ) sql = layer.compile(metrics=["orders.revenue"], dimensions=["customers.country"], order_by=["customers.country"]) - assert "valid_to AS valid_to" in sql assert "customers_cte.valid_to IS NULL" in sql rows = df_rows( @@ -600,7 +599,6 @@ def test_dotted_graph_metric_projects_sql_column_and_orders_by_alias(layer): order_by=["events.p95.latency DESC"], ) - assert "latency AS latency" in sql assert "ORDER BY" in sql assert '"events.p95.latency" DESC' in sql diff --git a/tests/queries/test_non_additive_option_transport.py b/tests/queries/test_non_additive_option_transport.py new file mode 100644 index 000000000..f12e1f312 --- /dev/null +++ b/tests/queries/test_non_additive_option_transport.py @@ -0,0 +1,78 @@ +"""Explicit snapshot escape-hatch preferences reach every compilation entrypoint.""" + +import json + +import pytest + +from sidemantic import Dimension, Metric, Model, SemanticLayer +from sidemantic.rust_bridge import rewrite_semantic_input, validate_semantic_input + + +@pytest.fixture +def layer(monkeypatch): + monkeypatch.setattr("sidemantic.core.semantic_layer.get_rust_module", lambda: object()) + result = SemanticLayer(engine="rust", fallback=False, allow_non_additive_unsafe=True, auto_register=False) + result.add_model( + Model( + name="accounts", + table="accounts", + dimensions=[Dimension(name="day", type="time", granularity="day")], + metrics=[Metric(name="balance", agg="sum", sql="amount", non_additive_dimension="day")], + ) + ) + yield result + result.adapter.close() + + +@pytest.mark.parametrize("method", ["compile", "query", "sql", "explain_sql"]) +def test_layer_passes_live_snapshot_preference(monkeypatch, layer, method): + calls = [] + + def compile(graph, query, **kwargs): + calls.append(query.get("allow_non_additive_unsafe", False)) + return "SELECT 42 AS balance" + + def rewrite(graph, query, **kwargs): + calls.append(kwargs.get("allow_non_additive_unsafe", False)) + return "SELECT 42 AS balance" + + monkeypatch.setattr("sidemantic.rust_bridge.compile_semantic_input", compile) + monkeypatch.setattr("sidemantic.rust_bridge.validate_semantic_input", lambda *args, **kwargs: []) + monkeypatch.setattr("sidemantic.sql.query_rewriter.rewrite_semantic_input", rewrite) + for enabled in [True, False]: + layer.allow_non_additive_unsafe = enabled + if method in {"compile", "query"}: + result = getattr(layer, method)(metrics=["accounts.balance"]) + else: + result = getattr(layer, method)("SELECT balance FROM accounts") + if method in {"query", "sql"}: + assert result.fetchall() == [(42,)] + assert calls[-1] is enabled + assert calls == [True, False] + + +@pytest.mark.parametrize("method", ["validate", "rewrite"]) +def test_bridge_serializes_snapshot_preference(layer, method): + calls = [] + + class Extension: + def validate_with_semantic_input(self, source, query): + calls.append(json.loads(query)) + return [] + + def rewrite_with_semantic_input_context_diagnostics(self, source, sql, context): + calls.append(json.loads(context)) + return json.dumps({"sql": "SELECT 42", "warnings": []}) + + if method == "validate": + validate_semantic_input( + layer.graph, ["accounts.balance"], [], allow_non_additive_unsafe=True, rust_module=Extension() + ) + else: + rewrite_semantic_input( + layer.graph, + "SELECT balance FROM accounts", + allow_non_additive_unsafe=True, + rust_module=Extension(), + ) + assert calls[0]["allow_non_additive_unsafe"] is True diff --git a/tests/queries/test_post_process_comments.py b/tests/queries/test_post_process_comments.py new file mode 100644 index 000000000..fb5f3a17d --- /dev/null +++ b/tests/queries/test_post_process_comments.py @@ -0,0 +1,26 @@ +"""Embedding compiled SQL must terminate trailing line comments.""" + +import pytest + +from sidemantic import SemanticLayer + + +@pytest.mark.parametrize( + "suffix", + [ + "\n-- used_preagg=true", + "\n-- used_preagg=true\n-- sidemantic: models=orders", + " -- source comment", + "\r\n-- source comment", + ], +) +def test_post_process_preserves_query_before_trailing_comment(suffix): + layer = SemanticLayer(auto_register=False) + try: + sql = layer._apply_post_process( + "SELECT '-- literal text' AS label, 41 AS value" + suffix, + "SELECT label, value + 1 AS result FROM ({inner}) AS scoped", + ) + assert layer.adapter.execute(sql).fetchall() == [("-- literal text", 42)] + finally: + layer.adapter.close() diff --git a/tests/queries/test_relationship_key_validation.py b/tests/queries/test_relationship_key_validation.py new file mode 100644 index 000000000..f1bdb8350 --- /dev/null +++ b/tests/queries/test_relationship_key_validation.py @@ -0,0 +1,34 @@ +"""Declared local relationship keys remain usable as grouping fields.""" + +import pytest + +from sidemantic import Metric, Model, Relationship, SemanticLayer +from sidemantic.validation import QueryValidationError + + +@pytest.mark.parametrize("engine", ["python", "rust"]) +def test_declared_foreign_key_groups_without_dimension(engine): + if engine == "rust": + pytest.importorskip("sidemantic_rs") + layer = SemanticLayer(engine=engine, fallback=False, auto_register=False) + layer.add_model(Model(name="customers", table="customers", primary_key="id")) + layer.add_model( + Model( + name="orders", + table="orders", + primary_key="id", + metrics=[Metric(name="revenue", agg="sum", sql="amount")], + relationships=[Relationship(name="customers", type="many_to_one", foreign_key="customer_id")], + ) + ) + try: + layer.adapter.execute("create table orders(id integer, customer_id integer, amount integer)") + layer.adapter.execute("insert into orders values (1, 10, 2), (2, 10, 3), (3, 20, 7)") + sql = layer.compile( + metrics=["orders.revenue"], dimensions=["orders.customer_id"], order_by=["orders.customer_id"] + ) + assert layer.adapter.execute(sql).fetchall() == [(10, 5), (20, 7)] + with pytest.raises(QueryValidationError, match="unknown_key"): + layer.compile(metrics=["orders.revenue"], dimensions=["orders.unknown_key"]) + finally: + layer.adapter.close() diff --git a/tests/queries/test_rust_rewrite_preaggregation_options.py b/tests/queries/test_rust_rewrite_preaggregation_options.py new file mode 100644 index 000000000..310916b0a --- /dev/null +++ b/tests/queries/test_rust_rewrite_preaggregation_options.py @@ -0,0 +1,62 @@ +"""Preaggregation preferences cross the SQL rewrite boundary without fallback.""" + +import json + +import pytest + +from sidemantic import Dimension, Metric, Model +from sidemantic.core.semantic_graph import SemanticGraph +from sidemantic.rust_bridge import rewrite_semantic_input +from sidemantic.sql.query_rewriter import QueryRewriter + + +def graph(): + result = SemanticGraph() + result.add_model( + Model( + name="orders", + table="raw_orders", + dimensions=[Dimension(name="status", type="categorical")], + metrics=[Metric(name="revenue", agg="sum", sql="amount")], + ) + ) + return result + + +@pytest.mark.parametrize("method", ["rewrite", "explain"]) +@pytest.mark.parametrize( + "sql", + [ + "SELECT revenue FROM orders", + "WITH orders_cte AS (SELECT revenue FROM orders) SELECT * FROM orders_cte", + ], +) +def test_rollup_preference_reaches_native_compiler(monkeypatch, method, sql): + calls = [] + + def rewrite(graph, sql, **options): + calls.append(options) + return "SELECT 42 AS revenue" + + monkeypatch.setattr("sidemantic.sql.query_rewriter.rewrite_semantic_input", rewrite) + rewriter = QueryRewriter(graph(), use_rust_rewriter=True, rust_no_fallback=True, use_preaggregations=True) + result = getattr(rewriter, method)(sql) + assert (result if method == "rewrite" else result.rewritten_sql) == "SELECT 42 AS revenue" + assert calls[0]["use_preaggregations"] is True + assert rewriter.last_engine_selection == {"engine": "rust", "reason": None} + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_bridge_encodes_rollup_preference_in_context(enabled): + calls = [] + + class Extension: + def rewrite_with_semantic_input_context_diagnostics(self, source, sql, context): + calls.append(json.loads(context)) + return json.dumps({"sql": "SELECT 42 AS revenue", "warnings": []}) + + result = rewrite_semantic_input( + graph(), "SELECT revenue FROM orders", use_preaggregations=enabled, rust_module=Extension() + ) + assert result == "SELECT 42 AS revenue" + assert calls[0].get("use_preaggregations", False) is enabled diff --git a/tests/queries/test_semantic_sql_planner.py b/tests/queries/test_semantic_sql_planner.py index f63870dd0..9dfb673b5 100644 --- a/tests/queries/test_semantic_sql_planner.py +++ b/tests/queries/test_semantic_sql_planner.py @@ -1,4 +1,9 @@ -"""Tests for semantic SQL rewrite planning and explanations.""" +"""Python QueryRewriter optimization rules, candidates, and plan explanations. + +These implementation tests compare optimized plans with the Python rewriter's +unoptimized baseline. Shared SQL execution contracts live in test_sql_rewriter +and semantic_conformance and run against both engines. +""" import pytest @@ -14,7 +19,7 @@ @pytest.fixture def semantic_layer(): - layer = SemanticLayer(auto_register=False) + layer = SemanticLayer(auto_register=False, engine="python") orders = Model( name="orders", @@ -2278,8 +2283,11 @@ def test_wrapped_fanout_preserves_aliases_and_executes(semantic_layer): "orders.revenue": "total_revenue", "customers.count": "customer_count", } - assert "orders_preagg.total_revenue AS total_revenue" in explanation.rewritten_sql - assert "customers_preagg.customer_count AS customer_count" in explanation.rewritten_sql + assert fetch_columns(semantic_layer.adapter.execute(explanation.rewritten_sql)) == [ + "total_revenue", + "customer_count", + ] + assert fetch_rows(semantic_layer.adapter.execute(explanation.rewritten_sql)) == [(450, 2)] def test_wrapped_fanout_uses_child_preaggregations(semantic_layer): diff --git a/tests/queries/test_sql_projection_order.py b/tests/queries/test_sql_projection_order.py new file mode 100644 index 000000000..f781c4844 --- /dev/null +++ b/tests/queries/test_sql_projection_order.py @@ -0,0 +1,35 @@ +"""SQL projections retain their requested order across raw and rollup plans.""" + +import pytest + +from sidemantic import Dimension, Metric, Model, PreAggregation, SemanticLayer + + +@pytest.mark.parametrize("engine", ["python", "rust"]) +@pytest.mark.parametrize("rollup", [False, True]) +@pytest.mark.parametrize( + "wrapper", ["{query}", "SELECT * FROM ({query}) AS totals", "WITH totals AS ({query}) SELECT * FROM totals"] +) +def test_metric_first_projection_keeps_aliases_grouping_and_order(engine, rollup, wrapper): + layer = SemanticLayer(engine=engine, fallback=False, auto_register=False, use_preaggregations=rollup) + try: + layer.adapter.execute("CREATE TABLE orders(status VARCHAR, amount INTEGER)") + layer.adapter.execute("INSERT INTO orders VALUES ('b', 2), ('a', 3), ('a', 4)") + layer.add_model( + Model( + name="orders", + table="orders", + dimensions=[Dimension(name="status", type="categorical")], + metrics=[Metric(name="revenue", agg="sum", sql="amount")], + pre_aggregations=[PreAggregation(name="status", dimensions=["status"], measures=["revenue"])], + ) + ) + layer.adapter.execute( + "CREATE TABLE orders_preagg_status AS SELECT status, SUM(amount) AS revenue_raw FROM orders GROUP BY status" + ) + query = wrapper.format(query='SELECT revenue AS "Total", status AS "Status" FROM orders') + ' ORDER BY "Status"' + result = layer.sql(query) + assert [column[0] for column in result.description] == ["Total", "Status"] + assert result.fetchall() == [(7, "a"), (2, "b")] + finally: + layer.adapter.close() diff --git a/tests/queries/test_sql_rewriter.py b/tests/queries/test_sql_rewriter.py index 2bd2ac005..520fd0fb5 100644 --- a/tests/queries/test_sql_rewriter.py +++ b/tests/queries/test_sql_rewriter.py @@ -1,6 +1,7 @@ """Tests for SQL query rewriter.""" import pytest +import sqlglot from sidemantic.core.dimension import Dimension from sidemantic.core.metric import Metric @@ -185,8 +186,9 @@ def test_zero_limit_and_offset_are_preserved(semantic_layer): sql = "SELECT orders.revenue, orders.status FROM orders ORDER BY orders.status LIMIT 0 OFFSET 0" rewritten = QueryRewriter(semantic_layer.graph).rewrite(sql) - assert "\nLIMIT 0" in rewritten - assert "\nOFFSET 0" in rewritten + parsed = sqlglot.parse_one(rewritten) + assert parsed.args["limit"].expression.this == "0" + assert parsed.args["offset"].expression.this == "0" result = semantic_layer.sql(sql) rows = _rows(result) @@ -1478,8 +1480,9 @@ def test_postprocess_zero_limit_and_offset_in_outer(semantic_layer): """ rewritten = QueryRewriter(semantic_layer.graph).rewrite(sql) - assert "\nLIMIT 0" in rewritten - assert "\nOFFSET 0" in rewritten + parsed = sqlglot.parse_one(rewritten) + assert parsed.args["limit"].expression.this == "0" + assert parsed.args["offset"].expression.this == "0" result = semantic_layer.sql(sql) rows = _rows(result) diff --git a/tests/queries/test_ungrouped_queries.py b/tests/queries/test_ungrouped_queries.py index d4f512f19..25bb9b640 100644 --- a/tests/queries/test_ungrouped_queries.py +++ b/tests/queries/test_ungrouped_queries.py @@ -5,6 +5,7 @@ import pytest from sidemantic import Dimension, Metric, Model, SemanticLayer +from sidemantic.semantic_handoff import UnsupportedSemanticFeaturesError from tests.utils import fetch_dicts @@ -360,7 +361,7 @@ def test_with_totals_ignores_configured_default_limit(): def test_with_totals_unsupported_window_path_raises(layer): - """with_totals on a window-function (cumulative) metric raises NotImplementedError.""" + """with_totals on a window-function metric explicitly rejects the unsupported path.""" orders = Model( name="orders", table="orders", @@ -371,5 +372,9 @@ def test_with_totals_unsupported_window_path_raises(layer): layer.add_model(orders) layer.graph.add_metric(Metric(name="cumulative_revenue", type="cumulative", sql="orders.revenue")) - with pytest.raises(NotImplementedError, match="with_totals is not yet supported"): + with pytest.raises((NotImplementedError, UnsupportedSemanticFeaturesError)) as exc_info: layer.compile(metrics=["cumulative_revenue"], dimensions=["orders.order_date"], with_totals=True) + if isinstance(exc_info.value, UnsupportedSemanticFeaturesError): + assert exc_info.value.capabilities == ["query.totals.window"] + else: + assert "with_totals is not yet supported" in str(exc_info.value) diff --git a/tests/queries/test_yardstick_query_rewriter.py b/tests/queries/test_yardstick_query_rewriter.py index 6e7986cf9..a352d8b8d 100644 --- a/tests/queries/test_yardstick_query_rewriter.py +++ b/tests/queries/test_yardstick_query_rewriter.py @@ -413,13 +413,14 @@ def test_yardstick_plain_measure_reference_with_where_context(yardstick_layer): ] -def test_yardstick_curly_measure_reference_without_semantic_prefix(yardstick_layer): +@pytest.mark.parametrize("measure", ["{revenue}", "{sales_v.revenue}"]) +def test_yardstick_curly_measure_reference_without_semantic_prefix(yardstick_layer, measure): rows = fetch_dicts( yardstick_layer.sql( - """ + f""" SELECT year, - {revenue} AS revenue + {measure} AS revenue FROM sales_v WHERE region = 'US' GROUP BY year @@ -457,10 +458,11 @@ def test_yardstick_mixed_non_semantic_at_routing(yardstick_layer): ] -def test_yardstick_listing8_rollup_parity(yardstick_paper_layer): +@pytest.mark.parametrize("grouping", ["ROLLUP(o.prodName)", "CUBE(o.prodName)", "GROUPING SETS ((o.prodName), ())"]) +def test_yardstick_listing8_rollup_parity(yardstick_paper_layer, grouping): rows = fetch_dicts( yardstick_paper_layer.sql( - """ + f""" SELECT o.prodName, COUNT(*) AS c, @@ -469,7 +471,7 @@ def test_yardstick_listing8_rollup_parity(yardstick_paper_layer): o.sumRevenue AS r FROM paper_orders_v o WHERE o.custName <> 'Var Bob' -GROUP BY ROLLUP(o.prodName) +GROUP BY {grouping} ORDER BY o.prodName """ ) @@ -1903,6 +1905,8 @@ def test_yardstick_ordered_set_aggregates(tmp_path): category, PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY value) AS MEASURE p50, PERCENTILE_DISC(0.5) WITHIN GROUP (ORDER BY value) AS MEASURE p50d, + PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY value DESC) FILTER (WHERE value > 1) AS MEASURE p25_desc, + PERCENTILE_DISC(0.25) WITHIN GROUP (ORDER BY value DESC) FILTER (WHERE value > 1) AS MEASURE p25d_desc, QUANTILE_CONT(value, 0.5) AS MEASURE q50, QUANTILE_DISC(value, 0.5) AS MEASURE q50d, MODE(value) AS MEASURE mode_value @@ -1927,6 +1931,12 @@ def test_yardstick_ordered_set_aggregates(tmp_path): p50d = fetch_dicts(layer.sql("SEMANTIC SELECT category, AGGREGATE(p50d) AS p50d FROM ordered_set_v")) assert {(row["category"], int(row["p50d"])) for row in p50d} == {("A", 2), ("B", 10)} + p25_desc = fetch_dicts(layer.sql("SEMANTIC SELECT category, AGGREGATE(p25_desc) AS p25_desc FROM ordered_set_v")) + assert {(row["category"], float(row["p25_desc"])) for row in p25_desc} == {("A", 3.5), ("B", 15.0)} + + p25d_desc = fetch_dicts(layer.sql("SEMANTIC SELECT category, AGGREGATE(p25d_desc) AS p25d_desc FROM ordered_set_v")) + assert {(row["category"], int(row["p25d_desc"])) for row in p25d_desc} == {("A", 4), ("B", 20)} + q50 = fetch_dicts(layer.sql("SEMANTIC SELECT category, AGGREGATE(q50) AS q50 FROM ordered_set_v")) assert {(row["category"], float(row["q50"])) for row in q50} == {("A", 2.0), ("B", 10.0)} diff --git a/tests/semantic_conformance/test_absent_counts.py b/tests/semantic_conformance/test_absent_counts.py index 6e4c29615..34d0c6048 100644 --- a/tests/semantic_conformance/test_absent_counts.py +++ b/tests/semantic_conformance/test_absent_counts.py @@ -105,9 +105,7 @@ def test_count_zero_flows_into_derived_and_ratio_before_outer_defaults(layer): assert cursor.fetchall() == [("empty", 1, 0, -2), ("matched", 3, 2, 0.5)] -def test_native_source_populations_preserve_orphans_with_zero_customer_count(layer): - if layer.engine != "rust": - pytest.skip("Native independent sources retain orphan groups omitted by Python's dimension-first joins") +def test_source_populations_preserve_orphans_with_zero_customer_count(layer): cursor = layer.adapter.execute( layer.compile( metrics=["orders.rows", "orders.people", "customers.customers", "combined", "fraction", "reverse_fraction"], @@ -134,6 +132,31 @@ def test_policy_does_not_restore_unauthorized_groups(layer): assert cursor.fetchall() == [("empty", 0, 1)] +def test_three_source_groups_merge_when_the_first_source_has_no_row(layer): + layer.add_model( + Model( + name="refunds", + table="count_refunds", + primary_key="id", + metrics=[Metric(name="refunds", agg="count")], + relationships=[Relationship(name="customers", type="many_to_one", foreign_key="customer_id")], + ) + ) + layer.adapter.execute( + "create table count_refunds(id integer, customer_id integer); insert into count_refunds values (1, 2)" + ) + cursor = layer.adapter.execute( + layer.compile( + metrics=["orders.rows", "customers.customers", "refunds.refunds"], + dimensions=["customers.region"], + order_by=["customers.region"], + ) + ) + # The empty group exists in the second and third sources, but not the first. + # Joining every later source only against the first would split it into two rows. + assert cursor.fetchall() == [("empty", 0, 1, 1), ("matched", 2, 1, 0), (None, 1, 0, 0)] + + def test_absent_count_zero_is_used_by_aggregate_filter_and_order(layer): cursor = layer.adapter.execute( layer.compile( diff --git a/tests/semantic_conformance/test_aggregate_aliases.py b/tests/semantic_conformance/test_aggregate_aliases.py index 6bfafaa28..3a69ec842 100644 --- a/tests/semantic_conformance/test_aggregate_aliases.py +++ b/tests/semantic_conformance/test_aggregate_aliases.py @@ -6,10 +6,11 @@ from sidemantic.semantic_handoff import UnsupportedSemanticFeaturesError -@pytest.fixture -def layer(): - pytest.importorskip("sidemantic_rs", reason="Alias acceptance requires the real Rust extension") - layer = SemanticLayer(engine="rust", auto_register=False) +@pytest.fixture(params=["python", "rust"]) +def layer(request): + if request.param == "rust": + pytest.importorskip("sidemantic_rs", reason="Alias acceptance requires the real Rust extension") + layer = SemanticLayer(engine=request.param, auto_register=False) layer.add_model( Model( name="customers", @@ -51,7 +52,7 @@ def layer(): def assert_result(layer, query, columns, expected): sql = layer.compile(**query) - assert layer.last_engine_selection["engine"] == "rust" + assert layer.last_engine_selection["engine"] == layer.engine result = layer.adapter.execute(sql) assert [column[0] for column in result.description] == columns assert result.fetchall() == expected diff --git a/tests/semantic_conformance/test_aggregate_breadth_parity.py b/tests/semantic_conformance/test_aggregate_breadth_parity.py index b535176cb..667dec1d3 100644 --- a/tests/semantic_conformance/test_aggregate_breadth_parity.py +++ b/tests/semantic_conformance/test_aggregate_breadth_parity.py @@ -80,6 +80,19 @@ def test_complete_sql_and_wrappers_run_after_entity_dedup(layer): assert layer.adapter.execute(sql).fetchall() == [("all", 30.0, 3, 40.0, 60.0, 30.0)] +def test_complete_aggregates_keep_groups_without_source_entities(layer): + layer.adapter.execute("insert into raw_items values (5, 99, 'orphan'), (6, 99, 'orphan')") + cursor = layer.query( + metrics=["orders.opaque_count", "orders.average"], + dimensions=["items.category"], + order_by=["items.category"], + ) + assert [field[0] for field in cursor.description] == ["category", "opaque_count", "average"] + # Authored COUNT(*) counts the deduplicated null-extended SQL row. Ordinary + # semantic counts instead count source entities and return zero here. + assert cursor.fetchall() == [("all", 3, 30.0), ("orphan", 1, None)] + + def test_complete_and_advanced_totals_deduplicate_overlapping_entity_groups(layer): layer.adapter.execute("insert into raw_items values (5, 1, 'overlap'), (6, 2, null), (7, 2, null)") cursor = layer.query( diff --git a/tests/semantic_conformance/test_approximate_distinct.py b/tests/semantic_conformance/test_approximate_distinct.py index d73c02800..da60a5110 100644 --- a/tests/semantic_conformance/test_approximate_distinct.py +++ b/tests/semantic_conformance/test_approximate_distinct.py @@ -98,7 +98,12 @@ def test_joined_populations(layer, relationship_type): Model(name="users", table="users", primary_key="id", dimensions=[Dimension(name="region", type="categorical")]) ) layer.graph.get_model("events").relationships.append( - Relationship(name="users", type=relationship_type, sql="user_id", foreign_key="id") + Relationship( + name="users", + type=relationship_type, + foreign_key="user_id" if relationship_type == "many_to_one" else "id", + primary_key="id" if relationship_type == "many_to_one" else "user_id", + ) ) sql = layer.compile(metrics=["events.users"], dimensions=["users.region"], filters=["events.user_id is not null"]) expected = layer.adapter.execute(""" diff --git a/tests/semantic_conformance/test_cohort_parity.py b/tests/semantic_conformance/test_cohort_parity.py index 21842a598..8af6dcf96 100644 --- a/tests/semantic_conformance/test_cohort_parity.py +++ b/tests/semantic_conformance/test_cohort_parity.py @@ -4,7 +4,7 @@ import pytest -from sidemantic import Dimension, Explore, Metric, Model, SecurityPolicy, SemanticLayer +from sidemantic import Dimension, Explore, Metric, Model, Relationship, SecurityPolicy, SemanticLayer from sidemantic.core.semantic_layer import SecurityError from sidemantic.semantic_handoff import graph_to_semantic_input @@ -116,6 +116,14 @@ def test_query_grouping_changes_inner_and_outer_grain(layer): ) +def test_cohort_order_preserves_explicit_null_placement(layer): + layer.adapter.execute("update events set region = null where user_id = 'u3'") + assert result(layer, dimensions=["events.region"], order_by=["events.region DESC NULLS FIRST"]) == ( + ["region", "qualified"], + [(None, 1), ("US", 1), ("EU", 1)], + ) + + def test_time_bucket_is_carried_through_to_outer_output(layer): assert result(layer, dimensions=["events.day__month"], order_by=["events.day__month"]) == ( ["day__month", "qualified"], @@ -304,18 +312,13 @@ def test_nonexistent_explicit_owner_does_not_fall_back_to_entity_match(layer): @pytest.mark.parametrize("layer", ["rust_owned"], indirect=True) -@pytest.mark.parametrize( - "mutation", ["unowned", "unknown_metric", "filled_ignored_offset", "wrapper", "joined_dimension"] -) +@pytest.mark.parametrize("mutation", ["unowned", "unknown_metric", "wrapper", "joined_dimension"]) def test_owned_graph_cohort_unsupported_shapes_remain_gated(layer, mutation): query = {} if mutation == "unowned": layer.graph.metric_owners.clear() elif mutation == "unknown_metric": layer.graph.metric_owners["ghost"] = "events" - elif mutation == "filled_ignored_offset": - cohort_metric(layer).fill_nulls_with = 0 - cohort_metric(layer).time_offset = "1 day" elif mutation == "wrapper": layer.graph.add_metric(Metric(name="wrapped", type="derived", sql="qualified * 2")) query["metrics"] = ["wrapped"] @@ -328,11 +331,29 @@ def test_owned_graph_cohort_unsupported_shapes_remain_gated(layer, mutation): dimensions=[Dimension(name="region", type="categorical")], ) ) + layer.graph.models["events"].relationships.append( + Relationship(name="other", type="many_to_one", foreign_key="user_id") + ) query["dimensions"] = ["other.region"] with pytest.raises(ValueError): result(layer, **query) +@pytest.mark.parametrize("layer", ["python_owned", "rust_owned"], indirect=True) +def test_owned_graph_cohort_default_and_ignored_offset_preserve_population(layer): + metric = cohort_metric(layer) + metric.agg = "sum" + metric.sql = "amount" + metric.fill_nulls_with = -9 + metric.time_offset = "1 day" + assert result(layer) == (["qualified"], [(103,)]) + assert result(layer, user_attributes={"tenant": 99}) == (["qualified"], [(-9,)]) + assert result(layer, dimensions=["events.region"], order_by=["events.region"]) == ( + ["region", "qualified"], + [("EU", 73), ("US", 30)], + ) + + @pytest.mark.parametrize("layer", ["python_owned", "rust_owned"], indirect=True) def test_owned_graph_cohort_visibility_is_enforced(layer): cohort_metric(layer).public = False @@ -397,10 +418,8 @@ def test_selected_calculations_preserve_implicit_entity_dimensions(layer): payload = {**query, "metrics": [cohort_reference(layer)], "table_calculations": [c.name for c in calculations]} assert ( - json.loads( - sidemantic_rs.validate_with_semantic_input( - json.dumps(graph_to_semantic_input(layer.graph)), json.dumps(payload) - ) + sidemantic_rs.validate_with_semantic_input( + json.dumps(graph_to_semantic_input(layer.graph)), json.dumps(payload) ) == [] ) diff --git a/tests/semantic_conformance/test_complete_filters.py b/tests/semantic_conformance/test_complete_filters.py index e00bfd873..e95d110b7 100644 --- a/tests/semantic_conformance/test_complete_filters.py +++ b/tests/semantic_conformance/test_complete_filters.py @@ -133,22 +133,11 @@ def test_filter_conjunction_preserves_disjunction_grouping(layer): "COUNT(CAST(NULL AS INTEGER))", "COUNT((SELECT 1))", "SUM(1)", - "SUM(CAST(amount AS DOUBLE))", - "SUM(amount / 2)", - "COUNT(COALESCE(amount, '{model}'))", - "COUNT(CASE WHEN amount > 0 THEN 'prefix{model}.value' ELSE 'other' END)", "SUM(COALESCE(other.amount, amount))", "SUM(CASE WHEN amount > 0 THEN other.amount ELSE 0 END)", - "SUM(COALESCE(SUM(amount), 0))", - "SUM(amount) + COUNT(amount)", - "SUM(amount) OVER ()", "SUM((SELECT amount))", "SUM(other.amount)", - "COUNT(DISTINCT ABS(amount))", - "AVG(DISTINCT amount)", - "AVG(amount) OVER ()", "AVG(other.amount)", - "COUNT(*) FILTER (WHERE amount > 0)", "COUNT(*) OVER ()", "COUNT(other.*)", ], @@ -172,6 +161,76 @@ def test_unproven_complete_filter_shapes_fail_explicitly(expression): layer.adapter.close() +@pytest.mark.parametrize("engine", ["python", "rust"]) +@pytest.mark.parametrize( + "expression, expected", + [ + ("SUM(CAST(amount AS DOUBLE))", [(8.0,)]), + ("SUM(amount / 2)", [(4.0,)]), + ("SUM(amount) + COUNT(amount)", [(11,)]), + ("COUNT(DISTINCT ABS(amount))", [(2,)]), + ("AVG(DISTINCT amount)", [(3.0,)]), + ("COUNT(*) FILTER (WHERE amount > 0)", [(3,)]), + # Complete expressions filter their physical inputs before evaluating + # the formula, including its CASE fallback and window cardinality. + ("COUNT(CASE WHEN amount > 0 THEN 'prefix{model}.value' ELSE 'other' END)", [(5,)]), + ("SUM(amount) OVER ()", [(8,)] * 5), + ("AVG(amount) OVER ()", [(pytest.approx(8 / 3),)] * 5), + ], +) +def test_complete_filter_formulas_execute_with_physical_inputs(engine, expression, expected): + if engine == "rust": + pytest.importorskip("sidemantic_rs") + layer = SemanticLayer(engine=engine, fallback=False, auto_register=False) + try: + layer.add_model( + Model( + name="orders", + table="orders", + primary_key="id", + metrics=[Metric(name="value", sql=expression, sql_is_complete=True, filters=["amount > 0"])], + ) + ) + layer.adapter.execute(""" + create table orders(id integer, amount integer); + insert into orders values (1, 2), (2, 2), (3, 4), (4, -9), (5, null); + """) + assert_result(layer, {"metrics": ["orders.value"]}, ["value"], expected) + finally: + layer.adapter.close() + + +@pytest.mark.parametrize("engine", ["python", "rust"]) +@pytest.mark.parametrize( + "expression, error", + [ + ("SUM(COALESCE(SUM(amount), 0))", "aggregate function calls cannot be nested"), + ("COUNT(COALESCE(amount, '{model}'))", "Could not convert string"), + ], +) +def test_invalid_complete_sql_keeps_database_validation(engine, expression, error): + import duckdb + + if engine == "rust": + pytest.importorskip("sidemantic_rs") + layer = SemanticLayer(engine=engine, fallback=False, auto_register=False) + try: + layer.add_model( + Model( + name="orders", + table="orders", + primary_key="id", + metrics=[Metric(name="value", sql=expression, sql_is_complete=True, filters=["amount > 0"])], + ) + ) + layer.adapter.execute("create table orders(id integer, amount integer); insert into orders values (1, null)") + sql = layer.compile(metrics=["orders.value"]) + with pytest.raises(duckdb.Error, match=error): + layer.adapter.execute(sql) + finally: + layer.adapter.close() + + def test_foreign_filter_population_fails_explicitly(): pytest.importorskip("sidemantic_rs", reason="Complete filter rejection requires the real Rust extension") layer = SemanticLayer(engine="rust", fallback=False, auto_register=False) diff --git a/tests/semantic_conformance/test_computed_keys.py b/tests/semantic_conformance/test_computed_keys.py index 02dce21ff..9338a0121 100644 --- a/tests/semantic_conformance/test_computed_keys.py +++ b/tests/semantic_conformance/test_computed_keys.py @@ -107,10 +107,11 @@ def test_from_metrics_sql_uses_structured_key_planning(layer): assert data.fetchall() == [(101, 7), (201, 17)] -def test_legacy_sql_shape_is_explicitly_unsupported(layer): - with pytest.raises(UnsupportedSemanticFeaturesError) as caught: - rewrite_semantic_input(layer.graph, "select accounts.id, accounts.budget from accounts") - assert "rewrite.computed_key_query_shape" in caught.value.capabilities +def test_model_sql_uses_computed_identity(layer): + sql = rewrite_semantic_input(layer.graph, "select accounts.id, accounts.budget from accounts order by accounts.id") + data = layer.adapter.execute(sql) + assert [column[0] for column in data.description] == ["id", "budget"] + assert data.fetchall() == [(101, 10), (201, 20)] def test_compound_computed_keys_join_componentwise(layer): diff --git a/tests/semantic_conformance/test_dialect_boundary.py b/tests/semantic_conformance/test_dialect_boundary.py index a303ed22e..65c074590 100644 --- a/tests/semantic_conformance/test_dialect_boundary.py +++ b/tests/semantic_conformance/test_dialect_boundary.py @@ -66,6 +66,11 @@ def test_native_declared_graph_and_query_syntax(layer, dialect, expression, filt } sql = compile_semantic_input(layer.graph, query, input_dialect=dialect) assert layer.adapter.execute(sql).fetchall() == [("a", 10), (None, 5)] + # Quoted metric references must also resolve to the aggregate output, + # retaining explicit ordering rather than leaking the physical qualifier. + query["order_by"] = [order_sql.replace("label", "value").replace("DESC", "ASC")] + sql = compile_semantic_input(layer.graph, query, input_dialect=dialect) + assert layer.adapter.execute(sql).fetchall() == [(None, 5), ("a", 10)] assert graph_to_semantic_input(layer.graph, input_dialect=dialect) == before @@ -93,7 +98,7 @@ def test_native_fragment_framing_and_source_snapshot(layer): source = graph_to_semantic_input(layer.graph) for field, sql in [("filters", "1 = 1 LIMIT 1"), ("order_by", "orders.value DESC LIMIT 1")]: query = {"metrics": ["orders.value"], "query_dialect": "bigquery", field: [sql]} - with pytest.raises(Exception, match="extra clauses"): + with pytest.raises(ValueError, match="extra clauses"): sidemantic_rs.compile_with_semantic_input(json.dumps(source), json.dumps(query)) source["models"][0]["metrics"][0]["metadata"] = {"ossie_target_dialect": "bigquery"} source["models"][0]["metrics"][0]["sql"] = "IFNULL(`amount`, 0)" diff --git a/tests/semantic_conformance/test_filtered_count_family.py b/tests/semantic_conformance/test_filtered_count_family.py index 49e1d9ead..a3f6d8b2d 100644 --- a/tests/semantic_conformance/test_filtered_count_family.py +++ b/tests/semantic_conformance/test_filtered_count_family.py @@ -186,7 +186,7 @@ def test_count_family_cross_source_counts_restore_absent_zero(counts, restricted ) assert result(counts, metrics=["combined"], dimensions=["regions.region"], order_by=["regions.region"]) == ( ["region", "combined"], - [("a", 12), ("b", 20)] if restricted else [("a", 13), ("b", 21), ("c", 30)], + [("a", 12), ("b", 20), (None, None)] if restricted else [("a", 13), ("b", 21), ("c", 30), (None, None)], ) diff --git a/tests/semantic_conformance/test_multistep_conversion_parity.py b/tests/semantic_conformance/test_multistep_conversion_parity.py index 2f6a72320..a56b3c9fe 100644 --- a/tests/semantic_conformance/test_multistep_conversion_parity.py +++ b/tests/semantic_conformance/test_multistep_conversion_parity.py @@ -154,6 +154,16 @@ def test_time_bucket_belongs_to_first_step(layer): ) +@pytest.mark.parametrize("placement", ["FIRST", "LAST"]) +def test_funnel_order_preserves_explicit_null_placement(layer, placement): + rows = [("a", 4, 4, 1, 1, 1), ("b", 1, 1, 1, 1, 1)] + null_row = [(None, 1, 1, 1, 0, 0)] + assert result(layer, dimensions=["events.region"], order_by=[f"events.region ASC NULLS {placement}"]) == ( + ["region", *COLUMNS], + null_row + rows if placement == "FIRST" else rows + null_row, + ) + + @pytest.mark.parametrize("layer", ["rust"], indirect=True) @pytest.mark.parametrize( "expression", @@ -166,13 +176,23 @@ def test_step_cannot_change_source_scope(layer, expression): @pytest.mark.parametrize("layer", ["rust"], indirect=True) -@pytest.mark.parametrize("alias", ["TOTAL_ENTITIES", "STEP_2_COUNT", "FUNNEL", "ENTITY", "STEP_1_TS"]) +@pytest.mark.parametrize("alias", ["TOTAL_ENTITIES", "STEP_2_COUNT", "FUNNEL"]) def test_fixed_output_alias_collisions_are_explicit(layer, alias): layer.graph.models["events"].dimensions.append(Dimension(name=alias, sql="region", type="categorical")) with pytest.raises(ValueError, match="conversion_output_alias"): layer.compile(metrics=["events.funnel"], dimensions=[f"events.{alias}"], user_attributes={"tenant": 1}) +@pytest.mark.parametrize("layer", ["rust"], indirect=True) +@pytest.mark.parametrize("alias", ["ENTITY", "STEP_1_TS", "step_2_ts", "STEP_3_TS"]) +def test_output_aliases_can_share_names_with_scoped_internal_columns(layer, alias): + layer.graph.models["events"].dimensions.append(Dimension(name=alias, sql="region", type="categorical")) + assert result(layer, dimensions=[f"events.{alias}"], order_by=[f"events.{alias}"]) == ( + [alias, *COLUMNS], + [("a", 4, 4, 1, 1, 1), ("b", 1, 1, 1, 1, 1), (None, 1, 1, 1, 0, 0)], + ) + + @pytest.mark.parametrize("layer", ["rust"], indirect=True) @pytest.mark.parametrize("source", ["sum(tenant) over ()", "(select max(occurred) from funnel_events)"]) def test_mapped_time_cannot_change_source_scope(layer, source): diff --git a/tests/semantic_conformance/test_null_fill.py b/tests/semantic_conformance/test_null_fill.py index d65907f9c..a018adb7c 100644 --- a/tests/semantic_conformance/test_null_fill.py +++ b/tests/semantic_conformance/test_null_fill.py @@ -54,7 +54,15 @@ def test_metric_result_and_dependency_defaults(layer, metrics, filters, expected assert layer.adapter.execute(sql).fetchall() == expected -def test_filled_leaf_participates_in_cross_source_calculation(layer): +@pytest.mark.parametrize( + "metric,expected", + [ + ("filled", [("a", 2), ("b", 4), ("c", 6)]), + ("fractional", [("a", 2), ("b", 2.5), ("c", 4.5)]), + ("raw", [("a", 2), ("b", None), ("c", None)]), + ], +) +def test_filled_leaf_participates_in_cross_source_calculation(layer, metric, expected): layer.add_model( Model( name="groups", @@ -67,12 +75,12 @@ def test_filled_leaf_participates_in_cross_source_calculation(layer): layer.graph.models["events"].relationships = [ Relationship(name="groups", type="many_to_one", foreign_key="category", primary_key="category") ] - layer.add_metric(Metric(name="total", type="derived", sql="events.filled + groups.quota")) + layer.add_metric(Metric(name="total", type="derived", sql=f"events.{metric} + groups.quota")) layer.adapter.execute( "create table fill_groups(category varchar, quota integer); insert into fill_groups values ('a', 2), ('b', 4), ('c', 6)" ) sql = layer.compile(metrics=["total"], dimensions=["groups.category"], order_by=["groups.category"]) - assert layer.adapter.execute(sql).fetchall() == [("a", 2), ("b", 4), ("c", 6)] + assert layer.adapter.execute(sql).fetchall() == expected def test_policy_excluded_groups_are_not_created_by_fill(layer): diff --git a/tests/semantic_conformance/test_query_catalogs.py b/tests/semantic_conformance/test_query_catalogs.py index 1fb1872cb..0c33771e4 100644 --- a/tests/semantic_conformance/test_query_catalogs.py +++ b/tests/semantic_conformance/test_query_catalogs.py @@ -124,12 +124,14 @@ def test_raw_active_catalog_requests_are_never_ignored(layer, field, value): layer.graph, {"metrics": ["orders.revenue"], "user_attributes": {"tenant": "a"}, field: value} ) runtime = pytest.importorskip("sidemantic_rs") - with pytest.raises(ValueError, match=f"rewrite.context.{field}"): + with pytest.raises(ValueError, match=f"rewrite.context.{field}") as caught: runtime.rewrite_with_semantic_input_context( json.dumps(graph_to_semantic_input(layer.graph)), "select orders.revenue from metrics", json.dumps({"user_attributes": {"tenant": "a"}, field: value}), ) + assert isinstance(caught.value, runtime.UnsupportedSemanticFeaturesError) + assert caught.value.capabilities == [f"rewrite.context.{field}"] @pytest.mark.parametrize("layer", ["rust"], indirect=True) diff --git a/tests/semantic_conformance/test_retention_parity.py b/tests/semantic_conformance/test_retention_parity.py index d387ad759..40e3f486e 100644 --- a/tests/semantic_conformance/test_retention_parity.py +++ b/tests/semantic_conformance/test_retention_parity.py @@ -198,10 +198,8 @@ def test_selected_calculations_preserve_retention_fixed_projection(layer): # caller attributes; compile separately performs authorization. payload = {**query, "metrics": ["events.retained"], "table_calculations": [c.name for c in calculations]} assert ( - json.loads( - sidemantic_rs.validate_with_semantic_input( - json.dumps(graph_to_semantic_input(layer.graph)), json.dumps(payload) - ) + sidemantic_rs.validate_with_semantic_input( + json.dumps(graph_to_semantic_input(layer.graph)), json.dumps(payload) ) == [] ) diff --git a/tests/semantic_conformance/test_rollup_routing.py b/tests/semantic_conformance/test_rollup_routing.py index d77af774e..76ac2fea5 100644 --- a/tests/semantic_conformance/test_rollup_routing.py +++ b/tests/semantic_conformance/test_rollup_routing.py @@ -147,7 +147,7 @@ def test_relationship_foreign_key_dimension_declines_unrelated_rollup(layer): { "metrics": ["orders.revenue"], "dimensions": ["orders.customer_id"], - "order_by": ["orders.customer_id"], + "order_by": ["orders.customer_id NULLS LAST"], }, [(10, 30), (20, 940), (None, None)], routed=False, diff --git a/tests/semantic_conformance/test_selected_table_calculations.py b/tests/semantic_conformance/test_selected_table_calculations.py index 145be528c..86706d8a9 100644 --- a/tests/semantic_conformance/test_selected_table_calculations.py +++ b/tests/semantic_conformance/test_selected_table_calculations.py @@ -182,11 +182,12 @@ def test_canonical_validation_checks_active_calculations(mutation): elif mutation == "unknown_option": source["table_calculations"][0]["unknown_window"] = "future" if mutation is None: - assert json.loads(runtime.validate_with_semantic_input(json.dumps(source), json.dumps(query))) == [] + assert runtime.validate_with_semantic_input(json.dumps(source), json.dumps(query)) == [] else: - with pytest.raises(ValueError): + error = runtime.UnsupportedSemanticFeaturesError if mutation == "formula" else ValueError + with pytest.raises(error): runtime.validate_with_semantic_input(json.dumps(source), json.dumps(query)) - with pytest.raises(ValueError): + with pytest.raises(error): runtime.compile_with_semantic_input(json.dumps(source), json.dumps(query)) diff --git a/tests/semantic_conformance/test_snapshot_parity.py b/tests/semantic_conformance/test_snapshot_parity.py index 444204a87..41eb9e0e7 100644 --- a/tests/semantic_conformance/test_snapshot_parity.py +++ b/tests/semantic_conformance/test_snapshot_parity.py @@ -58,6 +58,25 @@ def test_simple_model_snapshot_is_not_silently_summed(layer): assert result(layer, metrics=["snapshots.closing"]) == [(1480,)] +def test_explicit_unsafe_snapshot_option_preserves_all_authorized_rows(layer): + from sidemantic import Metric + + layer.allow_non_additive_unsafe = True + layer.add_metric(Metric(name="wrapped", type="derived", sql="snapshots.closing + 1")) + assert result(layer, metrics=["snapshots.closing", "snapshots.activity", "wrapped"]) == [(1480, 36, 1481)] + assert result( + layer, metrics=["snapshots.closing"], dimensions=["snapshots.account"], order_by=["snapshots.account"] + ) == [("A", 420), ("B", 130), ("C", 30), ("D", 900)] + assert result(layer, metrics=["snapshots.closing"], user_attributes={"tenant": 2}) == [(999,)] + sql = "select closing from snapshots" + assert layer.sql(sql, user_attributes={"tenant": 1}).fetchall() == [(1480,)] + # The option changes this layer's planning, not the graph annotation. + assert layer.graph.models["snapshots"].get_metric("closing").non_additive_dimension == "day" + layer.allow_non_additive_unsafe = False + assert result(layer, metrics=["snapshots.closing", "wrapped"]) == [(250, 251)] + assert layer.sql(sql, user_attributes={"tenant": 1}).fetchall() == [(250,)] + + def test_declared_snapshot_groups_do_not_become_global_latest_date(layer): # The global latest authorized date belongs to C and has a NULL balance. assert result(layer, metrics=["snapshots.closing", "snapshots.global_closing"]) == [(250, None)] @@ -72,6 +91,15 @@ def test_snapshot_groups_preserve_all_null_time_and_latest_null_value(layer): ) == [("A", 170, 100, 6), ("B", 80, 50, 9), ("C", None, 30, 13), ("D", None, None, 8)] +def test_snapshot_order_preserves_explicit_null_placement(layer): + assert result( + layer, + metrics=["snapshots.closing"], + dimensions=["snapshots.account"], + order_by=["snapshots.closing DESC NULLS FIRST", "snapshots.account"], + ) == [("C", None), ("D", None), ("A", 170), ("B", 80)] + + def test_declared_groups_roll_up_into_selected_region(layer): assert result( layer, @@ -205,17 +233,19 @@ def test_snapshot_default_handles_empty_totals_without_creating_groups(layer): ) -def test_filled_snapshot_calculated_wrapper_stays_explicitly_unsupported_in_rust(layer): +def test_filled_snapshot_calculated_wrapper_uses_final_aggregate_default(layer): from sidemantic import Metric - from sidemantic.semantic_handoff import UnsupportedSemanticFeaturesError layer.graph.models["snapshots"].get_metric("closing").fill_nulls_with = -9 layer.add_metric(Metric(name="wrapped", type="derived", sql="snapshots.closing + 1")) - if layer.engine == "rust": - with pytest.raises(UnsupportedSemanticFeaturesError): - result(layer, metrics=["wrapped"]) - else: - assert result(layer, metrics=["wrapped"]) == [(251,)] + assert result(layer, metrics=["wrapped"]) == [(251,)] + assert result(layer, metrics=["wrapped"], user_attributes={"tenant": 99}) == [(-8,)] + assert result(layer, metrics=["wrapped"], dimensions=["snapshots.account"], order_by=["snapshots.account"]) == [ + ("A", 171), + ("B", 81), + ("C", -8), + ("D", -8), + ] def test_snapshot_and_additive_sibling_keep_distinct_final_defaults(layer): diff --git a/tests/semantic_conformance/test_window_output_parity.py b/tests/semantic_conformance/test_window_output_parity.py index 4bb793e70..2a621cea4 100644 --- a/tests/semantic_conformance/test_window_output_parity.py +++ b/tests/semantic_conformance/test_window_output_parity.py @@ -43,6 +43,37 @@ def layer(request): layer.adapter.close() +def test_window_dimension_replaces_same_named_source_for_grouping_and_filters(layer): + layer.graph.models["events"].get_dimension("day").window = "MIN(day) OVER ()" + query = { + "metrics": ["events.daily_amount"], + "dimensions": ["events.day"], + "user_attributes": {"tenant": 1}, + } + assert layer.query(**query).fetchall() == [(date(2024, 1, 1), 82)] + for predicate in ["events.day > '2024-01-02'", "coalesce(events.day, '2024-01-01') > '2024-01-02'"]: + assert layer.query(**query, filters=[predicate]).fetchall() == [] + assert layer.query(**query, filters=["events.day = '2024-01-01'"]).fetchall() == [(date(2024, 1, 1), 82)] + + +def test_graph_calculation_keeps_its_identity_beside_cumulative_metric(layer): + layer.add_metric(Metric(name="total", type="derived", sql="events.daily_amount")) + layer.add_metric(Metric(name="running", type="cumulative", sql="events.daily_amount")) + cursor = layer.query( + metrics=["total", "running"], + dimensions=["events.day"], + order_by=["events.day"], + user_attributes={"tenant": 1}, + ) + columns = [column[0] for column in cursor.description] + records = [dict(zip(columns, row)) for row in cursor.fetchall()] + assert [(record["day"], record["total"], record["running"]) for record in records] == [ + (date(2024, 1, 1), 15, 15), + (date(2024, 1, 3), 27, 42), + (date(2024, 1, 4), 40, 82), + ] + + def run( layer, *, @@ -243,7 +274,7 @@ def test_window_dependency_cycles_return_validation_errors(rust_layer, reference window_expression=f"SUM(base.{reference})", ) ) - with pytest.raises(Exception, match="[Cc]ycl"): + with pytest.raises(Exception, match="[Cc](?:ircular|ycl)"): rust_layer.compile(metrics=["events.windowed"], dimensions=["events.day"], user_attributes={"tenant": 1}) diff --git a/tests/semantic_conformance/test_yardstick_source_dialects.py b/tests/semantic_conformance/test_yardstick_source_dialects.py index a32d73044..6333ec6f0 100644 --- a/tests/semantic_conformance/test_yardstick_source_dialects.py +++ b/tests/semantic_conformance/test_yardstick_source_dialects.py @@ -54,12 +54,14 @@ def test_source_functions_and_quoted_all_dimensions(layer, source, quote, condit @pytest.mark.parametrize("source,quote", [("bigquery", "`"), ("snowflake", '"')]) -def test_source_current_modifier_uses_canonical_group_context(layer, source, quote): +@pytest.mark.parametrize("qualified", [False, True]) +def test_source_current_modifier_uses_canonical_group_context(layer, source, quote, qualified): q = quote + current = f"{q}sales_v{q}.{q}year{q}" if qualified else f"{q}year{q}" assert result( layer, source, - f"SELECT {q}year{q}, AGGREGATE({q}revenue{q}) AT (SET {q}year{q} = CURRENT {q}year{q} - 1) AS prior " + f"SELECT {q}year{q}, AGGREGATE({q}revenue{q}) AT (SET {q}year{q} = CURRENT {current} - 1) AS prior " f"FROM {q}sales_v{q} GROUP BY {q}year{q} ORDER BY {q}year{q}", ) == [(2022, None), (2023, 150)] diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 8b3340dd1..c7c78b110 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -203,6 +203,8 @@ def test_explain_sql_outputs_planner_json(tmp_path): app, [ "explain-sql", + "--engine", + "python", "SELECT * FROM (SELECT order_count, status FROM orders) sq WHERE status = 'completed'", "--models", str(tmp_path), diff --git a/tests/test_cli_t2_contract.py b/tests/test_cli_t2_contract.py index 7405f8335..1b8f1fd98 100644 --- a/tests/test_cli_t2_contract.py +++ b/tests/test_cli_t2_contract.py @@ -21,7 +21,7 @@ @pytest.fixture(autouse=True) -def _reset_cli_state(monkeypatch: pytest.MonkeyPatch): +def _reset_cli_state(monkeypatch: pytest.MonkeyPatch, request): cli_module._loaded_config = None cli_module._project_context = None for name in ( @@ -48,6 +48,8 @@ def _reset_cli_state(monkeypatch: pytest.MonkeyPatch): "CI", ): monkeypatch.delenv(name, raising=False) + # Preserve the selected engine after clearing host CLI settings. + monkeypatch.setenv("SIDEMANTIC_ENGINE", request.config.getoption("--test-engine")) yield cli_module._loaded_config = None cli_module._project_context = None diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 5d487f9eb..7e3bf780f 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -236,14 +236,20 @@ def test_run_query_base_time_filter_uses_raw_expression(demo_layer): def test_run_query_explicit_time_grain_filter_truncates_expression(demo_layer): + demo_layer.conn.execute("ALTER TABLE orders_table ALTER COLUMN order_date TYPE TIMESTAMP") + demo_layer.conn.execute( + "INSERT INTO orders_table VALUES (99, '99', 'Midday', '2024-01-02 12:00:00', 999, 'completed')" + ) result = run_query( metrics=["orders.total_revenue"], - where="orders.order_date__day >= DATE '2024-01-02'", + where="orders.order_date__day > DATE '2024-01-02'", dry_run=True, ) - where_clause = result["sql"].split("WHERE", 1)[1].split(")\nSELECT", 1)[0] - assert "DATE_TRUNC('DAY', ORDER_DATE)" in where_clause.upper() + expected = demo_layer.conn.execute( + "SELECT SUM(amount) FROM orders_table WHERE DATE_TRUNC('day', order_date) > DATE '2024-01-02'" + ).fetchall() + assert demo_layer.conn.execute(result["sql"]).fetchall() == expected def test_run_query_freezes_current_timestamp_for_partition_pruning(demo_layer): diff --git a/tests/test_metric_expressions.py b/tests/test_metric_expressions.py index 2ba02cb47..a0dfd40c7 100644 --- a/tests/test_metric_expressions.py +++ b/tests/test_metric_expressions.py @@ -196,7 +196,10 @@ def test_ratio_prefers_exact_graph_metric_with_dotted_name(): sql = layer.compile(metrics=["exact_ratio"], dimensions=["orders.status"]) assert "orders_cte.revenue_raw" not in sql - assert "SUM(orders_cte.amount) * 2" in sql + graph_rows = layer.query( + metrics=["orders.revenue"], dimensions=["orders.status"], order_by=["orders.status"] + ).fetchall() + assert graph_rows == [("open", 50), ("paid", 300)] rows = layer.query(metrics=["exact_ratio"], dimensions=["orders.status"], order_by=["orders.status"]).fetchall() assert rows == [("open", 1.0), ("paid", 1.0)] diff --git a/tests/test_native_build_cache.py b/tests/test_native_build_cache.py new file mode 100644 index 000000000..a75287894 --- /dev/null +++ b/tests/test_native_build_cache.py @@ -0,0 +1,116 @@ +"""Exercise uv's local wheel reuse with the native packages' real cache keys.""" + +import os +import shutil +import subprocess +import sys +import time +from pathlib import Path +from textwrap import dedent + +import pytest + + +@pytest.mark.parametrize( + ("package", "changed_file"), + [ + ("sidemantic-rs", "src/lib.rs"), + ("sidemantic-rs", "build.rs"), + ("sidemantic-rs", "../Cargo.lock"), + ("crates/dax-pyo3", "src/lib.rs"), + ("crates/dax-pyo3", "../dax-parser/src/functions.rs"), + ("crates/dax-pyo3", "python/sidemantic_dax/__init__.py"), + ], +) +def test_native_source_changes_invalidate_uv_wheels(tmp_path, package, changed_file): + uv = shutil.which("uv") + if uv is None: + pytest.skip("uv is required to exercise its wheel cache") + + repo = Path(__file__).resolve().parents[1] + project = tmp_path / "source" / package + project.mkdir(parents=True) + source = project / changed_file + source.parent.mkdir(parents=True, exist_ok=True) + source.write_text("first source\n") + cache_keys = (repo / package / "pyproject.toml").read_text().split("[tool.uv]\n", 1)[1] + (project / "pyproject.toml").write_text( + '[build-system]\nrequires = []\nbuild-backend = "backend"\nbackend-path = ["."]\n' + '[project]\nname = "cache-probe"\nversion = "1.0"\n' + f"[tool.uv]\n{cache_keys}" + ) + # A dependency-free PEP 517 backend stands in for maturin. The observable + # wheel contents prove when uv reuses a wheel and when it calls the backend; + # no Rust compiler or network is needed for this invalidation contract. + (project / "backend.py").write_text( + dedent( + f"""\ + from pathlib import Path + from zipfile import ZipFile + + def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): + name = "cache_probe-1.0-py3-none-any.whl" + source = Path({changed_file!r}).read_text() + with ZipFile(Path(wheel_directory) / name, "w") as wheel: + wheel.writestr("cache_probe.py", source) + wheel.writestr( + "cache_probe-1.0.dist-info/METADATA", + "Metadata-Version: 2.1\\nName: cache-probe\\nVersion: 1.0\\n", + ) + wheel.writestr( + "cache_probe-1.0.dist-info/WHEEL", + "Wheel-Version: 1.0\\nRoot-Is-Purelib: true\\nTag: py3-none-any\\n", + ) + wheel.writestr("cache_probe-1.0.dist-info/RECORD", "") + counter = Path("build-count") + previous = int(counter.read_text()) if counter.exists() else 0 + counter.write_text(str(previous + 1)) + return name + """ + ) + ) + + requirement = tmp_path / "requirements.txt" + requirement.write_text(f"cache-probe @ {project.as_uri()}\n") + + def install(attempt): + target = tmp_path / f"installed-{attempt}" + result = subprocess.run( + [ + uv, + "pip", + "install", + "--python", + sys.executable, + "--target", + str(target), + "--no-deps", + "--no-index", + "--requirement", + str(requirement), + ], + env={**os.environ, "UV_CACHE_DIR": str(tmp_path / "cache")}, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + return (target / "cache_probe.py").read_text() + + assert install(1) == "first source\n" + first_builds = (project / "build-count").read_text() + if package == "crates/dax-pyo3": + # Import and editable-build outputs must not invalidate their own wheel. + generated = project / "python/sidemantic_dax" + (generated / "__pycache__").mkdir(parents=True, exist_ok=True) + (generated / "__pycache__/__init__.pyc").write_bytes(b"bytecode") + (generated / "_native.so").write_bytes(b"compiled extension") + assert install(2) == "first source\n" + assert (project / "build-count").read_text() == first_builds + + source.write_text("changed source\n") + # uv's file keys use mtimes; avoid depending on filesystem clock resolution. + changed_at = time.time() + 2 + os.utime(source, (changed_at, changed_at)) + assert install(3) == "changed source\n" + assert int((project / "build-count").read_text()) > int(first_builds) diff --git a/tests/test_query_fragment_boundary.py b/tests/test_query_fragment_boundary.py index 58ed88428..1dd92f51c 100644 --- a/tests/test_query_fragment_boundary.py +++ b/tests/test_query_fragment_boundary.py @@ -320,12 +320,20 @@ def test_escaped_static_quote_before_parameter_keeps_literal_boundary(query_laye @pytest.mark.parametrize("use_segment", [False, True]) -def test_escaped_static_quote_before_parameter_through_query(query_layer, use_segment): +@pytest.mark.parametrize( + "literal,static_prefix", + [ + (r"E'prefix\'{{ value }}'", "prefix'"), + (r"e'é prefix\\{{ value }}'", "é prefix\\"), + (r"E'line\n{{ value }}'", "line\n"), + ], +) +def test_escaped_static_quote_before_parameter_through_query(query_layer, use_segment, literal, static_prefix): from sidemantic.core.segment import Segment query_layer.graph.add_parameter(Parameter(name="value", type="string")) - predicate = "{# c #}events.event_type = E'prefix\\'{{ value }}'" - query_layer.conn.execute("insert into events_raw values (3, 1, ?, '2024-01-03')", ["prefix'ok"]) + predicate = "{# c #}events.event_type = " + literal + query_layer.conn.execute("insert into events_raw values (3, 1, ?, '2024-01-03')", [static_prefix + "ok"]) if use_segment: query_layer.graph.models["events"].segments.append(Segment(name="prefixed", sql=predicate)) query_args = {"segments": ["events.prefixed"]} diff --git a/tests/test_sql_generation_security.py b/tests/test_sql_generation_security.py index 1b8439e9b..12eb87505 100644 --- a/tests/test_sql_generation_security.py +++ b/tests/test_sql_generation_security.py @@ -4,6 +4,7 @@ """ import pytest +from sqlglot import exp, parse_one from sidemantic import Dimension, Metric, Model from sidemantic.core.table_calculation import TableCalculation @@ -219,9 +220,16 @@ def test_model_ref_rewrite_matches_cte_identifier_quoting(layer): sql = layer.compile(metrics=["ORDERS.inline_total"]) - assert "WITH ORDERS_cte AS" in sql - assert "SUM(ORDERS_cte.amount) AS inline_total" in sql - assert 'SUM("ORDERS_cte".amount) AS inline_total' not in sql + parsed = parse_one(sql, dialect="postgres") + cte = next(cte for cte in parsed.find_all(exp.CTE) if cte.alias == "ORDERS_cte") + definition = cte.args["alias"].this + references = [table.this for table in parsed.find_all(exp.Table) if table.name == cte.alias] + references.extend(column.args["table"] for column in parsed.find_all(exp.Column) if column.table == cte.alias) + assert references + assert all(ref.args.get("quoted", False) == definition.args.get("quoted", False) for ref in references) + layer.conn.execute("CREATE TABLE orders_table (order_id INTEGER, amount INTEGER)") + layer.conn.execute("INSERT INTO orders_table VALUES (1, 10), (2, 25)") + assert layer.conn.execute(sql).fetchall() == [(35,)] def test_inline_aggregate_dependency_alias_uses_identifier_quoting(layer): @@ -245,8 +253,12 @@ def test_inline_aggregate_dependency_alias_uses_identifier_quoting(layer): sql = layer.compile(metrics=["orders.inline_total"]) - assert 'AS "order total"' in sql - assert "AS order total" not in sql + columns = [column for column in parse_one(sql).find_all(exp.Column) if column.name == "order total"] + assert columns + assert all(column.this.args.get("quoted") for column in columns) + layer.conn.execute('CREATE TABLE orders_table (id INTEGER, amount INTEGER, "order total" INTEGER)') + layer.conn.execute("INSERT INTO orders_table VALUES (1, 100, 10), (2, 200, 25)") + assert layer.conn.execute(sql).fetchall() == [(35,)] def test_count_metrics_with_filters(layer): diff --git a/tests/test_validation.py b/tests/test_validation.py index b44e8385f..2068fc273 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -43,8 +43,8 @@ def test_model_validation_no_table(layer): assert "must have one of 'table', 'sql', 'dax', or 'source_uri' defined" in str(exc_info.value) -def test_source_uri_model_validates_but_python_compile_is_not_supported(layer): - """source_uri-only models can load, but Python SQL generation cannot query them yet.""" +def test_source_uri_model_validates_but_compile_is_not_supported(layer): + """source_uri-only models can load, but SQL generation cannot query them yet.""" layer.add_model( Model( name="events", @@ -60,7 +60,7 @@ def test_source_uri_model_validates_but_python_compile_is_not_supported(layer): message = str(exc_info.value) assert "source_uri" in message - assert "Python SQL generation does not load source_uri data" in message + assert "SQL generation does not load source_uri data" in message def test_metric_validation_simple_no_measure(): @@ -269,9 +269,11 @@ def test_query_validation_reports_ambiguous_join_routes(layer): layer.compile(metrics=["a.total"], dimensions=["d.label"]) message = str(exc_info.value) - assert "Ambiguous join paths between a and d" in message - assert "a -> b -> d" in message - assert "a -> c -> d" in message + if "Ambiguous join paths between a and d" in message: + assert "a -> b -> d" in message + assert "a -> c -> d" in message + else: + assert "Ambiguous join path between 'a' and 'd': multiple paths exist" in message def test_query_validation_invalid_granularity(layer): diff --git a/uv.lock b/uv.lock index 38c3fba83..b7c9b6485 100644 --- a/uv.lock +++ b/uv.lock @@ -2,18 +2,14 @@ version = 1 revision = 3 requires-python = ">=3.11" resolution-markers = [ - "python_full_version >= '3.15' and platform_python_implementation == 'PyPy'", - "python_full_version >= '3.15' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.14.*' and platform_python_implementation == 'PyPy'", - "python_full_version == '3.14.*' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", + "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version < '3.12'", ] - required-markers = [ - "platform_machine == 'x86_64' and sys_platform == 'linux'", + "python_full_version < '3.15' and implementation_name == 'cpython' and platform_machine == 'x86_64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'", ] [[package]] @@ -60,8 +56,7 @@ name = "altair" version = "5.5.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy'", + "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version < '3.12'", ] @@ -82,10 +77,8 @@ name = "altair" version = "6.2.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.15' and platform_python_implementation == 'PyPy'", - "python_full_version >= '3.15' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.14.*' and platform_python_implementation == 'PyPy'", - "python_full_version == '3.14.*' and platform_python_implementation != 'PyPy'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", ] dependencies = [ { name = "jinja2" }, @@ -181,14 +174,11 @@ name = "argon2-cffi-bindings" version = "21.2.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.15' and platform_python_implementation == 'PyPy'", - "python_full_version >= '3.15' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.14.*' and platform_python_implementation == 'PyPy'", - "python_full_version == '3.14.*' and platform_python_implementation != 'PyPy'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", ] dependencies = [ - { name = "cffi", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation == 'PyPy'" }, - { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation != 'PyPy'" }, + { name = "cffi" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b9/e9/184b8ccce6683b0aa2fbb7ba5683ea4b9c5763f1356347f1312c32e3c66e/argon2-cffi-bindings-21.2.0.tar.gz", hash = "sha256:bb89ceffa6c791807d1305ceb77dbfacc5aa499891d2c55661c6459651fc39e3", size = 1779911, upload-time = "2021-12-01T08:52:55.68Z" } wheels = [ @@ -209,14 +199,12 @@ name = "argon2-cffi-bindings" version = "25.1.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy'", + "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version < '3.12'", ] dependencies = [ - { name = "cffi", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy'" }, - { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.13.*' or platform_python_implementation != 'PyPy'" }, + { name = "cffi" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5c/2d/db8af0df73c1cf454f71b2bbe5e356b8c1f8041c979f505b3d3186e520a9/argon2_cffi_bindings-25.1.0.tar.gz", hash = "sha256:b957f3e6ea4d55d820e40ff76f450952807013d361a65d7f28acc0acbf29229d", size = 1783441, upload-time = "2025-07-30T10:02:05.147Z" } wheels = [ @@ -430,67 +418,10 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e4/37/af0d2ef3967ac0d6113837b44a4f0bfe1328c2b9763bd5b1744520e5cfed/certifi-2025.10.5-py3-none-any.whl", hash = "sha256:0f212c2744a9bb6de0c56639a6f68afe01ecd92d91f14ae897c4fe7bbeeef0de", size = 163286, upload-time = "2025-10-05T04:12:14.03Z" }, ] -[[package]] -name = "cffi" -version = "1.17.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.15' and platform_python_implementation == 'PyPy'", - "python_full_version == '3.14.*' and platform_python_implementation == 'PyPy'", - "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy'", -] -dependencies = [ - { name = "pycparser" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621, upload-time = "2024-09-04T20:45:21.852Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/f4/927e3a8899e52a27fa57a48607ff7dc91a9ebe97399b357b85a0c7892e00/cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401", size = 182264, upload-time = "2024-09-04T20:43:51.124Z" }, - { url = "https://files.pythonhosted.org/packages/6c/f5/6c3a8efe5f503175aaddcbea6ad0d2c96dad6f5abb205750d1b3df44ef29/cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf", size = 178651, upload-time = "2024-09-04T20:43:52.872Z" }, - { url = "https://files.pythonhosted.org/packages/94/dd/a3f0118e688d1b1a57553da23b16bdade96d2f9bcda4d32e7d2838047ff7/cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4", size = 445259, upload-time = "2024-09-04T20:43:56.123Z" }, - { url = "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", size = 469200, upload-time = "2024-09-04T20:43:57.891Z" }, - { url = "https://files.pythonhosted.org/packages/1c/a0/a4fa9f4f781bda074c3ddd57a572b060fa0df7655d2a4247bbe277200146/cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1", size = 477235, upload-time = "2024-09-04T20:44:00.18Z" }, - { url = "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", size = 459721, upload-time = "2024-09-04T20:44:01.585Z" }, - { url = "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", size = 467242, upload-time = "2024-09-04T20:44:03.467Z" }, - { url = "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", size = 477999, upload-time = "2024-09-04T20:44:05.023Z" }, - { url = "https://files.pythonhosted.org/packages/44/74/f2a2460684a1a2d00ca799ad880d54652841a780c4c97b87754f660c7603/cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f", size = 454242, upload-time = "2024-09-04T20:44:06.444Z" }, - { url = "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", size = 478604, upload-time = "2024-09-04T20:44:08.206Z" }, - { url = "https://files.pythonhosted.org/packages/34/33/e1b8a1ba29025adbdcda5fb3a36f94c03d771c1b7b12f726ff7fef2ebe36/cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655", size = 171727, upload-time = "2024-09-04T20:44:09.481Z" }, - { url = "https://files.pythonhosted.org/packages/3d/97/50228be003bb2802627d28ec0627837ac0bf35c90cf769812056f235b2d1/cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0", size = 181400, upload-time = "2024-09-04T20:44:10.873Z" }, - { url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178, upload-time = "2024-09-04T20:44:12.232Z" }, - { url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840, upload-time = "2024-09-04T20:44:13.739Z" }, - { url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803, upload-time = "2024-09-04T20:44:15.231Z" }, - { url = "https://files.pythonhosted.org/packages/1a/df/f8d151540d8c200eb1c6fba8cd0dfd40904f1b0682ea705c36e6c2e97ab3/cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", size = 478850, upload-time = "2024-09-04T20:44:17.188Z" }, - { url = "https://files.pythonhosted.org/packages/28/c0/b31116332a547fd2677ae5b78a2ef662dfc8023d67f41b2a83f7c2aa78b1/cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", size = 485729, upload-time = "2024-09-04T20:44:18.688Z" }, - { url = "https://files.pythonhosted.org/packages/91/2b/9a1ddfa5c7f13cab007a2c9cc295b70fbbda7cb10a286aa6810338e60ea1/cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", size = 471256, upload-time = "2024-09-04T20:44:20.248Z" }, - { url = "https://files.pythonhosted.org/packages/b2/d5/da47df7004cb17e4955df6a43d14b3b4ae77737dff8bf7f8f333196717bf/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", size = 479424, upload-time = "2024-09-04T20:44:21.673Z" }, - { url = "https://files.pythonhosted.org/packages/0b/ac/2a28bcf513e93a219c8a4e8e125534f4f6db03e3179ba1c45e949b76212c/cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", size = 484568, upload-time = "2024-09-04T20:44:23.245Z" }, - { url = "https://files.pythonhosted.org/packages/d4/38/ca8a4f639065f14ae0f1d9751e70447a261f1a30fa7547a828ae08142465/cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", size = 488736, upload-time = "2024-09-04T20:44:24.757Z" }, - { url = "https://files.pythonhosted.org/packages/86/c5/28b2d6f799ec0bdecf44dced2ec5ed43e0eb63097b0f58c293583b406582/cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", size = 172448, upload-time = "2024-09-04T20:44:26.208Z" }, - { url = "https://files.pythonhosted.org/packages/50/b9/db34c4755a7bd1cb2d1603ac3863f22bcecbd1ba29e5ee841a4bc510b294/cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", size = 181976, upload-time = "2024-09-04T20:44:27.578Z" }, - { url = "https://files.pythonhosted.org/packages/8d/f8/dd6c246b148639254dad4d6803eb6a54e8c85c6e11ec9df2cffa87571dbe/cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e", size = 182989, upload-time = "2024-09-04T20:44:28.956Z" }, - { url = "https://files.pythonhosted.org/packages/8b/f1/672d303ddf17c24fc83afd712316fda78dc6fce1cd53011b839483e1ecc8/cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2", size = 178802, upload-time = "2024-09-04T20:44:30.289Z" }, - { url = "https://files.pythonhosted.org/packages/0e/2d/eab2e858a91fdff70533cab61dcff4a1f55ec60425832ddfdc9cd36bc8af/cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3", size = 454792, upload-time = "2024-09-04T20:44:32.01Z" }, - { url = "https://files.pythonhosted.org/packages/75/b2/fbaec7c4455c604e29388d55599b99ebcc250a60050610fadde58932b7ee/cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683", size = 478893, upload-time = "2024-09-04T20:44:33.606Z" }, - { url = "https://files.pythonhosted.org/packages/4f/b7/6e4a2162178bf1935c336d4da8a9352cccab4d3a5d7914065490f08c0690/cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5", size = 485810, upload-time = "2024-09-04T20:44:35.191Z" }, - { url = "https://files.pythonhosted.org/packages/c7/8a/1d0e4a9c26e54746dc08c2c6c037889124d4f59dffd853a659fa545f1b40/cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4", size = 471200, upload-time = "2024-09-04T20:44:36.743Z" }, - { url = "https://files.pythonhosted.org/packages/26/9f/1aab65a6c0db35f43c4d1b4f580e8df53914310afc10ae0397d29d697af4/cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd", size = 479447, upload-time = "2024-09-04T20:44:38.492Z" }, - { url = "https://files.pythonhosted.org/packages/5f/e4/fb8b3dd8dc0e98edf1135ff067ae070bb32ef9d509d6cb0f538cd6f7483f/cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed", size = 484358, upload-time = "2024-09-04T20:44:40.046Z" }, - { url = "https://files.pythonhosted.org/packages/f1/47/d7145bf2dc04684935d57d67dff9d6d795b2ba2796806bb109864be3a151/cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", size = 488469, upload-time = "2024-09-04T20:44:41.616Z" }, - { url = "https://files.pythonhosted.org/packages/bf/ee/f94057fa6426481d663b88637a9a10e859e492c73d0384514a17d78ee205/cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", size = 172475, upload-time = "2024-09-04T20:44:43.733Z" }, - { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009, upload-time = "2024-09-04T20:44:45.309Z" }, -] - [[package]] name = "cffi" version = "2.0.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.15' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.14.*' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.12.*'", - "python_full_version < '3.12'", -] dependencies = [ { name = "pycparser", marker = "implementation_name != 'PyPy'" }, ] @@ -783,7 +714,7 @@ name = "cryptography" version = "50.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation != 'PyPy'" }, + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/de/41/6cbdcf9142d00fe82836fbb51e503e58088575cf7a0fe1dbff6695bf0840/cryptography-50.0.0.tar.gz", hash = "sha256:eeac2acb5a20ed25e0ad6d1df9891a520b78b404266b6d11778f25d5d691a6c9", size = 880201, upload-time = "2026-07-31T14:25:10.11Z" } wheels = [ @@ -1880,8 +1811,7 @@ name = "lz4" version = "4.4.4" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy'", + "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", "python_full_version < '3.12'", ] @@ -1918,10 +1848,8 @@ name = "lz4" version = "4.4.5" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.15' and platform_python_implementation == 'PyPy'", - "python_full_version >= '3.15' and platform_python_implementation != 'PyPy'", - "python_full_version == '3.14.*' and platform_python_implementation == 'PyPy'", - "python_full_version == '3.14.*' and platform_python_implementation != 'PyPy'", + "python_full_version >= '3.15'", + "python_full_version == '3.14.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/57/51/f1b86d93029f418033dddf9b9f79c8d2641e7454080478ee2aab5123173e/lz4-4.4.5.tar.gz", hash = "sha256:5f0b9e53c1e82e88c10d7c180069363980136b9d7a8306c4dca4f760d60c39f0", size = 172886, upload-time = "2025-11-03T13:02:36.061Z" } wheels = [ @@ -3293,8 +3221,7 @@ name = "pyzmq" version = "27.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13' and implementation_name == 'pypy' and platform_python_implementation == 'PyPy'" }, - { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and implementation_name == 'pypy') or (implementation_name == 'pypy' and platform_python_implementation != 'PyPy')" }, + { name = "cffi", marker = "implementation_name == 'pypy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/04/0b/3c9baedbdf613ecaa7aa07027780b8867f57b6293b6ee50de316c9f3222b/pyzmq-27.1.0.tar.gz", hash = "sha256:ac0765e3d44455adb6ddbf4417dcce460fc40a05978c08efdf2948072f6db540", size = 281750, upload-time = "2025-09-08T23:10:18.157Z" } wheels = [ @@ -3628,6 +3555,7 @@ dependencies = [ { name = "jinja2" }, { name = "pydantic" }, { name = "pyyaml" }, + { name = "sidemantic-rs", marker = "sys_platform != 'emscripten'" }, { name = "sqlglot" }, { name = "typer" }, ] @@ -3850,6 +3778,7 @@ requires-dist = [ { name = "sidemantic", extras = ["postgres", "bigquery", "snowflake", "clickhouse", "databricks", "spark", "adbc"], marker = "extra == 'all-databases'" }, { name = "sidemantic", extras = ["workbench", "mcp", "apps", "charts", "lsp", "dax", "lookml", "malloy", "metricflow", "widget", "api", "ossie"], marker = "extra == 'full'" }, { name = "sidemantic-dax", marker = "extra == 'dax'", directory = "crates/dax-pyo3" }, + { name = "sidemantic-rs", marker = "sys_platform != 'emscripten'", directory = "sidemantic-rs" }, { name = "snowflake-connector-python", marker = "extra == 'snowflake'", specifier = ">=3.0.0" }, { name = "sqlglot", specifier = ">=30.1.0" }, { name = "sqlglot", extras = ["c"], marker = "extra == 'dev'", specifier = ">=30.1.0" }, @@ -3893,6 +3822,11 @@ name = "sidemantic-dax" version = "0.12.0" source = { directory = "crates/dax-pyo3" } +[[package]] +name = "sidemantic-rs" +version = "0.12.0" +source = { directory = "sidemantic-rs" } + [[package]] name = "six" version = "1.17.0"