From 2a0346f061cd368055e6b9ddc2320887b63c3d2a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 07:04:04 +0000 Subject: [PATCH 1/8] feat(rules): run SQL Rules through the SQLClean engine (work in progress) Checkpoint of the SQLClean integration: SQLCLEAN Rule codes, default rule selection with the human-only notice, shared expansion inputs for early lint, and SQL Rules skipped under --no-sql-analysis. e2e fixture classification and Fensu faults are still open. --- Cargo.lock | 17 + Cargo.toml | 2 + benchmarks/dbt-compare/README.md | 2 +- benchmarks/dbt-compare/write_dbt_dense.py | 2 +- .../duckshop-10k/duckshop10k/declarations.py | 6 +- .../duckshop-10k/duckshop10k/project.py | 6 +- benchmarks/duckshop/README.md | 4 +- .../duckshop/project/sqlbuild_project.toml | 40 +- crates/README.md | 2 - crates/sqlbuild-analysis/Cargo.toml | 6 +- .../_helpers/type_locations.rs | 2 +- .../sql_tokens/_helpers/builtin_functions.rs | 151 - .../src/sql_tokens/_helpers/mod.rs | 3 - .../sql_tokens/_helpers/reserved_keywords.rs | 43 - .../src/sql_tokens/_helpers/token_context.rs | 30 - .../src/sql_tokens/constants.rs | 35 +- .../sql_tokens/main/builtin_function_names.rs | 6 - .../src/sql_tokens/main/canonical_tokens.rs | 25 - .../src/sql_tokens/main/case_folding.rs | 23 - .../src/sql_tokens/main/is_unquoted_word.rs | 11 - .../src/sql_tokens/main/mod.rs | 8 - .../src/sql_tokens/main/name_positions.rs | 16 - .../src/sql_tokens/main/projection_spans.rs | 41 - .../src/sql_tokens/main/query_fingerprint.rs | 2 +- .../src/sql_tokens/main/token_texts.rs | 16 - .../sqlbuild-analysis/src/sql_tokens/mod.rs | 7 +- .../src/sql_tokens/models.rs | 7 - .../sql_tokens/reserved_keywords/bigquery.txt | 99 - .../reserved_keywords/databricks.txt | 4 - .../sql_tokens/reserved_keywords/duckdb.txt | 77 - .../sql_tokens/reserved_keywords/generic.txt | 28 - .../sql_tokens/reserved_keywords/postgres.txt | 80 - .../reserved_keywords/snowflake.txt | 65 - .../src/sql_tokens/reserved_keywords/tsql.txt | 186 - .../sqlbuild-analysis/src/sql_tokens/tests.rs | 4 - .../sql_tokens/tests/test_canonical_tokens.rs | 218 - .../src/sql_tokens/tests/test_types.rs | 7 - .../src/cli_text/main/format_compile_text.rs | 4 + crates/sqlbuild-core/src/cli_text/models.rs | 2 + .../src/cli_text/tests/test_cli_text.rs | 2 + .../src/format_command/tests/helpers.rs | 4 +- .../tests/test_format_command.rs | 4 +- crates/sqlbuild-python/Cargo.toml | 1 + .../src/bindings/_helpers/cli/entry.rs | 17 + .../_helpers/compile_command/report.rs | 2 + .../_helpers/compile_driver/analysis.rs | 14 +- .../_helpers/compile_driver/command.rs | 15 +- .../_helpers/compile_driver/pipeline.rs | 7 + .../_helpers/compile_driver/plan_pipeline.rs | 51 +- .../_helpers/compiled_views/relation_keys.rs | 4 +- .../_helpers/rule_records/constants.rs | 2 +- .../src/bindings/_helpers/rules/catalogue.rs | 4 +- .../src/bindings/_helpers/rules/constants.rs | 17 +- .../bindings/_helpers/rules/early_sql_lint.rs | 33 +- .../bindings/_helpers/rules/fix_session.rs | 8 +- .../src/bindings/_helpers/rules/lint.rs | 8 +- .../bindings/_helpers/rules/lint_pipeline.rs | 8 +- .../src/bindings/_helpers/rules/results.rs | 2 +- .../src/bindings/_helpers/rules/rule_plan.rs | 12 + .../bindings/_helpers/rules/rules_config.rs | 8 +- .../src/bindings/_helpers/rules/selection.rs | 23 +- .../src/bindings/_helpers/rules/sql_rules.rs | 18 +- crates/sqlbuild-rules/Cargo.toml | 2 +- .../src/configuration/_helpers/loading.rs | 2 +- crates/sqlbuild-rules/src/constants.rs | 7 +- .../src/engine/_helpers/evaluation.rs | 12 +- .../src/engine/tests/helpers.rs | 2 +- .../engine/tests/test_incremental_rules.rs | 4 +- .../engine/tests/test_relation_exceptions.rs | 4 +- .../src/engine/tests/test_sql_scanning.rs | 2 +- crates/sqlbuild-rules/src/lib.rs | 5 +- .../src/lint_text/_helpers/format_bodies.rs | 3 +- .../src/lint_text/_helpers/mod.rs | 1 + .../_helpers/preparation.rs | 2 +- .../src/lint_text/_helpers/sites.rs | 2 +- .../src/lint_text/_helpers/sql_diagnostics.rs | 2 +- .../src/lint_text/_helpers/sql_requests.rs | 53 +- .../sqlbuild-rules/src/lint_text/constants.rs | 41 +- .../main/backtick_identifiers.rs | 2 +- .../main/defined_cte_names.rs | 2 +- .../src/lint_text/main/format_sql_bodies.rs | 4 +- .../sqlbuild-rules/src/lint_text/main/mod.rs | 3 + .../main/neutralize_interpolation.rs | 2 +- .../main/preparation.rs | 4 +- .../src/lint_text/tests/test_lint_text.rs | 80 +- crates/sqlbuild-rules/src/lint_text/types.rs | 7 +- crates/sqlbuild-rules/src/models.rs | 31 +- .../_helpers/declared_keys.rs | 4 +- .../src/relation_keys/_helpers/mod.rs | 1 + .../src/relation_keys/constants.rs | 8 + .../main/declared_relation_keys.rs | 6 +- .../main/mod.rs | 1 - .../sqlbuild-rules/src/relation_keys/mod.rs | 6 + .../{sql_quality => relation_keys}/models.rs | 19 - .../src/rule_cache/tests/test_rule_cache.rs | 2 +- .../src/rule_catalogue/constants.rs | 6 +- .../rule_catalogue/main/render_skills_body.rs | 2 +- .../src/rule_results/constants.rs | 2 +- .../rule_results/main/default_rules_notice.rs | 18 + .../main/duplicate_literal_hints.rs | 2 +- .../src/rule_results/main/mod.rs | 1 + .../rule_results/tests/test_rule_results.rs | 8 +- .../src/rules/_helpers/catalogue.rs | 40 +- .../src/rules/_helpers/dynamic_contracts.rs | 2 +- .../sqlbuild-rules/src/rules/main/select.rs | 2 +- .../src/rules/main/selected_sql_codes.rs | 40 +- .../src/sql_lint/_helpers/additional.rs | 1873 ------ .../src/sql_lint/_helpers/engine.rs | 1999 ------ .../src/sql_lint/_helpers/formatter.rs | 343 -- .../src/sql_lint/_helpers/formatter_syntax.rs | 459 -- .../src/sql_lint/_helpers/inline_relations.rs | 223 - .../src/sql_lint/_helpers/join_predicates.rs | 694 --- .../src/sql_lint/_helpers/mod.rs | 10 - .../src/sql_lint/_helpers/printed_layout.rs | 931 --- .../src/sql_lint/_helpers/terminal_shape.rs | 303 - .../src/sql_lint/_helpers/token_layout.rs | 772 --- .../sqlbuild-rules/src/sql_lint/constants.rs | 61 - .../src/sql_lint/main/batch_engine.rs | 19 - .../src/sql_lint/main/catalogue.rs | 6 - .../src/sql_lint/main/cross_join_subject.rs | 15 - .../src/sql_lint/main/engine.rs | 5 - .../src/sql_lint/main/fill_finding_path.rs | 9 - .../src/sql_lint/main/format_requests.rs | 8 - .../src/sql_lint/main/formatter.rs | 5 - .../sqlbuild-rules/src/sql_lint/main/mod.rs | 10 - crates/sqlbuild-rules/src/sql_lint/mod.rs | 10 - crates/sqlbuild-rules/src/sql_lint/models.rs | 177 - crates/sqlbuild-rules/src/sql_lint/tests.rs | 34 - .../tests/fixtures/FORMAT_CORPUS_NOTICE.md | 32 - .../tests/fixtures/format_corpus.json | 5354 ----------------- .../src/sql_lint/tests/helpers.rs | 304 - .../sql_lint/tests/test_authored_tokens.rs | 416 -- .../src/sql_lint/tests/test_batch_lint.rs | 43 - .../sql_lint/tests/test_comment_formatter.rs | 95 - .../src/sql_lint/tests/test_cross_joins.rs | 286 - .../sql_lint/tests/test_empty_fixture_lint.rs | 61 - .../src/sql_lint/tests/test_format_corpus.rs | 61 - .../tests/test_function_call_spacing.rs | 117 - .../tests/test_idempotency_reprint.rs | 48 - .../src/sql_lint/tests/test_layout_rules.rs | 287 - .../src/sql_lint/tests/test_line_wrap.rs | 107 - .../sql_lint/tests/test_lint_exemptions.rs | 130 - .../tests/test_set_operation_formatter.rs | 59 - .../src/sql_lint/tests/test_sql_lint.rs | 1969 ------ .../src/sql_lint/tests/test_types.rs | 182 - .../sql_lint/tests/test_unused_table_alias.rs | 317 - .../sql_lint/tests/test_values_formatter.rs | 53 - crates/sqlbuild-rules/src/sql_lint/types.rs | 4 - .../src/sql_quality/_helpers/cte_columns.rs | 457 -- .../src/sql_quality/_helpers/json_parses.rs | 201 - .../src/sql_quality/_helpers/keys.rs | 437 -- .../src/sql_quality/_helpers/literals.rs | 64 - .../src/sql_quality/_helpers/ranking.rs | 216 - .../sql_quality/_helpers/removal_safety.rs | 102 - .../src/sql_quality/_helpers/syntax.rs | 124 - .../src/sql_quality/constants.rs | 202 - .../src/sql_quality/main/diagnostics.rs | 32 - crates/sqlbuild-rules/src/sql_quality/mod.rs | 25 - .../sqlbuild-rules/src/sql_quality/tests.rs | 10 - .../src/sql_quality/tests/helpers.rs | 116 - .../src/sql_quality/tests/test_json_parses.rs | 282 - .../sql_quality/tests/test_quality_rules.rs | 405 -- .../sql_quality/tests/test_quality_scale.rs | 43 - .../src/sql_quality/tests/test_types.rs | 38 - .../sqlbuild-rules/src/sql_quality/types.rs | 6 - fensu.toml | 5 - .../_helpers/dense_project.py | 2 +- .../_helpers/corpus/failure_cases.py | 8 +- scripts/rules_benchmark/_helpers/workflow.py | 2 +- src/sqlbuild/_native.pyi | 9 +- .../cli/commands/main/compile/helpers.py | 4 +- .../main/compile/sql_macros/helpers.py | 2 +- .../sql_macros/test_sql_macro_body_rules.py | 14 +- .../test_sql_macro_expansion_text.py | 36 +- .../test_sql_macro_relation_slots.py | 10 +- .../main/compile/test_compile_delta.py | 2 +- .../main/compile/test_compile_early_lint.py | 8 +- .../compile/test_parameterised_test_lint.py | 4 +- .../commands/main/format/test_rule_fixes.py | 6 +- .../playground/test_unicode_empty_fixture.py | 6 +- .../cli/commands/main/rules/_test_types.py | 13 +- .../cli/commands/main/rules/helpers.py | 6 +- .../main/rules/test_cross_join_one_row.py | 10 +- .../main/rules/test_default_rule_selection.py | 117 + .../main/rules/test_rule_exception_config.py | 20 +- .../main/rules/test_rules_performance.py | 2 +- .../main/rules/test_sql_quality_rules.py | 58 +- .../main/rules/test_sql_rule_paths.py | 22 +- .../commands/main/rules/test_unevaluated.py | 18 +- .../main/rules/test_unused_cte_contexts.py | 4 +- .../cli/commands/main/scenario/helpers.py | 2 +- .../main/scenario/test_empty_fixture.py | 4 +- .../main/test/test_macro_mock_alias.py | 10 +- .../main/test/test_optional_test_ceremony.py | 2 +- ...e-unused-cte-after-variable-and-macro.json | 4 +- .../compiler/failure/rule-unused-cte.json | 4 +- .../sqlbuild/cli/commands/main/_test_types.py | 13 +- .../main/column_migrations/helpers.py | 5 +- .../test_column_rename_equivalence.py | 7 +- .../commands/main/compile_outputs/helpers.py | 2 +- .../src/sqlbuild/cli/commands/main/helpers.py | 23 +- .../cli/commands/main/test_compile.py | 4 +- .../main/test_compile_native_facts.py | 35 +- .../main/test_compile_prepared_artifacts.py | 10 +- .../main/test_compile_rule_failures.py | 4 +- .../sqlbuild/cli/commands/main/test_format.py | 19 +- .../cli/commands/main/test_format_fix.py | 51 +- .../commands/main/test_format_preservation.py | 11 +- .../main/test_macro_slot_qualification.py | 16 +- .../commands/main/test_repeated_json_parse.py | 6 +- .../main/test_rule_exception_policy.py | 18 +- .../sqlbuild/cli/commands/main/test_rules.py | 108 +- .../commands/main/test_rules_concurrent.py | 4 +- .../cli/commands/main/test_sql_readability.py | 36 +- .../cli/commands/main/test_unicode_lint.py | 20 +- .../cli/entry/test_python_free_commands.py | 4 +- .../cli/recall/test_recall_run_facts.py | 4 +- .../src/sqlbuild/compiler/pipeline/helpers.py | 15 +- .../sqlbuild/compiler/pipeline/test_main.py | 11 +- .../pipeline/test_semantic_binding.py | 13 +- .../compiler/pipeline/test_semantic_scopes.py | 6 +- .../executor/build/test_build_execution.py | 21 +- .../build/test_measurement_audit_execution.py | 3 +- .../rule_engine/native_rows/helpers.py | 2 +- .../test_sql_rule_cache_publication.py | 2 +- .../_helpers/running/test_options.py | 4 +- .../main/compile/test_compile_command.py | 1 - .../cli/commands/main/entry/test_main.py | 4 +- .../sqlbuild/lint/_helpers/test_expansion.py | 4 +- .../sqlbuild/lint/_helpers/test_native_sql.py | 10 +- .../lint/_helpers/test_sqlbuild_tokens.py | 50 +- tests/unit/src/sqlbuild/lint/helpers.py | 53 +- .../unit/src/sqlbuild/lint/test_cli_format.py | 4 +- .../src/sqlbuild/lint/test_expanded_lint.py | 2 +- tests/unit/src/sqlbuild/lint/test_run_lint.py | 12 +- .../_helpers/engine/_test_types.py | 2 +- .../_helpers/engine/test_catalogue.py | 35 +- .../_helpers/engine/test_config.py | 14 +- .../engine/test_incremental_native_rules.py | 4 +- website/src/content/docs/docs/cli/compile.mdx | 2 +- website/src/content/docs/docs/cli/format.mdx | 24 +- website/src/content/docs/docs/cli/rules.mdx | 2 +- .../docs/docs/concepts/macros/sql-macros.mdx | 2 +- .../docs/concepts/project-configuration.mdx | 6 +- .../src/content/docs/docs/concepts/rules.mdx | 40 +- .../rules/configuration-and-selection.mdx | 59 +- .../rules/findings-and-exceptions.mdx | 26 +- 247 files changed, 1253 insertions(+), 22645 deletions(-) delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/_helpers/builtin_functions.rs delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/_helpers/mod.rs delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/_helpers/reserved_keywords.rs delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/_helpers/token_context.rs delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/main/builtin_function_names.rs delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/main/canonical_tokens.rs delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/main/case_folding.rs delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/main/is_unquoted_word.rs delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/main/name_positions.rs delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/main/projection_spans.rs delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/main/token_texts.rs delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/models.rs delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/bigquery.txt delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/databricks.txt delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/duckdb.txt delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/generic.txt delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/postgres.txt delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/snowflake.txt delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/tsql.txt delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/tests.rs delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/tests/test_canonical_tokens.rs delete mode 100644 crates/sqlbuild-analysis/src/sql_tokens/tests/test_types.rs rename crates/sqlbuild-rules/src/{sql_lint => lint_text}/_helpers/preparation.rs (98%) rename crates/sqlbuild-rules/src/{sql_lint => lint_text}/main/backtick_identifiers.rs (64%) rename crates/sqlbuild-rules/src/{sql_lint => lint_text}/main/defined_cte_names.rs (75%) rename crates/sqlbuild-rules/src/{sql_lint => lint_text}/main/preparation.rs (76%) rename crates/sqlbuild-rules/src/{sql_quality => relation_keys}/_helpers/declared_keys.rs (94%) create mode 100644 crates/sqlbuild-rules/src/relation_keys/_helpers/mod.rs create mode 100644 crates/sqlbuild-rules/src/relation_keys/constants.rs rename crates/sqlbuild-rules/src/{sql_quality => relation_keys}/main/declared_relation_keys.rs (83%) rename crates/sqlbuild-rules/src/{sql_quality => relation_keys}/main/mod.rs (60%) create mode 100644 crates/sqlbuild-rules/src/relation_keys/mod.rs rename crates/sqlbuild-rules/src/{sql_quality => relation_keys}/models.rs (60%) create mode 100644 crates/sqlbuild-rules/src/rule_results/main/default_rules_notice.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/_helpers/formatter.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/_helpers/formatter_syntax.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/_helpers/inline_relations.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/_helpers/join_predicates.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/_helpers/mod.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/_helpers/printed_layout.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/_helpers/terminal_shape.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/_helpers/token_layout.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/constants.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/main/batch_engine.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/main/catalogue.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/main/cross_join_subject.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/main/engine.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/main/fill_finding_path.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/main/format_requests.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/main/formatter.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/main/mod.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/mod.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/models.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/tests.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/fixtures/FORMAT_CORPUS_NOTICE.md delete mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/fixtures/format_corpus.json delete mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/helpers.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/test_authored_tokens.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/test_batch_lint.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/test_comment_formatter.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/test_cross_joins.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/test_empty_fixture_lint.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/test_format_corpus.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/test_function_call_spacing.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/test_idempotency_reprint.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/test_layout_rules.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/test_line_wrap.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/test_lint_exemptions.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/test_set_operation_formatter.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/test_sql_lint.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/test_unused_table_alias.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/test_values_formatter.rs delete mode 100644 crates/sqlbuild-rules/src/sql_lint/types.rs delete mode 100644 crates/sqlbuild-rules/src/sql_quality/_helpers/cte_columns.rs delete mode 100644 crates/sqlbuild-rules/src/sql_quality/_helpers/json_parses.rs delete mode 100644 crates/sqlbuild-rules/src/sql_quality/_helpers/keys.rs delete mode 100644 crates/sqlbuild-rules/src/sql_quality/_helpers/literals.rs delete mode 100644 crates/sqlbuild-rules/src/sql_quality/_helpers/ranking.rs delete mode 100644 crates/sqlbuild-rules/src/sql_quality/_helpers/removal_safety.rs delete mode 100644 crates/sqlbuild-rules/src/sql_quality/_helpers/syntax.rs delete mode 100644 crates/sqlbuild-rules/src/sql_quality/constants.rs delete mode 100644 crates/sqlbuild-rules/src/sql_quality/main/diagnostics.rs delete mode 100644 crates/sqlbuild-rules/src/sql_quality/mod.rs delete mode 100644 crates/sqlbuild-rules/src/sql_quality/tests.rs delete mode 100644 crates/sqlbuild-rules/src/sql_quality/tests/helpers.rs delete mode 100644 crates/sqlbuild-rules/src/sql_quality/tests/test_json_parses.rs delete mode 100644 crates/sqlbuild-rules/src/sql_quality/tests/test_quality_rules.rs delete mode 100644 crates/sqlbuild-rules/src/sql_quality/tests/test_quality_scale.rs delete mode 100644 crates/sqlbuild-rules/src/sql_quality/tests/test_types.rs delete mode 100644 crates/sqlbuild-rules/src/sql_quality/types.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/rules/test_default_rule_selection.py diff --git a/Cargo.lock b/Cargo.lock index 5e2d5968a1..f833ee8a04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1228,6 +1228,7 @@ dependencies = [ "sqlbuild-render", "sqlbuild-scopes", "sqlbuild-sqltext", + "sqlclean", ] [[package]] @@ -1395,6 +1396,7 @@ dependencies = [ "sqlbuild-scopes", "sqlbuild-sqltext", "sqlbuild-state", + "sqlclean", "stacker", ] @@ -1458,6 +1460,7 @@ dependencies = [ "sqlbuild-core", "sqlbuild-scopes", "sqlbuild-sqltext", + "sqlclean", "stacker", "tempfile", "toml", @@ -1495,6 +1498,20 @@ dependencies = [ "sqlbuild-cursor", ] +[[package]] +name = "sqlclean" +version = "0.4.0" +source = "git+https://github.com/sqlstack-io/sqlclean.git?tag=v0.4.0#58511b8ad9750e3275d80c2f22bccee932f19d23" +dependencies = [ + "polyglot-sql", + "polyglot-sql-function-catalogs", + "rayon", + "rustc-hash", + "serde", + "serde_json", + "sha2", +] + [[package]] name = "stacker" version = "0.1.25" diff --git a/Cargo.toml b/Cargo.toml index 6014fffc19..f51c001f33 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -139,6 +139,8 @@ sqlbuild-recall = { path = "crates/sqlbuild-recall" } sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } sqlbuild-state = { path = "crates/sqlbuild-state" } +# SQL lint Rules, quality Rules, the formatter and the canonical token stream. +sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", tag = "v0.4.0" } stacker = "0.1" syn = { version = "2", features = ["full", "visit", "extra-traits"] } tempfile = "3" diff --git a/benchmarks/dbt-compare/README.md b/benchmarks/dbt-compare/README.md index f0c437c0e0..8e55044904 100644 --- a/benchmarks/dbt-compare/README.md +++ b/benchmarks/dbt-compare/README.md @@ -34,7 +34,7 @@ average. | Python macros | Jinja macros producing the same SQL | | `audits [not_null]` | `data_tests: [not_null]` | | SQL tests with fixture CTEs | `unit_tests` with the same input and expected rows | -| `[rules] select = ["SQBR", "XSQBR"]` (built-in Rules plus one custom Rule) | none | +| `[rules] select = ["SQBR", "SQLCLEAN", "XSQBR"]` (built-in Rules, SQLClean Rules and one custom Rule) | none | `sqlbuild-norules/` is the same project with `[rules] select = []`. That turns off configurable Rules; column binding, type inference and contract checks still run. diff --git a/benchmarks/dbt-compare/write_dbt_dense.py b/benchmarks/dbt-compare/write_dbt_dense.py index 9c5d33ba0e..64b937ff5d 100644 --- a/benchmarks/dbt-compare/write_dbt_dense.py +++ b/benchmarks/dbt-compare/write_dbt_dense.py @@ -66,7 +66,7 @@ def write_norules(project: Path) -> Path: shutil.copytree(project, target) toml = target / "sqlbuild_project.toml" text = toml.read_text() - new = text.replace('select = ["SQBR", "XSQBR"]', "select = []") + new = text.replace('select = ["SQBR", "SQLCLEAN", "XSQBR"]', "select = []") assert new != text toml.write_text(new) return target diff --git a/benchmarks/duckshop-10k/duckshop10k/declarations.py b/benchmarks/duckshop-10k/duckshop10k/declarations.py index 5967a76ec2..efabbd5a6a 100644 --- a/benchmarks/duckshop-10k/duckshop10k/declarations.py +++ b/benchmarks/duckshop-10k/duckshop10k/declarations.py @@ -321,7 +321,7 @@ def union_relations(relations: list[SqlResourceRef], columns: list[str]) -> str: MACRO_EXCEPTIONS: tuple[tuple[str, str, str], ...] = ( ( - "SQBRSQL018", + "SQLCLEAN018", "cdc_latest", """ The macro cannot see its callers' keys on its own. Every call site is still checked, against the @@ -329,7 +329,7 @@ def union_relations(relations: list[SqlResourceRef], columns: list[str]) -> str: """, ), ( - "SQBRSQL018", + "SQLCLEAN018", "cdc_history", """ The macro cannot see its callers' keys on its own. _seq is unique and non-null in every change-log @@ -337,7 +337,7 @@ def union_relations(relations: list[SqlResourceRef], columns: list[str]) -> str: """, ), ( - "SQBRSQL034", + "SQLCLEAN034", "load_metadata", """ A column-list macro body is a select list by construction. Its columns are inserted into the diff --git a/benchmarks/duckshop-10k/duckshop10k/project.py b/benchmarks/duckshop-10k/duckshop10k/project.py index 1e93aa0cd5..a399a80e44 100644 --- a/benchmarks/duckshop-10k/duckshop10k/project.py +++ b/benchmarks/duckshop-10k/duckshop10k/project.py @@ -192,12 +192,12 @@ def project_toml() -> str: lookup_customer_id = "C000001" [rules] -select = ["SQBRSQL", "SQBRGRAPH", "SQBRCONTRACT", "SQBRTEST", "XSQBRDUCK"] -# SQBRSQL021: `SELECT * FROM final` and the change-log macro's pass-through CTEs are the house +select = ["SQLCLEAN", "SQBRGRAPH", "SQBRCONTRACT", "SQBRTEST", "XSQBRDUCK"] +# SQLCLEAN021: `SELECT * FROM final` and the change-log macro's pass-through CTEs are the house # style; each model's columns block, not its terminal SELECT, owns the output shape. # SQBRCONTRACT101: exact contracts are reserved for published dim_ and fct_ models. # SQBRTEST202: a sample of models carries unit tests, as in a project of this size. -ignore = ["SQBRSQL021", "SQBRCONTRACT101", "SQBRTEST202"] +ignore = ["SQLCLEAN021", "SQBRCONTRACT101", "SQBRTEST202"] ''' diff --git a/benchmarks/duckshop/README.md b/benchmarks/duckshop/README.md index 00a284348c..298e8b8f62 100644 --- a/benchmarks/duckshop/README.md +++ b/benchmarks/duckshop/README.md @@ -48,7 +48,7 @@ The core project, without the [edge-case area](#edge-cases): - 5 seeds, 2 SQL functions (a scalar UDF and a table function) and a Python UDF. - SQL and Python macros, folder-scoped constants and enums, generic and singular audits, model schemas, hooks, incremental models with cursor windows, and snapshots with both strategies. -- Rules select `SQBRSQL`, `SQBRGRAPH`, `SQBRCONTRACT`, `SQBRTEST` and the project's own +- Rules select `SQLCLEAN`, `SQBRGRAPH`, `SQBRCONTRACT`, `SQBRTEST` and the project's own `XSQBRDUCK` rules, with reviewed exceptions in `sqlbuild_project.toml`. Every model must have at least one unit test (`SQBRTEST202`). ## Tests and scenarios @@ -225,7 +225,7 @@ noted here: static types of the columns it reads from a CTE through a table alias (`FROM listings AS l`), so `rpt_edge_partner_listing_export` declares only the name and audits of `"listingId"`, without types. -- `SQBRSQL018` can only take proof from single-column keys. Append-only feeds redeliver rows, so +- `SQLCLEAN018` can only take proof from single-column keys. Append-only feeds redeliver rows, so their id is unique only together with `_loaded_at` (the `unique_per_delivery` audit), and the staging models that keep the latest delivery carry reviewed exceptions. diff --git a/benchmarks/duckshop/project/sqlbuild_project.toml b/benchmarks/duckshop/project/sqlbuild_project.toml index 569b4fa2a0..fdcc6f7f58 100644 --- a/benchmarks/duckshop/project/sqlbuild_project.toml +++ b/benchmarks/duckshop/project/sqlbuild_project.toml @@ -104,14 +104,14 @@ edge_delivery_batch = "2" "ARRAY" = "JSON" [rules] -select = ["SQBRSQL", "SQBRGRAPH", "SQBRCONTRACT", "SQBRTEST", "XSQBRDUCK"] -# SQBRSQL021: `SELECT * FROM final` and the change-log macro's pass-through CTEs are the house +select = ["SQLCLEAN", "SQBRGRAPH", "SQBRCONTRACT", "SQBRTEST", "XSQBRDUCK"] +# SQLCLEAN021: `SELECT * FROM final` and the change-log macro's pass-through CTEs are the house # style; each model's columns block, not its terminal SELECT, owns the output shape. # SQBRCONTRACT101: exact contracts are reserved for published dim_ and fct_ models. -ignore = ["SQBRSQL021", "SQBRCONTRACT101"] +ignore = ["SQLCLEAN021", "SQBRCONTRACT101"] [[rules.rule_exceptions]] -rule = "SQBRSQL018" +rule = "SQLCLEAN018" path = "models/staging/_sqlbuild/macros/sql/cdc_latest.sql" reason = """ The macro cannot see its callers' keys on its own. Every call site is still checked, against the @@ -119,7 +119,7 @@ unique, non-null _seq that each change-log source declares. """ [[rules.rule_exceptions]] -rule = "SQBRSQL018" +rule = "SQLCLEAN018" path = "models/marts/core/dim_date.sql" reason = """ SEQ4() is distinct for every row of one GENERATOR call, so ordering by it is unique. ROW_NUMBER() @@ -127,7 +127,7 @@ renumbers it because SEQ4() alone may leave gaps. """ [[rules.rule_exceptions]] -rule = "SQBRSQL018" +rule = "SQLCLEAN018" path = "models/staging/clickstream/stg_app_events.sql" reason = """ Append-only feeds redeliver rows, so the source id repeats across loads and only (id, _loaded_at) @@ -137,7 +137,7 @@ whichever copy ranks first. """ [[rules.rule_exceptions]] -rule = "SQBRSQL018" +rule = "SQLCLEAN018" path = "models/staging/clickstream/stg_email_events.sql" reason = """ Append-only feeds redeliver rows, so the source id repeats across loads and only (id, _loaded_at) @@ -147,7 +147,7 @@ whichever copy ranks first. """ [[rules.rule_exceptions]] -rule = "SQBRSQL018" +rule = "SQLCLEAN018" path = "models/staging/clickstream/stg_experiment_assignments.sql" reason = """ Append-only feeds redeliver rows, so the source id repeats across loads and only (id, _loaded_at) @@ -157,7 +157,7 @@ whichever copy ranks first. """ [[rules.rule_exceptions]] -rule = "SQBRSQL018" +rule = "SQLCLEAN018" path = "models/staging/clickstream/stg_search_queries.sql" reason = """ Append-only feeds redeliver rows, so the source id repeats across loads and only (id, _loaded_at) @@ -167,7 +167,7 @@ whichever copy ranks first. """ [[rules.rule_exceptions]] -rule = "SQBRSQL018" +rule = "SQLCLEAN018" path = "models/staging/clickstream/stg_web_events.sql" reason = """ Append-only feeds redeliver rows, so the source id repeats across loads and only (id, _loaded_at) @@ -177,7 +177,7 @@ whichever copy ranks first. """ [[rules.rule_exceptions]] -rule = "SQBRSQL018" +rule = "SQLCLEAN018" path = "models/staging/finance/stg_settlement_reports.sql" reason = """ Append-only feeds redeliver rows, so the source id repeats across loads and only (id, _loaded_at) @@ -187,7 +187,7 @@ whichever copy ranks first. """ [[rules.rule_exceptions]] -rule = "SQBRSQL018" +rule = "SQLCLEAN018" path = "models/staging/fulfilment/stg_carrier_webhook_errors.sql" reason = """ Append-only feeds redeliver rows, so the source id repeats across loads and only (id, _loaded_at) @@ -197,7 +197,7 @@ whichever copy ranks first. """ [[rules.rule_exceptions]] -rule = "SQBRSQL018" +rule = "SQLCLEAN018" path = "models/staging/fulfilment/stg_carrier_webhooks.sql" reason = """ Append-only feeds redeliver rows, so the source id repeats across loads and only (id, _loaded_at) @@ -207,7 +207,7 @@ whichever copy ranks first. """ [[rules.rule_exceptions]] -rule = "SQBRSQL018" +rule = "SQLCLEAN018" path = "models/staging/inventory/stg_stock_movements.sql" reason = """ Append-only feeds redeliver rows, so the source id repeats across loads and only (id, _loaded_at) @@ -217,7 +217,7 @@ whichever copy ranks first. """ [[rules.rule_exceptions]] -rule = "SQBRSQL018" +rule = "SQLCLEAN018" path = "models/staging/partners/stg_partner_orders.sql" reason = """ Append-only feeds redeliver rows, so the source id repeats across loads and only (id, _loaded_at) @@ -227,7 +227,7 @@ whichever copy ranks first. """ [[rules.rule_exceptions]] -rule = "SQBRSQL034" +rule = "SQLCLEAN034" path = "models/staging/_sqlbuild/macros/sql/load_metadata.sql" reason = """ A column-list macro body is a select list by construction. Its columns are inserted into the @@ -235,7 +235,7 @@ caller's own CTE, where the rule is checked again. """ [[rules.rule_exceptions]] -rule = "SQBRSQL034" +rule = "SQLCLEAN034" path = "macros/sql/money_columns.sql" reason = """ A column-list macro body is a select list by construction. Its columns are inserted into the @@ -243,7 +243,7 @@ caller's own CTE, where the rule is checked again. """ [[rules.rule_exceptions]] -rule = "SQBRSQL028" +rule = "SQLCLEAN028" path = "models/intermediate/catalogue/int_category_paths.sql" reason = """ LEVEL and the CONNECT_BY_ROOT operator are hierarchical-query pseudo-columns that cannot be @@ -269,7 +269,7 @@ cart_funnel__assigns_furthest_stage_per_session scenario covers the funnel inste """ [[rules.rule_exceptions]] -rule = "SQBRSQL038" +rule = "SQLCLEAN038" path = "models/marts/executive/rpt_executive_kpis_daily.sql" relation = "metric_regions" reason = """ @@ -279,7 +279,7 @@ days without activity report zero instead of disappearing. Both sides are small # >>> edge-case area (scripts/core_project.py leaves this block out of the core copy) [[rules.rule_ignores]] -rules = ["SQBRSQL027"] +rules = ["SQLCLEAN027"] paths = ["models/marts/edge_cases/rpt_edge_stock_reconciliation.sql"] reason = """ The edge case selects the key columns that FULL OUTER JOIN ... USING merges. They belong to neither diff --git a/crates/README.md b/crates/README.md index 37c7b39166..4aaac11815 100644 --- a/crates/README.md +++ b/crates/README.md @@ -111,8 +111,6 @@ playground project on Windows. - Tests live next to the module they cover; run them for one crate with `cargo test -p ` or for all crates with `cargo test --workspace`. -- `sqlbuild-analysis` has a `test-support` feature that exposes test-only helpers to the tests of - dependent crates. Enable it only from `[dev-dependencies]`. - The build identity (`sqlbuild._native.BUILD_IDENTITY`) hashes the sources of every crate in this directory, so editing any crate invalidates the native caches keyed on it. - maturin builds `crates/sqlbuild-python`; the release version lives in diff --git a/crates/sqlbuild-analysis/Cargo.toml b/crates/sqlbuild-analysis/Cargo.toml index 84f9d6cbb5..864c22d7e7 100644 --- a/crates/sqlbuild-analysis/Cargo.toml +++ b/crates/sqlbuild-analysis/Cargo.toml @@ -7,11 +7,6 @@ license.workspace = true publish.workspace = true rust-version.workspace = true -[features] -default = [] -# Expose test-only helpers to the tests of dependent crates. -test-support = [] - [dependencies] memchr.workspace = true polyglot-sql.workspace = true @@ -30,6 +25,7 @@ sqlbuild-model-config.workspace = true sqlbuild-render.workspace = true sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true +sqlclean.workspace = true [lints] workspace = true diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/type_locations.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/type_locations.rs index 6939744c45..551e7d02f9 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/type_locations.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/type_locations.rs @@ -4,9 +4,9 @@ use crate::constants::DATETIME_TYPE; use polyglot_sql::tokens::{Token, TokenType}; use sqlbuild_core::constants::TIMESTAMP_TYPE; -use crate::sql_tokens::main::projection_spans::projection_spans; use polyglot_sql::{Dialect, DialectType, ValidationError}; use regex::Regex; +use sqlclean::sql_tokens::main::projection_spans::projection_spans; use std::sync::LazyLock; const INVALID_CAST: &str = "E218"; diff --git a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/builtin_functions.rs b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/builtin_functions.rs deleted file mode 100644 index 852563f311..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/builtin_functions.rs +++ /dev/null @@ -1,151 +0,0 @@ -//! Function names a dialect recognises as built-ins, from the SQL library's own metadata. - -use std::collections::{HashMap, HashSet}; -use std::sync::LazyLock; - -use crate::sql_tokens::constants::CALL_SYNTAX_FUNCTIONS; -use polyglot_sql::{Dialect, DialectType, Expression}; -use polyglot_sql_function_catalogs::types::{builtin_arity, type_signature}; -use polyglot_sql_function_catalogs::{ - CatalogSink, FunctionNameCase, FunctionSignature, register_enabled_catalogs, -}; - -const DUCKDB_CATALOG: &str = "duckdb"; - -static CATALOG_FUNCTIONS: LazyLock>> = LazyLock::new(|| { - let mut sink = NameSink::default(); - register_enabled_catalogs(&mut sink); - sink.names -}); -static SNOWFLAKE_TYPED: LazyLock> = - LazyLock::new(|| parser_typed_names(DialectType::Snowflake)); -static BIGQUERY_TYPED: LazyLock> = - LazyLock::new(|| parser_typed_names(DialectType::BigQuery)); -static POSTGRES_TYPED: LazyLock> = - LazyLock::new(|| parser_typed_names(DialectType::PostgreSQL)); -static DATABRICKS_TYPED: LazyLock> = - LazyLock::new(|| parser_typed_names(DialectType::Databricks)); -static TSQL_TYPED: LazyLock> = - LazyLock::new(|| parser_typed_names(DialectType::TSQL)); -static GENERIC_TYPED: LazyLock> = - LazyLock::new(|| parser_typed_names(DialectType::Generic)); - -/// Return whether `upper_name(...)` calls a function the dialect defines. -pub(crate) fn is_builtin_function(dialect: DialectType, upper_name: &str) -> bool { - if CALL_SYNTAX_FUNCTIONS.contains(&upper_name) { - return true; - } - let lower_name = upper_name.to_ascii_lowercase(); - let typed = match dialect { - DialectType::DuckDB => return catalogued(DUCKDB_CATALOG, &lower_name), - DialectType::Snowflake => &SNOWFLAKE_TYPED, - DialectType::BigQuery => &BIGQUERY_TYPED, - DialectType::PostgreSQL => &POSTGRES_TYPED, - DialectType::Databricks => &DATABRICKS_TYPED, - DialectType::TSQL => &TSQL_TYPED, - _ => &GENERIC_TYPED, - }; - typed.contains(&lower_name) || catalogued(catalog_key(dialect), &lower_name) -} - -/// Every catalogued or parser-typed built-in function name of `dialect`, lower-cased. -#[cfg(any(test, feature = "test-support"))] -pub(crate) fn builtin_function_names(dialect: DialectType) -> Vec { - let typed: &HashSet = match dialect { - DialectType::DuckDB => &HashSet::new(), - DialectType::Snowflake => &SNOWFLAKE_TYPED, - DialectType::BigQuery => &BIGQUERY_TYPED, - DialectType::PostgreSQL => &POSTGRES_TYPED, - DialectType::Databricks => &DATABRICKS_TYPED, - DialectType::TSQL => &TSQL_TYPED, - _ => &GENERIC_TYPED, - }; - let key = match dialect { - DialectType::DuckDB => DUCKDB_CATALOG, - _ => catalog_key(dialect), - }; - let catalogued = CATALOG_FUNCTIONS.get(key).into_iter().flatten(); - let mut names: Vec = typed.iter().chain(catalogued).cloned().collect(); - names.sort_unstable(); - names.dedup(); - names -} - -fn catalog_key(dialect: DialectType) -> &'static str { - match dialect { - DialectType::Snowflake => "snowflake", - DialectType::BigQuery => "bigquery", - DialectType::PostgreSQL => "postgres", - _ => "", - } -} - -fn catalogued(key: &str, lower_name: &str) -> bool { - CATALOG_FUNCTIONS - .get(key) - .is_some_and(|names| names.contains(lower_name)) - || builtin_arity(key, lower_name).is_some() - || type_signature(key, lower_name).is_some() -} - -/// Catalogued function names whose call the dialect's parser turns into a specific expression. -fn parser_typed_names(dialect: DialectType) -> HashSet { - let parser = Dialect::get(dialect); - let mut typed: HashSet = HashSet::new(); - for name in CATALOG_FUNCTIONS.get(DUCKDB_CATALOG).into_iter().flatten() { - if is_typed_call(&parser, name) { - typed.insert(name.clone()); - } - } - typed -} - -fn is_typed_call(parser: &Dialect, lower_name: &str) -> bool { - if !lower_name - .chars() - .all(|character| character.is_ascii_alphanumeric() || character == '_') - { - return false; - } - let Ok(statements) = parser.parse(&format!("SELECT {lower_name}(probe)")) else { - return false; - }; - let Some(Expression::Select(select)) = statements.first() else { - return false; - }; - select.expressions.first().is_some_and(|call| { - !matches!( - call, - Expression::Function(_) | Expression::Anonymous(_) | Expression::Column(_) - ) - }) -} - -#[derive(Default)] -struct NameSink { - names: HashMap<&'static str, HashSet>, -} - -impl CatalogSink for NameSink { - fn set_dialect_name_case(&mut self, _dialect: &'static str, _name_case: FunctionNameCase) {} - - fn set_function_name_case( - &mut self, - _dialect: &'static str, - _function_name: &str, - _name_case: FunctionNameCase, - ) { - } - - fn register( - &mut self, - dialect: &'static str, - function_name: &str, - _signatures: Vec, - ) { - self.names - .entry(dialect) - .or_default() - .insert(function_name.to_ascii_lowercase()); - } -} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/mod.rs b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/mod.rs deleted file mode 100644 index a2df8fb07e..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod builtin_functions; -pub mod reserved_keywords; -pub mod token_context; diff --git a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/reserved_keywords.rs b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/reserved_keywords.rs deleted file mode 100644 index ccf9004d8d..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/reserved_keywords.rs +++ /dev/null @@ -1,43 +0,0 @@ -//! Per-dialect reserved keywords, which can never be unquoted identifiers in that dialect. - -use std::collections::HashSet; -use std::sync::LazyLock; - -use polyglot_sql::DialectType; - -static DUCKDB: LazyLock> = - LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/duckdb.txt"))); -static POSTGRES: LazyLock> = - LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/postgres.txt"))); -static BIGQUERY: LazyLock> = - LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/bigquery.txt"))); -static SNOWFLAKE: LazyLock> = - LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/snowflake.txt"))); -static TSQL: LazyLock> = - LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/tsql.txt"))); -static DATABRICKS: LazyLock> = - LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/databricks.txt"))); -static GENERIC: LazyLock> = - LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/generic.txt"))); - -/// Return whether the upper-cased word is a reserved keyword of `dialect`. -pub(crate) fn is_reserved_keyword(dialect: DialectType, upper_word: &str) -> bool { - let keywords: &HashSet = match dialect { - DialectType::DuckDB => &DUCKDB, - DialectType::PostgreSQL => &POSTGRES, - DialectType::BigQuery => &BIGQUERY, - DialectType::Snowflake => &SNOWFLAKE, - DialectType::TSQL => &TSQL, - DialectType::Databricks => &DATABRICKS, - _ => &GENERIC, - }; - keywords.contains(upper_word) -} - -fn keyword_set(data: &str) -> HashSet { - data.lines() - .map(str::trim) - .filter(|line| !line.is_empty() && !line.starts_with('#')) - .map(str::to_ascii_uppercase) - .collect() -} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/token_context.rs b/crates/sqlbuild-analysis/src/sql_tokens/_helpers/token_context.rs deleted file mode 100644 index fabb747d27..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/_helpers/token_context.rs +++ /dev/null @@ -1,30 +0,0 @@ -//! Positional context of one authored token among its neighbours. - -use polyglot_sql::DialectType; - -use crate::sql_tokens::_helpers::builtin_functions::is_builtin_function; -use crate::sql_tokens::_helpers::reserved_keywords::is_reserved_keyword; -use crate::sql_tokens::constants::CALL_OPENER; -use crate::sql_tokens::main::is_unquoted_word::is_unquoted_word; - -pub(crate) struct TokenContext<'a> { - pub(crate) raws: &'a [String], - pub(crate) uppers: &'a [String], - pub(crate) index: usize, -} - -impl TokenContext<'_> { - /// Whether the token is a reserved keyword or a built-in function name at a call site. - pub(crate) fn is_foldable(&self, dialect: DialectType) -> bool { - if !is_unquoted_word(&self.raws[self.index]) { - return false; - } - let upper = self.uppers[self.index].as_str(); - is_reserved_keyword(dialect, upper) - || (self.next() == Some(CALL_OPENER) && is_builtin_function(dialect, upper)) - } - - fn next(&self) -> Option<&str> { - self.uppers.get(self.index + 1).map(String::as_str) - } -} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/constants.rs b/crates/sqlbuild-analysis/src/sql_tokens/constants.rs index df33fd5897..44e22ce088 100644 --- a/crates/sqlbuild-analysis/src/sql_tokens/constants.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/constants.rs @@ -1,38 +1,5 @@ -//! Token positions that decide which authored words may change letter case. +//! Fingerprint dialects and the function language fingerprinted like a query. -use polyglot_sql::tokens::TokenType; - -/// Column aliases after `AS` may be any keyword in DuckDB and PostgreSQL. -pub(crate) const ALIAS_KEYWORD: &str = "AS"; -/// The token that turns a preceding name into a function call. -pub(crate) const CALL_OPENER: &str = "("; -/// Field names after `.`, path keys after `:`, and parameter names after `@` or `$`. -pub(crate) const NAME_PREFIXES: [&str; 4] = [".", ":", "@", "$"]; -/// Built-in calls with keyword syntax inside their parentheses, which catalogues do not list. -pub const CALL_SYNTAX_FUNCTIONS: [&str; 6] = [ - "CAST", - "TRY_CAST", - "SAFE_CAST", - "EXTRACT", - "CONVERT", - "TRY_CONVERT", -]; - -pub(crate) const PROJECTION_ENDS: [TokenType; 13] = [ - TokenType::From, - TokenType::Where, - TokenType::Group, - TokenType::GroupBy, - TokenType::Having, - TokenType::Qualify, - TokenType::Order, - TokenType::OrderBy, - TokenType::Limit, - TokenType::Window, - TokenType::Union, - TokenType::Except, - TokenType::Intersect, -]; /// The fingerprint dialect when a project names none. pub(crate) const GENERIC_FINGERPRINT_DIALECT: &str = "generic"; /// Python's `NATIVE_DIALECT_ALIASES`. diff --git a/crates/sqlbuild-analysis/src/sql_tokens/main/builtin_function_names.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/builtin_function_names.rs deleted file mode 100644 index ec0c9e4260..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/main/builtin_function_names.rs +++ /dev/null @@ -1,6 +0,0 @@ -use polyglot_sql::DialectType; - -/// Every catalogued or parser-typed built-in function name of `dialect`, lower-cased. -pub fn builtin_function_names(dialect: DialectType) -> Vec { - crate::sql_tokens::_helpers::builtin_functions::builtin_function_names(dialect) -} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/main/canonical_tokens.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/canonical_tokens.rs deleted file mode 100644 index d1bfb837f9..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/main/canonical_tokens.rs +++ /dev/null @@ -1,25 +0,0 @@ -//! Canonical token stream of SQL text under one dialect tokenizer. - -use polyglot_sql::Dialect; -use polyglot_sql::tokens::Token; - -use crate::sql_tokens::main::case_folding::foldable_tokens; -use crate::sql_tokens::main::token_texts::token_texts; -use crate::sql_tokens::models::CanonicalToken; - -/// Drop whitespace and comments from tokenized `sql` and upper-case case-insensitive tokens. -pub fn canonical_tokens( - sql: &str, - tokens: &[Token], - dialect: &Dialect, -) -> Result, String> { - let raws = token_texts(sql, tokens)?; - let foldable = foldable_tokens(&raws, tokens, dialect); - Ok(raws - .into_iter() - .zip(foldable) - .map(|(raw, fold)| CanonicalToken { - text: if fold { raw.to_ascii_uppercase() } else { raw }, - }) - .collect()) -} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/main/case_folding.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/case_folding.rs deleted file mode 100644 index fc29290477..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/main/case_folding.rs +++ /dev/null @@ -1,23 +0,0 @@ -//! Which authored tokens are case-insensitive keywords or built-in function names. - -use polyglot_sql::Dialect; -use polyglot_sql::tokens::Token; - -use crate::sql_tokens::_helpers::token_context::TokenContext; -use crate::sql_tokens::main::name_positions::name_positions; - -/// Return, for every token, whether the formatter may recase it and the fingerprint folds it. -pub fn foldable_tokens(raws: &[String], tokens: &[Token], dialect: &Dialect) -> Vec { - let uppers: Vec = raws.iter().map(|raw| raw.to_ascii_uppercase()).collect(); - let names = name_positions(&uppers); - let mut foldable = Vec::with_capacity(tokens.len()); - for (index, name) in names.iter().enumerate().take(tokens.len()) { - let context = TokenContext { - raws, - uppers: &uppers, - index, - }; - foldable.push(!name && context.is_foldable(dialect.dialect_type())); - } - foldable -} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/main/is_unquoted_word.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/is_unquoted_word.rs deleted file mode 100644 index 6ea92ef370..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/main/is_unquoted_word.rs +++ /dev/null @@ -1,11 +0,0 @@ -//! Unquoted SQL word recognition shared by canonical tokens and the formatter. - -/// Return whether `text` is an unquoted ASCII word whose letter case carries no meaning. -pub fn is_unquoted_word(text: &str) -> bool { - let mut characters = text.chars(); - characters - .next() - .is_some_and(|first| first.is_ascii_alphabetic() || first == '_') - && characters - .all(|character| character.is_ascii_alphanumeric() || matches!(character, '_' | '$')) -} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/main/mod.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/mod.rs index 1c05934a81..7f7e8f95ad 100644 --- a/crates/sqlbuild-analysis/src/sql_tokens/main/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/main/mod.rs @@ -1,12 +1,4 @@ -#[cfg(any(test, feature = "test-support"))] -pub mod builtin_function_names; -pub mod canonical_tokens; -pub mod case_folding; pub mod compiler_query_fingerprint; pub mod definition_fingerprint; -pub mod is_unquoted_word; -pub mod name_positions; pub mod normalized_query_text; -pub mod projection_spans; pub mod query_fingerprint; -pub mod token_texts; diff --git a/crates/sqlbuild-analysis/src/sql_tokens/main/name_positions.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/name_positions.rs deleted file mode 100644 index b056a9bbef..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/main/name_positions.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Token positions where some dialect accepts even a reserved word as an unquoted name. - -use crate::sql_tokens::constants::{ALIAS_KEYWORD, NAME_PREFIXES}; - -/// Return, for every upper-cased token, whether it follows `.`, `:`, `@`, `$` or `AS`. -pub fn name_positions(uppers: &[String]) -> Vec { - let mut positions = Vec::with_capacity(uppers.len()); - let mut previous: Option<&str> = None; - for upper in uppers { - positions.push( - previous.is_some_and(|word| word == ALIAS_KEYWORD || NAME_PREFIXES.contains(&word)), - ); - previous = Some(upper.as_str()); - } - positions -} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/main/projection_spans.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/projection_spans.rs deleted file mode 100644 index 5965bd7875..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/main/projection_spans.rs +++ /dev/null @@ -1,41 +0,0 @@ -//! Authored spans of the top-level items of one select list. - -use polyglot_sql::tokens::{Token, TokenType}; - -use crate::sql_tokens::constants::PROJECTION_ENDS; - -/// Spans of each top-level projection item of the select starting at token `select`. -pub fn projection_spans(tokens: &[Token], select: usize) -> Vec<(usize, usize)> { - let mut spans: Vec<(usize, usize)> = Vec::new(); - let mut depth: usize = 0; - let mut start: Option = None; - let mut end: usize = 0; - for token in tokens.iter().skip(select + 1) { - if matches!( - token.token_type, - TokenType::Space | TokenType::Break | TokenType::LineComment | TokenType::BlockComment - ) { - continue; - } - match token.token_type { - TokenType::LParen => depth += 1, - TokenType::RParen if depth == 0 => break, - TokenType::RParen => depth -= 1, - TokenType::Comma if depth == 0 => { - if let Some(first) = start.take() { - spans.push((first, end)); - } - continue; - } - kind if depth == 0 && PROJECTION_ENDS.contains(&kind) => break, - TokenType::Distinct | TokenType::All if depth == 0 && start.is_none() => continue, - _ => {} - } - start.get_or_insert(token.span.start); - end = token.span.end; - } - if let Some(first) = start { - spans.push((first, end)); - } - spans -} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/main/query_fingerprint.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/query_fingerprint.rs index 4cf7751adc..9cde608f7d 100644 --- a/crates/sqlbuild-analysis/src/sql_tokens/main/query_fingerprint.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/main/query_fingerprint.rs @@ -5,7 +5,7 @@ use std::str::FromStr; use polyglot_sql::{Dialect, DialectType}; use sha2::{Digest, Sha256}; -use crate::sql_tokens::main::canonical_tokens::canonical_tokens; +use sqlclean::sql_tokens::main::canonical_tokens::canonical_tokens; /// Return the SHA-256 hex digest of the canonical token stream of `sql` under `dialect_name`. pub fn query_fingerprint(sql: &str, dialect_name: &str) -> Result { diff --git a/crates/sqlbuild-analysis/src/sql_tokens/main/token_texts.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/token_texts.rs deleted file mode 100644 index 8467369ccd..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/main/token_texts.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Authored text of tokens, taken from their character spans. - -use polyglot_sql::tokens::Token; - -/// Return the authored text of every token in `sql`. -pub fn token_texts(sql: &str, tokens: &[Token]) -> Result, String> { - let characters: Vec = sql.chars().collect(); - let mut texts = Vec::with_capacity(tokens.len()); - for token in tokens { - let slice = characters - .get(token.span.start..token.span.end) - .ok_or_else(|| "SQL tokenizer returned an invalid token span".to_string())?; - texts.push(slice.iter().collect()); - } - Ok(texts) -} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/mod.rs b/crates/sqlbuild-analysis/src/sql_tokens/mod.rs index 9904d54d66..4d3a25f7d4 100644 --- a/crates/sqlbuild-analysis/src/sql_tokens/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/mod.rs @@ -1,9 +1,4 @@ -//! SQLBuild's canonical SQL token stream: authored tokens without layout or comments. +//! SQLBuild's query and definition fingerprints over SQLClean's canonical SQL token stream. -pub mod _helpers; pub mod constants; pub mod main; -pub mod models; - -#[cfg(test)] -mod tests; diff --git a/crates/sqlbuild-analysis/src/sql_tokens/models.rs b/crates/sqlbuild-analysis/src/sql_tokens/models.rs deleted file mode 100644 index b18ebb1a3a..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/models.rs +++ /dev/null @@ -1,7 +0,0 @@ -//! Canonical token values shared by the formatter invariant and change fingerprints. - -/// One authored token with layout removed: keywords and built-in function names upper-cased. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct CanonicalToken { - pub text: String, -} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/bigquery.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/bigquery.txt deleted file mode 100644 index ded970a614..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/bigquery.txt +++ /dev/null @@ -1,99 +0,0 @@ -# BigQuery (GoogleSQL) reserved keywords: 'Reserved keywords' in -# https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical -all -and -any -array -as -asc -assert_rows_modified -at -between -by -case -cast -collate -contains -create -cross -cube -current -default -define -desc -distinct -else -end -enum -escape -except -exclude -exists -extract -false -fetch -following -for -from -full -graph_table -group -grouping -groups -hash -having -if -ignore -in -inner -intersect -interval -into -is -join -lateral -left -like -limit -lookup -merge -natural -new -no -not -null -nulls -of -on -or -order -outer -over -partition -preceding -proto -qualify -range -recursive -respect -right -rollup -rows -select -set -some -struct -tablesample -then -to -treat -true -unbounded -union -unnest -using -when -where -window -with -within diff --git a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/databricks.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/databricks.txt deleted file mode 100644 index dc43281979..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/databricks.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Databricks reserves no keywords as identifiers: -# https://docs.databricks.com/sql/language-manual/sql-ref-reserved-words -# ('Databricks does not formally disallow any specific literals from being used as identifiers'). - diff --git a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/duckdb.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/duckdb.txt deleted file mode 100644 index 4374c4e538..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/duckdb.txt +++ /dev/null @@ -1,77 +0,0 @@ -# DuckDB reserved keywords: SELECT keyword_name FROM duckdb_keywords() -# WHERE keyword_category = 'reserved' (DuckDB 1.5.2). -all -analyse -analyze -and -any -array -as -asc -asymmetric -both -case -cast -check -collate -column -constraint -create -default -deferrable -desc -describe -distinct -do -else -end -except -false -fetch -for -foreign -from -group -having -in -initially -intersect -into -lambda -lateral -leading -limit -not -null -offset -on -only -or -order -pivot -pivot_longer -pivot_wider -placing -primary -qualify -references -returning -select -show -some -summarize -symmetric -table -then -to -trailing -true -union -unique -unpivot -using -variadic -when -where -window -with diff --git a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/generic.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/generic.txt deleted file mode 100644 index 05f8ee42cd..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/generic.txt +++ /dev/null @@ -1,28 +0,0 @@ -# Generic dialect: keywords reserved in every dialect file above except Databricks -# (DuckDB, PostgreSQL, BigQuery, Snowflake and T-SQL). -all -and -any -as -create -distinct -else -for -from -group -having -in -intersect -into -not -null -on -or -order -select -some -then -to -union -where -with diff --git a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/postgres.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/postgres.txt deleted file mode 100644 index 7a57911cb5..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/postgres.txt +++ /dev/null @@ -1,80 +0,0 @@ -# PostgreSQL reserved keywords (non-function): RESERVED_KEYWORD entries of -# src/include/parser/kwlist.h (REL_17_STABLE); Appendix C 'reserved'. -all -analyse -analyze -and -any -array -as -asc -asymmetric -both -case -cast -check -collate -column -constraint -create -current_catalog -current_date -current_role -current_time -current_timestamp -current_user -default -deferrable -desc -distinct -do -else -end -except -false -fetch -for -foreign -from -grant -group -having -in -initially -intersect -into -lateral -leading -limit -localtime -localtimestamp -not -null -offset -on -only -or -order -placing -primary -references -returning -select -session_user -some -symmetric -system_user -table -then -to -trailing -true -union -unique -user -using -variadic -when -where -window -with diff --git a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/snowflake.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/snowflake.txt deleted file mode 100644 index 511b64668b..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/snowflake.txt +++ /dev/null @@ -1,65 +0,0 @@ -# Snowflake reserved keywords: entries marked 'Reserved by ANSI', 'Reserved by Snowflake' -# or 'Reserved by Snowflake and others' in https://docs.snowflake.com/en/sql-reference/reserved-keywords. -# Limited keywords (usable as identifiers in some positions) are excluded. -all -alter -and -any -as -between -by -check -column -connect -create -current -delete -distinct -drop -else -exists -following -for -from -grant -group -having -ilike -in -increment -insert -intersect -into -is -like -minus -not -null -of -on -or -order -qualify -regexp -revoke -rlike -row -rows -sample -select -set -some -start -table -tablesample -then -to -trigger -union -unique -update -values -whenever -where -window -with diff --git a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/tsql.txt b/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/tsql.txt deleted file mode 100644 index b83a35b59d..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/reserved_keywords/tsql.txt +++ /dev/null @@ -1,186 +0,0 @@ -# T-SQL reserved keywords (SQL Server and Azure Synapse table) in -# https://learn.microsoft.com/sql/t-sql/language-elements/reserved-keywords-transact-sql -add -all -alter -and -any -as -asc -authorization -backup -begin -between -break -browse -bulk -by -cascade -case -check -checkpoint -close -clustered -coalesce -collate -column -commit -compute -constraint -contains -containstable -continue -convert -create -cross -current -current_date -current_time -current_timestamp -current_user -cursor -database -dbcc -deallocate -declare -default -delete -deny -desc -disk -distinct -distributed -double -drop -dump -else -end -errlvl -escape -except -exec -execute -exists -exit -external -fetch -file -fillfactor -for -foreign -freetext -freetexttable -from -full -function -goto -grant -group -having -holdlock -identity -identity_insert -identitycol -if -in -index -inner -insert -intersect -into -is -join -key -kill -left -like -lineno -load -merge -national -nocheck -nonclustered -not -null -nullif -of -off -offsets -on -open -opendatasource -openquery -openrowset -openxml -option -or -order -outer -over -percent -pivot -plan -precision -primary -print -proc -procedure -public -raiserror -read -readtext -reconfigure -references -replication -restore -restrict -return -revert -revoke -right -rollback -rowcount -rowguidcol -rule -save -schema -securityaudit -select -semantickeyphrasetable -semanticsimilaritydetailstable -semanticsimilaritytable -session_user -set -setuser -shutdown -some -statistics -system_user -table -tablesample -textsize -then -to -top -tran -transaction -trigger -truncate -try_convert -tsequal -union -unique -unpivot -update -updatetext -use -user -values -varying -view -waitfor -when -where -while -with -writetext diff --git a/crates/sqlbuild-analysis/src/sql_tokens/tests.rs b/crates/sqlbuild-analysis/src/sql_tokens/tests.rs deleted file mode 100644 index 5468fc5562..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/tests.rs +++ /dev/null @@ -1,4 +0,0 @@ -#[path = "tests/test_canonical_tokens.rs"] -mod canonical_tokens; -#[path = "tests/test_types.rs"] -mod test_types; diff --git a/crates/sqlbuild-analysis/src/sql_tokens/tests/test_canonical_tokens.rs b/crates/sqlbuild-analysis/src/sql_tokens/tests/test_canonical_tokens.rs deleted file mode 100644 index 2dec3bf4cc..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/tests/test_canonical_tokens.rs +++ /dev/null @@ -1,218 +0,0 @@ -use polyglot_sql::{Dialect, DialectType}; - -use crate::sql_tokens::main::canonical_tokens::canonical_tokens; -use crate::sql_tokens::tests::test_types::CanonicalTokensTestCase; - -#[test] -fn given_sql_pairs_when_canonicalizing_then_only_layout_comments_and_word_case_are_ignored() --> Result<(), String> { - let test_cases = [ - CanonicalTokensTestCase { - description: "whitespace between tokens is ignored", - dialect: DialectType::DuckDB, - left: "select a,b from orders", - right: "SELECT\n a, b\nFROM orders", - expected_equal: true, - }, - CanonicalTokensTestCase { - description: "comments are ignored", - dialect: DialectType::Snowflake, - left: "select a -- note\nfrom orders /* block */", - right: "select a from orders", - expected_equal: true, - }, - CanonicalTokensTestCase { - description: "keyword and built-in function case is ignored", - dialect: DialectType::Snowflake, - left: "select count(*) from orders", - right: "SELECT COUNT(*) FROM orders", - expected_equal: true, - }, - CanonicalTokensTestCase { - description: "unquoted identifier case is significant", - dialect: DialectType::Snowflake, - left: "select count(*) from orders", - right: "SELECT COUNT(*) FROM ORDERS", - expected_equal: false, - }, - CanonicalTokensTestCase { - description: "Snowflake path key case is significant", - dialect: DialectType::Snowflake, - left: "select payload:customerId from t", - right: "select payload:customerid from t", - expected_equal: false, - }, - CanonicalTokensTestCase { - description: "BigQuery qualified table case is significant", - dialect: DialectType::BigQuery, - left: "select * from ds.Orders", - right: "select * from ds.orders", - expected_equal: false, - }, - CanonicalTokensTestCase { - description: "DuckDB alias case is significant", - dialect: DialectType::DuckDB, - left: "select 1 as Foo", - right: "select 1 as foo", - expected_equal: false, - }, - CanonicalTokensTestCase { - description: "keyword-named qualified name case is significant", - dialect: DialectType::BigQuery, - left: "select * from inventory.view", - right: "select * from inventory.VIEW", - expected_equal: false, - }, - CanonicalTokensTestCase { - description: "keyword-named alias case is significant", - dialect: DialectType::DuckDB, - left: "select pos as index from t", - right: "select pos AS INDEX from t", - expected_equal: false, - }, - CanonicalTokensTestCase { - description: "user-defined function case is significant", - dialect: DialectType::DuckDB, - left: "select my_udf(a) from t", - right: "select MY_UDF(a) from t", - expected_equal: false, - }, - CanonicalTokensTestCase { - description: "built-in function case is ignored", - dialect: DialectType::DuckDB, - left: "select coalesce(a, 0) from t", - right: "select COALESCE(a, 0) from t", - expected_equal: true, - }, - CanonicalTokensTestCase { - description: "Snowflake double-slash comments are ignored", - dialect: DialectType::Snowflake, - left: "select a // note\nfrom t", - right: "select a from t", - expected_equal: true, - }, - CanonicalTokensTestCase { - description: "BigQuery hash comments are ignored", - dialect: DialectType::BigQuery, - left: "select a # note\nfrom t", - right: "select a from t", - expected_equal: true, - }, - CanonicalTokensTestCase { - description: "DuckDB struct-literal key case is significant", - dialect: DialectType::DuckDB, - left: "select {Key: 1}", - right: "select {key: 1}", - expected_equal: false, - }, - CanonicalTokensTestCase { - description: "a non-reserved keyword operand before an operator keeps its case", - dialect: DialectType::DuckDB, - left: "select a, Rows ^ 2 from t", - right: "select a, rows ^ 2 from t", - expected_equal: false, - }, - CanonicalTokensTestCase { - description: "a non-reserved keyword table in a comma join keeps its case", - dialect: DialectType::BigQuery, - left: "select * from ds.a, View v", - right: "select * from ds.a, view v", - expected_equal: false, - }, - CanonicalTokensTestCase { - description: "a reserved word after a dot is a field name", - dialect: DialectType::BigQuery, - left: "select abc.Group from abc", - right: "select abc.group from abc", - expected_equal: false, - }, - CanonicalTokensTestCase { - description: "a reserved keyword column alias keeps its case", - dialect: DialectType::DuckDB, - left: "select 1 as Select", - right: "select 1 as select", - expected_equal: false, - }, - CanonicalTokensTestCase { - description: "reserved keyword case is ignored", - dialect: DialectType::DuckDB, - left: "select a from t where a in (1) limit 1", - right: "SELECT a FROM t WHERE a IN (1) LIMIT 1", - expected_equal: true, - }, - CanonicalTokensTestCase { - description: "Databricks reserves no keywords", - dialect: DialectType::Databricks, - left: "select a from t", - right: "SELECT a FROM t", - expected_equal: false, - }, - CanonicalTokensTestCase { - description: "whitespace inside a string literal is significant", - dialect: DialectType::DuckDB, - left: "select 'a b' as code", - right: "select 'a b' as code", - expected_equal: false, - }, - CanonicalTokensTestCase { - description: "string literal case is significant", - dialect: DialectType::DuckDB, - left: "select 'Open' as status", - right: "select 'OPEN' as status", - expected_equal: false, - }, - CanonicalTokensTestCase { - description: "quoted identifier case is significant", - dialect: DialectType::Snowflake, - left: "select \"Order\" from orders", - right: "select \"ORDER\" from orders", - expected_equal: false, - }, - CanonicalTokensTestCase { - description: "dollar-quoted literal content is significant", - dialect: DialectType::PostgreSQL, - left: "select $$a b$$ as note", - right: "select $$a b$$ as note", - expected_equal: false, - }, - CanonicalTokensTestCase { - description: "number spelling is significant", - dialect: DialectType::DuckDB, - left: "select 1.0 as amount", - right: "select 1.00 as amount", - expected_equal: false, - }, - CanonicalTokensTestCase { - description: "a renamed function is significant", - dialect: DialectType::Snowflake, - left: "select startswith(code, 'EU')", - right: "select starts_with(code, 'EU')", - expected_equal: false, - }, - CanonicalTokensTestCase { - description: "an added alias keyword is significant", - dialect: DialectType::DuckDB, - left: "select a b from orders", - right: "select a as b from orders", - expected_equal: false, - }, - ]; - for test_case in test_cases { - let dialect = Dialect::get(test_case.dialect); - let left_tokens = dialect - .tokenize(test_case.left) - .map_err(|error| error.to_string())?; - let right_tokens = dialect - .tokenize(test_case.right) - .map_err(|error| error.to_string())?; - let left = canonical_tokens(test_case.left, &left_tokens, &dialect)?; - let right = canonical_tokens(test_case.right, &right_tokens, &dialect)?; - assert_eq!( - left == right, - test_case.expected_equal, - "{}", - test_case.description - ); - } - Ok(()) -} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/tests/test_types.rs b/crates/sqlbuild-analysis/src/sql_tokens/tests/test_types.rs deleted file mode 100644 index 2e7671dfac..0000000000 --- a/crates/sqlbuild-analysis/src/sql_tokens/tests/test_types.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) struct CanonicalTokensTestCase { - pub(crate) description: &'static str, - pub(crate) dialect: polyglot_sql::DialectType, - pub(crate) left: &'static str, - pub(crate) right: &'static str, - pub(crate) expected_equal: bool, -} diff --git a/crates/sqlbuild-core/src/cli_text/main/format_compile_text.rs b/crates/sqlbuild-core/src/cli_text/main/format_compile_text.rs index 3c00f2252f..c00b5ca443 100644 --- a/crates/sqlbuild-core/src/cli_text/main/format_compile_text.rs +++ b/crates/sqlbuild-core/src/cli_text/main/format_compile_text.rs @@ -69,6 +69,10 @@ pub fn format_compile_text(report: &CompileTextReport) -> String { lines.push(diagnostics_text(&report.diagnostics, style)); lines.push(String::new()); } + if let Some(notice) = report.default_rules_notice.as_deref() { + lines.push(notice.to_owned()); + lines.push(String::new()); + } let summary: String = [ count_label(report.models.len(), "model"), count_label(report.seed_count, "seed"), diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index 6d22f84d96..572efa79f1 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -224,6 +224,8 @@ pub struct CompileTextReport { /// Every selected model, in project order. pub models: Vec, pub semantic_notice: Option, + /// How to turn off configurable Rules, when unselected default Rules reported a finding. + pub default_rules_notice: Option, /// The diagnostics shown, with repeated equal warnings already folded. pub diagnostics: Vec, pub seed_count: usize, diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs index f4f6cfd9a4..2820e5c46e 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs @@ -115,6 +115,7 @@ fn given_report_when_formatting_then_lists_models_and_summary() { column_count: 2, }], semantic_notice: None, + default_rules_notice: None, diagnostics: Vec::new(), seed_count: 1, function_count: 0, @@ -137,6 +138,7 @@ fn given_report_when_formatting_then_lists_models_and_summary() { column_count: 2, }], semantic_notice: None, + default_rules_notice: None, diagnostics: Vec::new(), seed_count: 0, function_count: 0, diff --git a/crates/sqlbuild-core/src/format_command/tests/helpers.rs b/crates/sqlbuild-core/src/format_command/tests/helpers.rs index cbcf23ea6f..95751b256c 100644 --- a/crates/sqlbuild-core/src/format_command/tests/helpers.rs +++ b/crates/sqlbuild-core/src/format_command/tests/helpers.rs @@ -12,7 +12,7 @@ pub(super) fn sample_report() -> LintReport { formatted_files: texts(&["/p/models/orders.sql"]), rule_fixes: vec![RuleFixLine { file: "/p/models/orders.sql".to_owned(), - code: "SQBRSQL002".to_owned(), + code: "SQLCLEAN002".to_owned(), line: 4, status: "applied".to_owned(), reason: "ok".to_owned(), @@ -25,7 +25,7 @@ pub(super) fn sample_report() -> LintReport { column: 8, end_line: Some(2), end_column: Some(15), - code: "SQBRSQL004".to_owned(), + code: "SQLCLEAN004".to_owned(), message: "Avoid LIMIT".to_owned(), fault: true, engine: "native".to_owned(), diff --git a/crates/sqlbuild-core/src/format_command/tests/test_format_command.rs b/crates/sqlbuild-core/src/format_command/tests/test_format_command.rs index 8c6d97144b..0360237550 100644 --- a/crates/sqlbuild-core/src/format_command/tests/test_format_command.rs +++ b/crates/sqlbuild-core/src/format_command/tests/test_format_command.rs @@ -14,8 +14,8 @@ fn given_format_report_when_rendering_then_text_and_json_match_python() { let test_cases = [ReportTestCase { description: "excerpt, wrapped help, location-only warning and a Rule fix", report: sample_report(), - expected_text: "Formatted files:\n /p/models/orders.sql\n\nerror[SQBRSQL004]: Avoid LIMIT\n --> models/orders.sql:2:8\n |\n2 | SELECT\t1 LIMIT 5\n | \t^^^^^^^\n |\n = help: Remove the LIMIT clause from the model body because the model must return every row it is\n expected to produce downstream.\n\nwarning[description-present]: Missing\n --> /other/x.sql:1:1\n\nAPPLIED SQBRSQL002 /p/models/orders.sql:4: ok\nCompleted. FAULT=1 WARN=1 FILES=3\n", - expected_json: "{\"faults\":1,\"files_checked\":3,\"formatted_files\":[\"/p/models/orders.sql\"],\"rule_fixes\":[{\"code\":\"SQBRSQL002\",\"file\":\"/p/models/orders.sql\",\"line\":4,\"reason\":\"ok\",\"status\":\"applied\"}],\"violations\":[{\"code\":\"SQBRSQL004\",\"column\":8,\"end_column\":15,\"end_line\":2,\"engine\":\"native\",\"file\":\"/p/models/orders.sql\",\"line\":2,\"message\":\"Avoid LIMIT\",\"remediation\":\"Remove the LIMIT clause from the model body because the model must return every row it is expected to produce downstream.\",\"severity\":\"fault\"},{\"code\":\"description-present\",\"column\":1,\"end_column\":null,\"end_line\":null,\"engine\":\"sqlbuild\",\"file\":\"/other/x.sql\",\"line\":1,\"message\":\"Missing\",\"remediation\":null,\"severity\":\"warning\"}],\"warnings\":1}\n", + expected_text: "Formatted files:\n /p/models/orders.sql\n\nerror[SQLCLEAN004]: Avoid LIMIT\n --> models/orders.sql:2:8\n |\n2 | SELECT\t1 LIMIT 5\n | \t^^^^^^^\n |\n = help: Remove the LIMIT clause from the model body because the model must return every row it is\n expected to produce downstream.\n\nwarning[description-present]: Missing\n --> /other/x.sql:1:1\n\nAPPLIED SQLCLEAN002 /p/models/orders.sql:4: ok\nCompleted. FAULT=1 WARN=1 FILES=3\n", + expected_json: "{\"faults\":1,\"files_checked\":3,\"formatted_files\":[\"/p/models/orders.sql\"],\"rule_fixes\":[{\"code\":\"SQLCLEAN002\",\"file\":\"/p/models/orders.sql\",\"line\":4,\"reason\":\"ok\",\"status\":\"applied\"}],\"violations\":[{\"code\":\"SQLCLEAN004\",\"column\":8,\"end_column\":15,\"end_line\":2,\"engine\":\"native\",\"file\":\"/p/models/orders.sql\",\"line\":2,\"message\":\"Avoid LIMIT\",\"remediation\":\"Remove the LIMIT clause from the model body because the model must return every row it is expected to produce downstream.\",\"severity\":\"fault\"},{\"code\":\"description-present\",\"column\":1,\"end_column\":null,\"end_line\":null,\"engine\":\"sqlbuild\",\"file\":\"/other/x.sql\",\"line\":1,\"message\":\"Missing\",\"remediation\":null,\"severity\":\"warning\"}],\"warnings\":1}\n", }]; for test_case in test_cases { diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 661bd3d6a1..2219287ac5 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -44,6 +44,7 @@ sqlbuild-rules.workspace = true sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true sqlbuild-state.workspace = true +sqlclean.workspace = true stacker.workspace = true [lints] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 3962b79206..9e2a381159 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -48,6 +48,7 @@ use crate::bindings::_helpers::records::construct::record_class; use crate::bindings::_helpers::records::construct::{error_class, new_error}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; +use crate::bindings::_helpers::rules::constants::DEFAULT_RULES_NOTICE_ATTR; use crate::bindings::_helpers::workspace_commands::commands::maintain_sqlbuild_skills; use sqlbuild_cli::launcher::constants::LAUNCH_RECORD_OPTION; use sqlbuild_core::cli_text::main::format_coded_error::format_coded_error; @@ -176,6 +177,14 @@ fn print_expected_error( use_color: invocation.use_color, }, ); + let notice: Option = match error.getattr_opt(DEFAULT_RULES_NOTICE_ATTR)? { + Some(notice) if !machine_output(invocation)? => Some(notice.extract()?), + _ => None, + }; + let text: String = match notice { + Some(notice) => format!("{text}\n\n{notice}"), + None => text, + }; print_object( &PyString::new(py, &format!("{text}\n")).into_any(), Some("stderr"), @@ -183,6 +192,14 @@ fn print_expected_error( ) } +/// Whether the command was asked for JSON output. +fn machine_output(invocation: &Invocation<'_>) -> PyResult { + match invocation.args.getattr_opt("json")? { + Some(json) => json.is_truthy(), + None => Ok(false), + } +} + /// The compile JSON error document `sqb compile --json` prints for a failure. fn write_compile_machine_error( invocation: &Invocation<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs index ef027c023e..6174faeaaa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -51,6 +51,7 @@ const REPORT_OPTIONS: (&[&str], &[&str]) = ( "sql_validation_enabled", "use_color", "timings_ms", + "default_rules_notice", ], ); const HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; @@ -404,6 +405,7 @@ pub(crate) fn format_compile_text<'py>( }, models, semantic_notice: inputs.semantic_notice(&reader)?, + default_rules_notice: arguments.value("default_rules_notice")?.extract()?, diagnostics: shown_diagnostics(&inputs, &reader)?, seed_count: inputs.selected_count(&[SEED_TYPE])?, function_count: inputs.selected_count(&FUNCTION_TYPES)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 3ca9c5226b..cf8472a4b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -156,8 +156,9 @@ pub(crate) fn analyze<'py>( let enabled: bool = !(native_field(selection, "select")?.is_truthy()? || native_field(selection, "exclude")?.is_truthy()?) && crate::bindings::_helpers::compile_context::delta::injection().is_none(); + let sql_lint_enabled: bool = enabled && !options.no_sql_validation.is_truthy()?; let early_lint: Bound<'py, EarlySqlLint> = - Bound::new(py, EarlySqlLint::new(enabled, options.no_cache()?))?; + Bound::new(py, EarlySqlLint::new(sql_lint_enabled, options.no_cache()?))?; let early_rules: EarlyRulesScope = EarlyRulesScope::new(); let analysis: PyResult> = analyzed(py, &options, (&early_lint, enabled.then_some(&early_rules))); @@ -383,7 +384,13 @@ impl<'py> Driver<'_, 'py> { )?; let on_inputs_ready: Bound<'py, PyAny> = imported(py, "functools", "partial")?.call( (self.early_lint.getattr("start")?,), - Some(&keywords(py, &[("dialect", lint_dialect.clone())])?), + Some(&keywords( + py, + &[ + ("dialect", lint_dialect.clone()), + ("value_renderer", adapter.clone()), + ], + )?), )?; let analysis_selection: Bound<'py, PyAny> = replaced( &options.analysis_selection.clone(), @@ -497,7 +504,8 @@ impl<'py> Driver<'_, 'py> { if !options.prepared_artifacts.is_none() && !any_error(core_diagnostics.as_any())? { let target_dir: Bound<'py, PyAny> = project_dir.call_method1("__truediv__", (TARGET_DIRECTORY,))?; - let stage_artifacts: bool = native_field(&rules_config, "select")?.is_truthy()?; + let select: Bound<'py, PyAny> = native_field(&rules_config, "select")?; + let stage_artifacts: bool = select.is_none() || select.is_truthy()?; call_with( &native_field(&options.prepared_artifacts, "start")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index d48526ba63..8c17251f89 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -5,7 +5,7 @@ use pyo3::sync::PyOnceLock; use pyo3::types::{ PyBool, PyDict, PyDictMethods, PyList, PyListMethods, PyMapping, PyString, PyTuple, }; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_host::compile_context::main::collect_compile_timings::collect_compile_timings; use sqlbuild_host::compile_context::main::timing_milliseconds::timing_milliseconds; use sqlbuild_host::compile_context::types::CompileTimingsHandle; @@ -43,6 +43,8 @@ use crate::bindings::_helpers::records::fields::replaced; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::rules::results::format_skipped_type_proof_rules; +use crate::bindings::_helpers::rules::rules_config::rules_config_of; +use sqlbuild_rules::rule_results::main::default_rules_notice::default_rules_notice; const BYPASS_OUTCOME: &str = "bypass"; @@ -588,6 +590,17 @@ fn record_output<'py>( PyBool::new(py, use_color).to_owned().into_any(), )); arguments.push(("sql_validation_enabled", validation_enabled)); + let project_dir: PathBuf = native_field(request, "project_dir")?.extract()?; + let select_omitted: bool = + matches!(rules_config_of(&project_dir)?, Ok(config) if config.select.is_none()); + let codes: Vec = diagnostics + .try_iter()? + .map(|diagnostic| native_field(&diagnostic?, "code")?.extract::>()) + .filter_map(Result::transpose) + .collect::>()?; + let notice: Option = + default_rules_notice(select_omitted, codes.iter().map(String::as_str)); + arguments.push(("default_rules_notice", notice.into_bound_py_any(py)?)); format_compile_text(py, Some(keywords(py, &arguments)?))? }; output.call_method1("record_stdout", (rendered,))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index c5a9f107fb..ddcf1ae6dd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -13,6 +13,7 @@ use crate::bindings::_helpers::diagnostics::trackers::current_tracker; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::{enum_member, new_record, new_record_with}; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::fields::replaced; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; /// The current build's phase timing tracker, when one is recording. @@ -51,6 +52,7 @@ pub(crate) fn compile_project_phase<'py>( ), &arguments.value("options")?, &arguments.value("on_progress")?, + None, ) } @@ -59,6 +61,7 @@ pub(crate) fn compiled_phase<'py>( (discovered, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), options: &Bound<'py, PyAny>, on_progress: &Bound<'py, PyAny>, + on_inputs_ready: Option<&Bound<'py, PyAny>>, ) -> PyResult> { let py: Python<'py> = discovered.py(); let options: Bound<'py, PyAny> = if options.is_truthy()? { @@ -80,6 +83,10 @@ pub(crate) fn compiled_phase<'py>( let started: f64 = monotonic(py)?; let compiled: PyResult> = (|| { let selection: Bound<'py, PyAny> = compile_analysis_selection(&options, discovered)?; + let selection: Bound<'py, PyAny> = match on_inputs_ready { + Some(hook) => replaced(&selection, &[("on_inputs_ready", hook.clone())])?, + None => selection, + }; build_compiled_project( py, Some(keywords( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs index 07e8d1173f..537fe036dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs @@ -4,8 +4,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::PyDict; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_rules::rule_results::main::default_rules_notice::default_rules_notice; -use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, keywords}; +use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, imported, keywords}; use crate::bindings::_helpers::compile_driver::phases::monotonic; use crate::bindings::_helpers::compile_driver::pipeline::{ compiled_phase, progress, timing_tracker, @@ -24,6 +25,8 @@ use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; +use crate::bindings::_helpers::rules::constants::DEFAULT_RULES_NOTICE_ATTR; +use crate::bindings::_helpers::rules::early_sql_lint::EarlySqlLint; use crate::bindings::_helpers::rules::results::{format_rules_result, rules_progress}; use crate::bindings::_helpers::rules::rule_dispatch::evaluate_rules; use crate::bindings::_helpers::rules::rules_config::load_rules_config_with_cache; @@ -54,12 +57,29 @@ pub(crate) fn run_compile_pipeline<'py>( hooks }; let on_progress: Bound<'py, PyAny> = native_field(&hooks, "on_progress")?; - let compiled: Bound<'py, PyAny> = - compiled_phase((&discovered, &adapter), &options, &on_progress)?; + let expansion_reuse: Bound<'py, EarlySqlLint> = + Bound::new(py, EarlySqlLint::new(false, false))?; + let on_inputs_ready: Bound<'py, PyAny> = imported(py, "functools", "partial")?.call( + (expansion_reuse.getattr("start")?,), + Some(&keywords( + py, + &[ + // Only expansion reuse is kept here; early lint is off, so the dialect is unused. + ("dialect", GENERIC_DIALECT.into_pyobject(py)?.into_any()), + ("value_renderer", adapter.clone()), + ], + )?), + )?; + let compiled: Bound<'py, PyAny> = compiled_phase( + (&discovered, &adapter), + &options, + &on_progress, + Some(&on_inputs_ready), + )?; let project: Bound<'py, PyAny> = native_field(&compiled, "project")?; let graph: Bound<'py, PyAny> = project_graph(&project)?; run_rules_gate( - (&discovered, &adapter), + (&discovered, &adapter, expansion_reuse.as_any()), (&options, &on_progress), (&project, &graph), )?; @@ -68,7 +88,11 @@ pub(crate) fn run_compile_pipeline<'py>( /// Run the configured rules and stop the command when any of them finds a fault. fn run_rules_gate<'py>( - (discovered, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (discovered, adapter, expansion_reuse): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), (options, on_progress): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), (project, graph): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult<()> { @@ -103,10 +127,11 @@ fn run_rules_gate<'py>( &[ ("graph", graph.clone()), ("discovered_inputs", discovered.clone()), - ("config", config), + ("config", config.clone()), ("project_dir", project_dir), ("dialect", dialect), ("selected_keys", selected_keys), + ("early_lint", expansion_reuse.clone()), ( "no_sql_analysis", native_field(options, "no_sql_validation")?, @@ -141,9 +166,17 @@ fn run_rules_gate<'py>( ], )?; let rendered: String = format_rules_result(&summary, false)?; - Err(PyErr::from_value(rules_error.call1((format!( - "configured compiler rules failed:\n{rendered}" - ),))?)) + let codes: Vec = native_field(&summary, "findings")? + .try_iter()? + .map(|finding| native_field(&finding?, "code")?.extract()) + .collect::>()?; + let error: Bound<'py, PyAny> = + rules_error.call1((format!("configured compiler rules failed:\n{rendered}"),))?; + let select_omitted: bool = native_field(&config, "select")?.is_none(); + if let Some(notice) = default_rules_notice(select_omitted, codes.iter().map(String::as_str)) { + error.setattr(DEFAULT_RULES_NOTICE_ATTR, notice)?; + } + Err(PyErr::from_value(error)) } /// Open the planning connection, plan, and record the compile and planning times. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs index 8967d13c1f..1b2dbe314c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs @@ -9,8 +9,8 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use sqlbuild_discovery::project_validation::main::at_most_one_csv_data_row::at_most_one_csv_data_row; use sqlbuild_discovery::project_validation::models::CsvSettings; -use sqlbuild_rules::sql_quality::main::declared_relation_keys::declared_relation_keys as derive_keys; -use sqlbuild_rules::sql_quality::models::{DeclaredKeyColumn, DeclaredRelation}; +use sqlbuild_rules::relation_keys::main::declared_relation_keys::declared_relation_keys as derive_keys; +use sqlbuild_rules::relation_keys::models::{DeclaredKeyColumn, DeclaredRelation}; use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs index 75fcfbcd84..e41a21ad38 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs @@ -262,7 +262,7 @@ const R_RULES_CONFIG: RecordSpec = RecordSpec { name: "RulesConfig", doc: "", fields: &[ - FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("select", FieldDefault::None), FieldSpec::new("ignore", FieldDefault::EmptyTuple), FieldSpec::new("thresholds", FieldDefault::EmptyDict), FieldSpec::new("threshold_overrides", FieldDefault::EmptyTuple), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs index 02b51f667e..1de57367a3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs @@ -56,7 +56,9 @@ pub(crate) fn native_rules_config(config: &Bound<'_, PyAny>) -> PyResult) -> PyResult { let mut selectors: Vec = Vec::new(); - for field in ["select", "ignore", "rule_options"] { + let select: Option> = record_field(config, "select")?.extract()?; + selectors.extend(select.unwrap_or_default()); + for field in ["ignore", "rule_options"] { for selector in record_field(config, field)?.try_iter()? { selectors.push(selector?.extract()?); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs index 71fe51be7d..69dd8a1a59 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs @@ -1,17 +1,8 @@ //! Authored SQL-test mode identifiers. pub(crate) const MODEL_TEST_MODE: &str = "model"; -/// The rule codes whose fixes `sqb format --fix` may apply after compiler verification. -pub(crate) const SAFE_RULE_FIX_CODES: [&str; 8] = [ - "SQBRSQL002", - "SQBRSQL003", - "SQBRSQL005", - "SQBRSQL006", - "SQBRSQL008", - "SQBRSQL017", - "SQBRSQL025", - "SQBRSQL042", -]; +/// The SQL Rules whose fixes may change a query's meaning, so `sqb format --fix` never applies them; as in SQLClean, every other SQL Rule fix is safe. +pub(crate) const UNSAFE_RULE_FIX_CODES: [&str; 1] = ["SQLCLEAN001"]; /// The note attached to a rule finding that has an available fix. pub(crate) const RULE_FIX_AVAILABLE_NOTE: &str = "Fix available: `sqb format --fix` applies it after compiler verification"; @@ -19,7 +10,7 @@ pub(crate) const RULES_NATIVE_API_VERSION: i64 = 1; /// File name of the custom-rule result cache under the project cache directory. pub(crate) const CUSTOM_RULES_CACHE_FILE: &str = "custom-rules.json"; /// Code prefix of the SQL lint rules, which check each selected file on its own. -pub(crate) const SQL_RULE_PREFIX: &str = "SQBRSQL"; +pub(crate) const SQL_RULE_PREFIX: &str = "SQLCLEAN"; /// Suffix of a custom host scratch directory abandoned by a cancelled run. pub(crate) const CUSTOM_HOST_ABANDONED_SUFFIX: &str = ".abandoned"; /// Marker file whose presence tells split custom hosts the run was cancelled. @@ -31,3 +22,5 @@ pub(crate) const CUSTOM_HOST_RUNTIME_VERSION: &str = "sqlbuild-rules-custom-v3"; /// Native rules evaluations answered from the response memo. pub(crate) const RULES_MEMO_REUSES: &str = "rules_memo_reuses"; +/// The `RulesError` attribute holding how to turn off default Rules, printed only with human error text. +pub(crate) const DEFAULT_RULES_NOTICE_ATTR: &str = "default_rules_notice"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index ce9c2b3e41..e7e331c2d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -105,14 +105,11 @@ fn early_lint_plan( return Ok(None); }; config.cache.enabled &= !no_cache; - if config.select.is_empty() { - return Ok(None); - } if config.cache.enabled && !stored_entries(&sql_rule_cache_path(&project_dir))?.is_empty() { return Ok(None); } let codes: Vec = - compiler_guard(|| Ok(selected_sql_codes(&config.select, &config.ignore)))?; + compiler_guard(|| Ok(selected_sql_codes(config.select.as_deref(), &config.ignore)))?; if codes.is_empty() { return Ok(None); } @@ -214,6 +211,7 @@ fn same_codes(left: &[String], right: &[String]) -> bool { /// Start early SQL lint on a native thread when the project selects SQL rules without a cache. fn start_early_lint( (inputs, dialect): (&Bound<'_, PyAny>, &str), + value_renderer: &Bound<'_, PyAny>, no_cache: bool, stop: &Arc, ) -> PyResult> { @@ -231,6 +229,7 @@ fn start_early_lint( settings: plan.settings, discovered: record_field(inputs, "discovered_inputs")?.unbind(), static_scope: record_field(inputs, "declaration_scope")?.unbind(), + value_renderer: value_renderer.clone().unbind(), compiled, stop: Arc::clone(stop), }; @@ -253,6 +252,7 @@ struct EarlyLintWork { settings: LintSettings, discovered: Py, static_scope: Py, + value_renderer: Py, compiled: HashMap, stop: Arc, } @@ -262,7 +262,7 @@ fn early_lint(py: Python<'_>, work: &EarlyLintWork) -> PyResult let project_dir: &Path = &work.project_dir; let context: SqlExpansionInputs = lint_expansion_inputs(LintExpansionSources { project_dir: project_dir.into_pyobject(py)?.into_any(), - value_renderer: None, + value_renderer: Some(work.value_renderer.bind(py).clone()), discovered_inputs: Some(work.discovered.bind(py).clone()), declaration_scope: None, static_declaration_scope: present(work.static_scope.bind(py).clone()), @@ -284,10 +284,11 @@ fn early_lint(py: Python<'_>, work: &EarlyLintWork) -> PyResult Ok(EarlyLintResult { outcome, context }) } -/// The compiler's discovery and declaration scope, which lint expansion reuses for the same discovery. +/// The compiler's discovery, declaration scope and adapter, which lint expansion reuses for the same discovery. pub(crate) struct ExpansionReuse { pub(crate) discovered_inputs: Py, pub(crate) declaration_scope: Py, + pub(crate) value_renderer: Py, } /// Own speculative SQL checks for one compilation invocation. @@ -359,21 +360,22 @@ impl EarlySqlLint { let _ = self.stop(py); } - /// Start early SQL lint for `inputs` when enabled and the project selects SQL rules. - #[pyo3(signature = (*, inputs, dialect))] + /// Keep the compile's expansion inputs, and start early SQL lint when enabled and SQL rules are selected. + #[pyo3(signature = (*, inputs, dialect, value_renderer))] fn start( mut slf: PyRefMut<'_, Self>, inputs: &Bound<'_, PyAny>, dialect: &str, + value_renderer: &Bound<'_, PyAny>, ) -> PyResult<()> { - if !slf.enabled { - return Ok(()); - } let py: Python<'_> = inputs.py(); - let stop: Arc = Arc::clone(&slf.stop_requested); - slf.preparation = start_early_lint((inputs, dialect), slf.no_cache, &stop)? - .map(|preparation| Py::new(py, preparation)) - .transpose()?; + if slf.enabled { + let stop: Arc = Arc::clone(&slf.stop_requested); + slf.preparation = + start_early_lint((inputs, dialect), value_renderer, slf.no_cache, &stop)? + .map(|preparation| Py::new(py, preparation)) + .transpose()?; + } let declaration_scope: Bound<'_, PyAny> = record_field(inputs, "declaration_scope")?; slf.expansion_reuse = if declaration_scope.is_none() { None @@ -381,6 +383,7 @@ impl EarlySqlLint { Some(ExpansionReuse { discovered_inputs: record_field(inputs, "discovered_inputs")?.unbind(), declaration_scope: declaration_scope.unbind(), + value_renderer: value_renderer.clone().unbind(), }) }; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index 04800a40a1..378561660d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -28,8 +28,8 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::{field, replaced}; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::render::authored_sql::SqlExpansionInputs; -use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; use crate::bindings::_helpers::rules::constants::SQL_RULE_PREFIX; +use crate::bindings::_helpers::rules::constants::UNSAFE_RULE_FIX_CODES; use crate::bindings::_helpers::rules::fix_facts::compile_facts; use crate::bindings::_helpers::rules::fix_verdict::fix_verdict; use crate::bindings::_helpers::rules::lint_expansion::{ @@ -214,7 +214,7 @@ impl<'py> FixSession<'py> { for violation in violations { if !field(&violation, "code")? .extract::()? - .starts_with("SQBRSQL") + .starts_with("SQLCLEAN") { continue; } @@ -272,8 +272,8 @@ impl<'py> FixSession<'py> { let facts = FixEligibility { model_body: model_paths.contains(&path)?, has_fix, - safe_code: SAFE_RULE_FIX_CODES.contains(&code.extract::()?.as_str()), - conditionless_join: code.eq("SQBRSQL003")?, + safe_code: !UNSAFE_RULE_FIX_CODES.contains(&code.extract::()?.as_str()), + conditionless_join: code.eq("SQLCLEAN003")?, snowflake: field(&self.config, "dialect")?.eq("snowflake")?, }; if let Some(reason) = refusal(&facts) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs index f0fb419415..a8c9f3894f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_rules::lint_text::models::InterpolationSite; -use sqlbuild_rules::sql_lint::types::PreparedSql; +use sqlbuild_rules::lint_text::types::PreparedSql; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::compiled_views::relation_keys; @@ -12,7 +12,7 @@ use crate::bindings::types::CompilerDetach; #[pyfunction] fn lint_sql_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| sqlbuild_rules::sql_lint::main::engine::lint_json(request_json)) + py.compiler_detach(|| sqlclean::sql_lint::main::engine::lint_json(request_json)) .map_err(value_error) } @@ -36,7 +36,7 @@ fn prepare_lint_sql( ) -> PyResult> { let prepared: Option = py .compiler_detach(|| { - sqlbuild_rules::sql_lint::main::preparation::prepare( + sqlbuild_rules::lint_text::main::preparation::prepare( &request.expanded, &request.before_expansion, &request.prior_sites, @@ -55,7 +55,7 @@ fn prepare_lint_sql( #[pyfunction] fn format_sql_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| sqlbuild_rules::sql_lint::main::formatter::format_json(request_json)) + py.compiler_detach(|| sqlclean::sql_lint::main::formatter::format_json(request_json)) .map_err(value_error) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs index 78155a1522..5b97c37151 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs @@ -22,6 +22,7 @@ use sqlbuild_rules::dsl_headers::models::{ HeaderRulesConfig, HeaderSeverity, HeaderSpan, HeaderViolation, }; use sqlbuild_rules::lint_text::constants::SUPPRESSION_CODE; +use sqlbuild_rules::lint_text::main::defined_cte_names::defined_cte_names; use sqlbuild_rules::lint_text::main::dependency_sites::dependency_sites; use sqlbuild_rules::lint_text::main::first_header_only::first_header_only; use sqlbuild_rules::lint_text::main::frame_sql_macro_expression::frame_sql_macro_expression; @@ -31,17 +32,16 @@ use sqlbuild_rules::lint_text::main::neutralize_audit_parameters::neutralize_aud use sqlbuild_rules::lint_text::main::neutralize_context_interpolation::neutralize_context_interpolation; use sqlbuild_rules::lint_text::main::neutralize_sql_macro_parameters::neutralize_sql_macro_parameters; use sqlbuild_rules::lint_text::main::neutralize_test_parameters::neutralize_test_parameters; +use sqlbuild_rules::lint_text::main::preparation::prepare; use sqlbuild_rules::lint_text::main::reject_inline_suppressions::reject_inline_suppressions; use sqlbuild_rules::lint_text::models::{ AuthoredDiagnostic, ExpansionSpan, FramedExpression, InterpolationSite, LintSqlBody, LintSqlOptions, Neutralized, SuppressionFinding, }; +use sqlbuild_rules::lint_text::types::PreparedSql; use sqlbuild_rules::project_tree::main::project_real_path::project_real_path; use sqlbuild_rules::project_tree::main::project_real_paths::project_real_paths; -use sqlbuild_rules::sql_lint::main::defined_cte_names::defined_cte_names; -use sqlbuild_rules::sql_lint::main::fill_finding_path::fill_finding_path; -use sqlbuild_rules::sql_lint::main::preparation::prepare; -use sqlbuild_rules::sql_lint::types::PreparedSql; +use sqlclean::sql_lint::main::fill_finding_path::fill_finding_path; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::boundary::python_calls::call_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/results.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/results.rs index 2363561727..aac6fab787 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/results.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/results.rs @@ -1,4 +1,4 @@ -//! Rules results rendered for the CLI, and SQBRSQL044 help naming files that repeat a literal. +//! Rules results rendered for the CLI, and SQLCLEAN044 help naming files that repeat a literal. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; use pyo3::{pyfunction, wrap_pyfunction}; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs index af2973d9ae..db398f032d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs @@ -89,6 +89,18 @@ pub(crate) fn rule_run_plan<'py>( } selected = kept; } + if no_sql_analysis { + let mut kept: Vec> = Vec::with_capacity(selected.len()); + for rule in selected { + if !field(&rule, "code")? + .extract::()? + .starts_with(SQL_RULE_PREFIX) + { + kept.push(rule); + } + } + selected = kept; + } let mut custom_rules: Vec> = Vec::new(); for rule in &selected { if field(rule, "custom")?.is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs index a59d0e0da2..b3f87cb241 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs @@ -180,7 +180,13 @@ fn config_object<'py>(py: Python<'py>, config: &RulesConfig) -> PyResult strings(py, select)?, + None => py.None().into_bound(py), + }, + ), ("ignore", strings(py, &config.ignore)?), ("thresholds", integers(py, &config.thresholds)?), ("threshold_overrides", threshold_overrides), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs index ebe0c454aa..e761aeb0d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs @@ -7,6 +7,7 @@ use pyo3::prelude::{ use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_rules::rules::main::selected_codes::selected_codes_json; +use sqlbuild_rules::rules::main::selected_sql_codes::selected_sql_codes; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::records::construct::error_class; @@ -80,20 +81,20 @@ fn selected_codes(py: Python<'_>, encoded: &str) -> PyResult> { }) } -/// Whether `code` is selected and not ignored by the configuration's prefixes. +/// Whether `code` is selected and not ignored by the configuration's prefixes; without `select`, only SQLClean's default Rules are. #[pyfunction] #[pyo3(signature = (*, config, code))] pub(crate) fn rule_selected(config: &Bound<'_, PyAny>, code: &str) -> PyResult { - let matches = |field: &str| -> PyResult { - for selector in record_field(config, field)?.try_iter()? { - let selector: String = selector?.extract()?; - if code.starts_with(&selector) { - return Ok(true); - } - } - Ok(false) - }; - Ok(matches("select")? && !matches("ignore")?) + let select: Option> = record_field(config, "select")?.extract()?; + let ignore: Vec = record_field(config, "ignore")?.extract()?; + let matches = + |selectors: &[String]| selectors.iter().any(|selector| code.starts_with(selector)); + Ok(match select { + None => selected_sql_codes(None, &ignore) + .iter() + .any(|selected| selected == code), + Some(select) => matches(&select) && !matches(&ignore), + }) } /// `RulesError(message)`, caused by `cause` when given. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index 4587763c2e..3e0a63232b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -29,10 +29,11 @@ use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; -use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; use crate::bindings::_helpers::rules::constants::SQL_RULE_PREFIX; +use crate::bindings::_helpers::rules::constants::UNSAFE_RULE_FIX_CODES; use crate::bindings::_helpers::rules::early_sql_lint::{ - EarlyLintResult, EarlySqlLint, PreparedSqlLint, sql_lint_settings, sql_rule_cache_path, + EarlyLintResult, EarlySqlLint, ExpansionReuse, PreparedSqlLint, sql_lint_settings, + sql_rule_cache_path, }; use crate::bindings::_helpers::rules::fact_replay::project_attribute; use crate::bindings::_helpers::rules::lint_expansion::{ @@ -52,7 +53,7 @@ const DATACLASSES_MODULE: &str = "dataclasses"; const JSON_MODULE: &str = "json"; const BUILTINS_MODULE: &str = "builtins"; const PATHLIB_MODULE: &str = "pathlib"; -const SQL_RULE_SUPPRESSION_CODE: &str = "SQBRSQL000"; +const SQL_RULE_SUPPRESSION_CODE: &str = "SQBRSUPPRESSION000"; const DEFAULT_REMEDIATION: &str = "Update the authored SQL to satisfy this rule."; const UTF8: &str = "utf-8"; @@ -615,18 +616,17 @@ fn lint_expansion_context( run: &SqlRun<'_, '_>, ) -> PyResult { let py: Python<'_> = run.project.py(); - let declaration_scope: Option> = run + let reuse: Option<&ExpansionReuse> = run .early_lint .as_ref() .and_then(|early| early.expansion_reuse()) - .filter(|reuse| reuse.discovered_inputs.bind(py).is(&run.discovered_inputs)) - .map(|reuse| reuse.declaration_scope.bind(py).clone()); + .filter(|reuse| reuse.discovered_inputs.bind(py).is(&run.discovered_inputs)); lint_expansion_inputs(LintExpansionSources { project_dir: run.project_dir.clone().into_pyobject(py)?.into_any(), - value_renderer: None, + value_renderer: reuse.map(|reuse| reuse.value_renderer.bind(py).clone()), discovered_inputs: (!run.discovered_inputs.is_none()) .then(|| run.discovered_inputs.clone()), - declaration_scope, + declaration_scope: reuse.map(|reuse| reuse.declaration_scope.bind(py).clone()), static_declaration_scope: None, }) } @@ -678,7 +678,7 @@ fn lint_finding( }; let path: String = posix(path); let fixable: bool = violation.fix.is_some() - && SAFE_RULE_FIX_CODES.contains(&violation.code.as_str()) + && !UNSAFE_RULE_FIX_CODES.contains(&violation.code.as_str()) && fixable_paths.contains(&path); RuleFinding { code: violation.code.clone(), diff --git a/crates/sqlbuild-rules/Cargo.toml b/crates/sqlbuild-rules/Cargo.toml index 559fa48c1a..535ee3687a 100644 --- a/crates/sqlbuild-rules/Cargo.toml +++ b/crates/sqlbuild-rules/Cargo.toml @@ -24,6 +24,7 @@ sqlbuild-cache.workspace = true sqlbuild-core.workspace = true sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true +sqlclean.workspace = true stacker.workspace = true toml.workspace = true @@ -31,7 +32,6 @@ toml.workspace = true sha2.workspace = true [dev-dependencies] -sqlbuild-analysis = { workspace = true, features = ["test-support"] } tempfile.workspace = true [lints] diff --git a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs index dcf05f036f..5017562819 100644 --- a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs +++ b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs @@ -237,7 +237,7 @@ pub(crate) fn validate(config: &RulesConfig) -> Result<(), String> { } } let grammar = crate::models::RulesCodeGrammar; - for selector in config.select.iter().chain(&config.ignore) { + for selector in config.select.iter().flatten().chain(&config.ignore) { if !grammar.rule_selector_is_valid(selector) { return Err(format!("malformed rule selector: {selector}")); } diff --git a/crates/sqlbuild-rules/src/constants.rs b/crates/sqlbuild-rules/src/constants.rs index 6952d90a7a..250c806f9d 100644 --- a/crates/sqlbuild-rules/src/constants.rs +++ b/crates/sqlbuild-rules/src/constants.rs @@ -26,15 +26,14 @@ pub(crate) const BOOLEAN_TYPE: &str = "BOOLEAN"; pub(crate) const DATE_TYPE: &str = "DATE"; pub(crate) const DECLARATION_DOMAIN_COMPONENTS: usize = 3; pub(crate) const RULE_CODE_NUMBER_LENGTH: usize = 3; -pub(crate) const EMPTY_FIXTURE_QUERY_TOKEN_COUNT: usize = 6; -pub(crate) const ENCLOSING_CTE_TOKEN_COUNT: usize = 3; pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; pub(crate) const NATIVE_ROWS_MEMO_VERSION: &str = "native-rules-rows-response-v2"; -pub(crate) const SQL_RULE_PREFIX: &str = "SQBRSQL"; +/// The namespace of the SQL Rules SQLClean owns, whose codes are `SQLCLEAN` and three digits. +pub(crate) const SQL_RULE_PREFIX: &str = "SQLCLEAN"; /// Threshold key setting the fewest test cases a custom rule must ship. pub const MIN_CUSTOM_RULE_TEST_CASES_KEY: &str = "min_custom_rule_test_cases"; /// Rules whose findings name a joined relation, so an exception may cover just that relation. -pub(crate) const RELATION_EXCEPTION_RULES: [&str; 1] = ["SQBRSQL038"]; +pub(crate) const RELATION_EXCEPTION_RULES: [&str; 1] = ["SQLCLEAN038"]; /// The layers the `____` model name grammar accepts. pub(crate) const MODEL_NAME_LAYERS: [&str; 7] = [ "stg", diff --git a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs index d38f6cb21d..64d11d4ead 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs @@ -12,7 +12,6 @@ use crate::rules::main::{ use crate::rules::models::{ ModelEvaluationRequest, ProjectEvaluationRequest, QueryShapes, ResolvedThresholdOverride, }; -use crate::sql_lint::main::cross_join_subject::cross_join_subject; use fensu_policy::apply_suppressions; use fensu_policy::lifecycle::errors::LifecycleError; use fensu_policy::lifecycle::models::{ @@ -21,6 +20,7 @@ use fensu_policy::lifecycle::models::{ #[cfg(test)] use rayon::iter::IndexedParallelIterator; use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; +use sqlclean::sql_lint::main::cross_join_subject::cross_join_subject; use std::cell::RefCell; use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; @@ -103,7 +103,7 @@ fn evaluate_native_models( if !context .selected .values() - .any(|rule| !rule.custom && !rule.code.starts_with("SQBRSQL")) + .any(|rule| !rule.custom && !rule.code.starts_with("SQLCLEAN")) { return Ok(NativeModelEvaluation::default()); } @@ -111,7 +111,7 @@ fn evaluate_native_models( models.sort_by(|left, right| left.relative_path.cmp(&right.relative_path)); let all_models_required = context.selected.values().any(|rule| { !rule.custom - && !rule.code.starts_with("SQBRSQL") + && !rule.code.starts_with("SQLCLEAN") && !matches!( rule.code.as_str(), "SQBRDECLARATION201" | "SQBRDECLARATION301" | "SQBRTEST301" @@ -364,7 +364,11 @@ fn evaluated_request( } config::validate(&request.config)?; let all_rules = assemble_catalogue::assemble_catalogue(&request.custom_rules)?; - let selected = select::select(&all_rules, &request.config.select, &request.config.ignore)?; + let selected = select::select( + &all_rules, + request.config.select.as_deref(), + &request.config.ignore, + )?; let request = if selected .iter() .any(|rule| matches!(rule.code.as_str(), "SQBRTEST202" | "SQBRTEST203")) diff --git a/crates/sqlbuild-rules/src/engine/tests/helpers.rs b/crates/sqlbuild-rules/src/engine/tests/helpers.rs index 85be479b67..b0c48450d6 100644 --- a/crates/sqlbuild-rules/src/engine/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/engine/tests/helpers.rs @@ -800,7 +800,7 @@ pub(crate) fn published_layer_faults( /// The model path the relation-exception cases use. pub(crate) const CROSS_JOIN_MODEL_PATH: &str = "models/orders.sql"; /// The Rule whose findings name a joined relation. -pub(crate) const CROSS_JOIN_CODE: &str = "SQBRSQL038"; +pub(crate) const CROSS_JOIN_CODE: &str = "SQLCLEAN038"; fn cross_join_fault(message: &str, line: u64) -> Fault { Fault { diff --git a/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs index 79678b5728..2fa5b42902 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs @@ -66,7 +66,7 @@ fn given_built_in_catalogue_when_reading_rule_scopes_then_every_rule_matches_rev for test_case in &test_cases { let declared: BTreeMap = catalogue() .into_iter() - .filter(|rule| !rule.code.starts_with("SQBRSQL")) + .filter(|rule| !rule.code.starts_with("SQLCLEAN")) .map(|rule| (rule.code, rule.scope)) .collect(); let declared: Vec<(&str, RuleScope)> = declared @@ -75,7 +75,7 @@ fn given_built_in_catalogue_when_reading_rule_scopes_then_every_rule_matches_rev .collect(); let sql_scopes_match = catalogue() .iter() - .filter(|rule| rule.code.starts_with("SQBRSQL")) + .filter(|rule| rule.code.starts_with("SQLCLEAN")) .all(|rule| rule.scope == test_case.expected_sql_scope); assert_eq!( diff --git a/crates/sqlbuild-rules/src/engine/tests/test_relation_exceptions.rs b/crates/sqlbuild-rules/src/engine/tests/test_relation_exceptions.rs index 46ec32a515..ae855bee86 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_relation_exceptions.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_relation_exceptions.rs @@ -50,12 +50,12 @@ fn given_relation_level_exceptions_matching_nothing_when_finalizing_then_they_ar StaleRelationExceptionTestCase { description: "a relation exception that matches nothing is stale", exceptions: json!([{"rule": CROSS_JOIN_CODE, "path": CROSS_JOIN_MODEL_PATH, "relation": "shipments", "reason": "Gone."}]), - expected_error: "stale rule exception suppresses no finding: SQBRSQL038 at models/orders.sql (relation shipments)", + expected_error: "stale rule exception suppresses no finding: SQLCLEAN038 at models/orders.sql (relation shipments)", }, StaleRelationExceptionTestCase { description: "a CTE-scoped exception for another CTE is stale", exceptions: json!([{"rule": CROSS_JOIN_CODE, "path": CROSS_JOIN_MODEL_PATH, "relation": "customers", "cte": "other", "reason": "Gone."}]), - expected_error: "stale rule exception suppresses no finding: SQBRSQL038 at models/orders.sql (relation customers in CTE other)", + expected_error: "stale rule exception suppresses no finding: SQLCLEAN038 at models/orders.sql (relation customers in CTE other)", }, ]; diff --git a/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs b/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs index bf3affa354..8c32966ca2 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs @@ -1,9 +1,9 @@ use crate::engine::tests::test_types::{ DialectCallsTestCase, DollarQuoteSiteTestCase, LintDialectSiteTestCase, SqlScannerTestCase, }; +use crate::lint_text::_helpers::preparation::prepare; use crate::query_facts::main::model_shape::model_shape; use crate::rules::_helpers::numeric_decisions::compact_sql; -use crate::sql_lint::_helpers::preparation::prepare; use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren; use sqlbuild_sqltext::sql_scan::main::quote_policy::quote_policy as rules_quote_policy; use sqlbuild_sqltext::sql_scan::models::QuotePolicy; diff --git a/crates/sqlbuild-rules/src/lib.rs b/crates/sqlbuild-rules/src/lib.rs index fdf32f0c54..7f34fa03b2 100644 --- a/crates/sqlbuild-rules/src/lib.rs +++ b/crates/sqlbuild-rules/src/lib.rs @@ -1,4 +1,4 @@ -//! Built-in rules, SQL lint, quality checks and formatting, with their configuration. +//! Built-in Rules and their configuration, hosting SQLClean's SQL lint, quality Rules and formatter. #![forbid(unsafe_code)] @@ -14,11 +14,10 @@ pub mod lint_text; pub mod models; pub mod project_tree; pub mod query_facts; +pub mod relation_keys; pub mod rule_cache; pub mod rule_catalogue; pub mod rule_results; pub mod rules; pub mod scope_metadata; -pub mod sql_lint; -pub mod sql_quality; pub mod types; diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs index e753aa7778..11e6a221d8 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs @@ -25,7 +25,7 @@ use crate::lint_text::errors::LintTextError; use crate::lint_text::main::neutralize_interpolation::neutralize_interpolation; use crate::lint_text::main::restore_interpolation::restore_interpolation; use crate::lint_text::models::{FormatFault, FormatFile, InterpolationSite, PreparedBody}; -use crate::sql_lint::models::{FormatRequest, FormatResponse}; +use sqlclean::sql_lint::models::{FormatRequest, FormatResponse}; static OMITTED_SELECT_SUFFIX: LazyLock> = LazyLock::new(|| Regex::new(OMITTED_SELECT_SUFFIX_PATTERN)); @@ -60,6 +60,7 @@ impl RequestTable { dialect: options.0.to_string(), max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH), line_width: Some(options.1), + indent: None, token_widths: widths.into_iter().collect(), }); let _ = self.index.insert(key, self.requests.len() - 1); diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/mod.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/mod.rs index a666bb0e05..f96b7b7983 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod fixture_formatting; pub(crate) mod fixtures; pub(crate) mod format_bodies; pub(crate) mod lines; +pub(crate) mod preparation; pub(crate) mod settings; pub(crate) mod sites; pub(crate) mod sql_diagnostics; diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/preparation.rs similarity index 98% rename from crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs rename to crates/sqlbuild-rules/src/lint_text/_helpers/preparation.rs index 227cc69725..dc41a4cdcd 100644 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/preparation.rs @@ -7,7 +7,7 @@ use regex::Regex; use crate::lint_text::main::neutralize_interpolation::neutralize_interpolation; use crate::lint_text::models::Neutralized; -use crate::sql_lint::types::PreparedSql; +use crate::lint_text::types::PreparedSql; use sqlbuild_sqltext::sql_scan::main::quote_policy::quote_policy; static CTES: LazyLock> = LazyLock::new(|| { diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sites.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sites.rs index a4c0c051f2..f4f342eaba 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sites.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sites.rs @@ -165,7 +165,7 @@ fn paren_end(bytes: &[u8], open: usize, policy: QuotePolicy) -> Option { } } -/// A numbered sentinel such as `__sqb_lint_0__`. +/// A numbered sentinel such as `__sqlclean_interpolation_0__`. pub(crate) fn sentinel(prefix: &str, index: usize) -> String { format!("{prefix}{index}{SENTINEL_SUFFIX}") } diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs index 57ad766e7a..577a1e5c81 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs @@ -15,8 +15,8 @@ use crate::lint_text::constants::{ PARSE_ERROR_POSITION_PATTERN, PARSE_FAILURE_PREFIX, PARSE_FAILURE_REMEDIATION, SQL_MACRO_PARAMETER_SENTINEL_PREFIX, UNSUPPORTED_VERSION, UNUSED_CTE_CODE, }; +use crate::lint_text::main::backtick_identifiers::backtick_identifiers; use crate::lint_text::models::{AuthoredDiagnostic, AuthoredEdit, ExpansionSpan, LintSqlBody}; -use crate::sql_lint::main::backtick_identifiers::backtick_identifiers; use sqlbuild_analysis::sql_macros::constants::SQL_CALL_PREFIX; use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs index 7835d03b1d..f3237e5385 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs @@ -5,17 +5,19 @@ use std::borrow::Cow; use serde_json::{Map, Value}; use sqlbuild_attachments::ceremonial_select::main::omitted_select_offset::omitted_select_offset; use sqlbuild_attachments::ceremonial_select::models::OmittedSelect; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::lint_text::_helpers::sql_diagnostics::{ sql_macro_output_spans, without_generated_comments, }; use crate::lint_text::constants::{ - CROSS_KEYWORD, CTE_ARGUMENT, INVALID_BATCH, LINE_COMMENT_PREFIX, LINT_API_VERSION, - OMITTED_CEREMONIAL_SELECT_SQL, + CROSS_KEYWORD, CTE_ARGUMENT, EMPTY_FIXTURE_RELATION, FIXTURE_CTE_MARKERS, + FIXTURE_HARNESS_CTE_PREFIXES, INVALID_BATCH, LINE_COMMENT_PREFIX, LINT_API_VERSION, + LONG_LITERAL_GUIDANCE, OMITTED_CEREMONIAL_SELECT_SQL, }; use crate::lint_text::models::{LintSqlBody, LintSqlOptions}; -use crate::sql_lint::main::batch_engine::lint_batch_json; +use sqlclean::sql_lint::main::batch_engine::lint_batch_json; /// Where a fixture body omits its ceremonial `SELECT 1`, so the parser sees one statement. pub(crate) fn omitted_select(body: &LintSqlBody) -> Option { @@ -96,6 +98,26 @@ fn top_level_arguments(arguments: &str) -> Vec<&str> { parts } +/// The harness SQLBuild wraps a test or macro body in; its inputs may read the empty fixture. +fn fixture_harness(body: &LintSqlBody) -> Value { + let mut harness: Map = Map::new(); + let _ = harness.insert( + "cte_prefixes".to_string(), + Value::from(FIXTURE_HARNESS_CTE_PREFIXES.to_vec()), + ); + if body.allows_empty_fixture_star { + let _ = harness.insert( + "input_cte_prefixes".to_string(), + Value::from(FIXTURE_CTE_MARKERS.to_vec()), + ); + let _ = harness.insert( + "empty_relation".to_string(), + Value::from(EMPTY_FIXTURE_RELATION), + ); + } + Value::Object(harness) +} + fn strings(values: &[String]) -> Value { Value::from(values.to_vec()) } @@ -131,6 +153,14 @@ pub(crate) fn lint_payload( let _ = payload.insert("version".to_string(), Value::from(LINT_API_VERSION)); let _ = payload.insert("sql".to_string(), Value::from(sql)); let _ = payload.insert("dialect".to_string(), Value::from(options.dialect.clone())); + let _ = payload.insert( + "config_file".to_string(), + Value::from(PROJECT_CONFIG_FILENAME), + ); + let _ = payload.insert( + "long_literal_guidance".to_string(), + Value::from(LONG_LITERAL_GUIDANCE), + ); if !outputs.is_empty() { let _ = payload.insert("sql_macro_outputs".to_string(), Value::from(outputs)); } @@ -151,18 +181,11 @@ pub(crate) fn lint_payload( let _ = payload.insert(key.to_string(), strings(values)); } } - let flags: [(&str, bool); 3] = [ - ("allows_ceremonial_select", body.allows_ceremonial_select), - ( - "allows_dynamic_output_star", - body.allows_dynamic_output_star, - ), - ("allows_empty_fixture_star", body.allows_empty_fixture_star), - ]; - for (key, enabled) in flags { - if enabled { - let _ = payload.insert(key.to_string(), Value::Bool(true)); - } + if body.allows_dynamic_output_star { + let _ = payload.insert("allows_dynamic_output_star".to_string(), Value::Bool(true)); + } + if body.allows_ceremonial_select { + let _ = payload.insert("fixture_harness".to_string(), fixture_harness(body)); } let _ = payload.insert( "max_literal_length".to_string(), diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs index f49086dad9..39092eed11 100644 --- a/crates/sqlbuild-rules/src/lint_text/constants.rs +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -1,11 +1,15 @@ //! Sentinel spellings, suppression texts and fixture-null markers. /// Sentinel standing in for one interpolation site in lint and format SQL. -pub const LINT_SENTINEL_PREFIX: &str = "__sqb_lint_"; +pub const LINT_SENTINEL_PREFIX: &str = sqlclean::constants::INTERPOLATION_PLACEHOLDER_PREFIX; pub(crate) const SENTINEL_SUFFIX: &str = "__"; -pub(crate) const AUDIT_PARAMETER_SENTINEL_PREFIX: &str = "__sqlbuild_audit_parameter_"; -pub(crate) const CONTEXT_SENTINEL_PREFIX: &str = "__sqlbuild_context_parameter_"; -pub(crate) const SQL_MACRO_PARAMETER_SENTINEL_PREFIX: &str = "__sqlbuild_macro_parameter_"; +/// An audit's relation parameter, which a `SELECT *` may read alone. +pub(crate) const AUDIT_PARAMETER_SENTINEL_PREFIX: &str = + sqlclean::constants::RELATION_PARAMETER_PLACEHOLDER_PREFIX; +pub(crate) const CONTEXT_SENTINEL_PREFIX: &str = + sqlclean::constants::VALUE_PARAMETER_PLACEHOLDER_PREFIX; +pub(crate) const SQL_MACRO_PARAMETER_SENTINEL_PREFIX: &str = + sqlclean::constants::MACRO_PARAMETER_PLACEHOLDER_PREFIX; pub(crate) const CONTEXT_INTERPOLATION_TOKEN: &str = "@@CTX:"; pub(crate) const TEST_PARAMETER_SENTINEL_PREFIX: &str = "__sqlbuild_test_parameter_"; pub(crate) const TEST_PARAMETER_TOKEN: &str = "@param"; @@ -29,8 +33,11 @@ pub(crate) const LINT_SITE_PATTERN: &str = r"^(?:@@|\$\{|@|(?:__dbt_ref|__ref|__seed|__source|__table_fn|__udf)\()"; pub(crate) const FORMAT_SITE_PATTERN: &str = r"(?i)^(?:@@|\$\{|@|(?:__cursor_end|__cursor_start|__dbt_ref|__empty_fixture|__ref|__seed|__source|__table_fn|__udf)\s*\()"; +/// SQLCLEAN044 advice pointing at SQLBuild constants, in place of SQLClean's generic advice. +pub(crate) const LONG_LITERAL_GUIDANCE: &str = "Define the value once as a constant and reference it with `@const`, in the narrowest scope that covers every use. (1) Only this model: add `constants (_status_pattern '')` to its `MODEL (...)` header and write `@const(\"_status_pattern\")`. (2) Other files use the same value: declare `CONSTANT (name status_pattern, value '');` in `_sqlbuild/_constants/` of the nearest folder that contains every use when they all sit directly in that folder, or in its `_sqlbuild/constants/` to include subfolders, and write `@const(\"status_pattern\")`. (3) Use top-level `constants/` only when the value is needed project-wide. Otherwise shorten the literal."; + /// Diagnostic code of every inline suppression directive finding. -pub const SUPPRESSION_CODE: &str = "SQBRSQL000"; +pub const SUPPRESSION_CODE: &str = "SQBRSUPPRESSION000"; /// Text every directive contains, checked before scanning a file's comments. pub(crate) const DIRECTIVE_MARKER: &[u8] = b"sqb:"; pub(crate) const BLOCK_COMMENT_OPENER: &str = "/*"; @@ -39,7 +46,7 @@ pub(crate) const BLOCK_COMMENT_CLOSER: &str = "*/"; pub(crate) const COMMENT_OPENER_BYTES: usize = 2; pub(crate) const DIRECTIVE_PATTERN: &str = r"(?i)sqb:\s*ignore\b\s*(?P[A-Za-z0-9_-]*)(?:\s+because\s+(?P.*\S))?"; -pub(crate) const CROSS_JOIN_CODE: &str = "SQBRSQL038"; +pub(crate) const CROSS_JOIN_CODE: &str = "SQLCLEAN038"; pub(crate) const CROSS_JOIN_RELATION_PATTERN: &str = r#"(?i)\bcross\s+join\s+(?:__(?:ref|seed|source)\s*\(\s*['"](?P[^'"]+)['"]\s*\)|(?P[A-Za-z_][A-Za-z0-9_$.]*))"#; pub(crate) const DIRECTIVE_MESSAGE: &str = "Inline Rule suppressions are not supported; declare the exception in sqlbuild_project.toml"; @@ -91,9 +98,9 @@ pub(crate) const GROUPED_NAMED_ROLES: [(&[&str], bool); 9] = [ ]; pub(crate) const LINT_API_VERSION: i64 = 1; -pub(crate) const UNUSED_CTE_CODE: &str = "SQBRSQL005"; -pub(crate) const LONG_LITERAL_CODE: &str = "SQBRSQL044"; -pub(crate) const MIXED_QUALIFICATION_CODE: &str = "SQBRSQL028"; +pub(crate) const UNUSED_CTE_CODE: &str = "SQLCLEAN005"; +pub(crate) const LONG_LITERAL_CODE: &str = "SQLCLEAN044"; +pub(crate) const MIXED_QUALIFICATION_CODE: &str = "SQLCLEAN028"; /// The reserved argument naming a CTE-list call's CTEs; it passes no relation. pub(crate) const CTE_ARGUMENT: &str = "__cte"; /// The help for mixed qualification in a SQL macro body, where slot reads are qualified through an alias. @@ -116,6 +123,22 @@ pub(crate) const HARNESS_CTE_PREFIXES: [&str; 13] = [ "__table_fn_actual__", "__table_fn_expected__", ]; +/// The CTE names a test or macro harness generates, which stay reachable when the body does not read them. +pub(crate) const FIXTURE_HARNESS_CTE_PREFIXES: [&str; 11] = [ + "__expected__", + "__assert__", + "__ref__", + "__source__", + "__seed__", + "__dbt_ref__", + "__macro_actual__", + "__macro_expected__", + "__udf_actual__", + "__udf_expected__", + "__table_fn_", +]; +/// The table function an empty fixture input reads, as `SELECT * FROM __empty_fixture()`. +pub(crate) const EMPTY_FIXTURE_RELATION: &str = "__empty_fixture"; pub(crate) const PARSE_FAILURE_PREFIX: &str = "Native SQL parser could not analyze this body: "; pub(crate) const PARSE_FAILURE_REMEDIATION: &str = "Use SQL syntax supported by the native parser or report the parser gap."; diff --git a/crates/sqlbuild-rules/src/sql_lint/main/backtick_identifiers.rs b/crates/sqlbuild-rules/src/lint_text/main/backtick_identifiers.rs similarity index 64% rename from crates/sqlbuild-rules/src/sql_lint/main/backtick_identifiers.rs rename to crates/sqlbuild-rules/src/lint_text/main/backtick_identifiers.rs index 79670356f6..c3006c0bf2 100644 --- a/crates/sqlbuild-rules/src/sql_lint/main/backtick_identifiers.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/backtick_identifiers.rs @@ -1,4 +1,4 @@ /// Return whether SQL lint treats backticks as identifier quotes for `dialect`. pub fn backtick_identifiers(dialect: &str) -> bool { - crate::sql_lint::_helpers::preparation::backtick_identifiers(dialect) + crate::lint_text::_helpers::preparation::backtick_identifiers(dialect) } diff --git a/crates/sqlbuild-rules/src/sql_lint/main/defined_cte_names.rs b/crates/sqlbuild-rules/src/lint_text/main/defined_cte_names.rs similarity index 75% rename from crates/sqlbuild-rules/src/sql_lint/main/defined_cte_names.rs rename to crates/sqlbuild-rules/src/lint_text/main/defined_cte_names.rs index c7177d64a9..a3a26cc297 100644 --- a/crates/sqlbuild-rules/src/sql_lint/main/defined_cte_names.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/defined_cte_names.rs @@ -3,5 +3,5 @@ /// Every CTE name `sql` defines, lowercased, in order and without repeats. #[must_use] pub fn defined_cte_names(sql: &str) -> Vec { - crate::sql_lint::_helpers::preparation::defined_cte_names(sql) + crate::lint_text::_helpers::preparation::defined_cte_names(sql) } diff --git a/crates/sqlbuild-rules/src/lint_text/main/format_sql_bodies.rs b/crates/sqlbuild-rules/src/lint_text/main/format_sql_bodies.rs index 6bf08424e0..674a0de7a9 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/format_sql_bodies.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/format_sql_bodies.rs @@ -7,8 +7,8 @@ use crate::lint_text::_helpers::format_bodies::{ }; use crate::lint_text::errors::LintTextError; use crate::lint_text::models::{FormatFault, FormatFile, FormattedBodies, PreparedBody}; -use crate::sql_lint::main::format_requests::format_requests; -use crate::sql_lint::models::FormatResponse; +use sqlclean::sql_lint::main::format_requests::format_requests; +use sqlclean::sql_lint::models::FormatResponse; /// Format SQL bodies, reporting every declined body as a fault and leaving it unchanged. pub fn format_sql_bodies( diff --git a/crates/sqlbuild-rules/src/lint_text/main/mod.rs b/crates/sqlbuild-rules/src/lint_text/main/mod.rs index 73885850a0..ae5b947776 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/mod.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/mod.rs @@ -1,3 +1,5 @@ +pub mod backtick_identifiers; +pub mod defined_cte_names; pub mod dependency_sites; pub mod first_header_only; pub mod format_fixture_blocks; @@ -11,6 +13,7 @@ pub mod neutralize_context_interpolation; pub mod neutralize_interpolation; pub mod neutralize_sql_macro_parameters; pub mod neutralize_test_parameters; +pub mod preparation; pub mod reject_inline_suppressions; pub mod resolve_format_settings; pub mod restore_interpolation; diff --git a/crates/sqlbuild-rules/src/lint_text/main/neutralize_interpolation.rs b/crates/sqlbuild-rules/src/lint_text/main/neutralize_interpolation.rs index 2cf0ef8f43..4bc32e3492 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/neutralize_interpolation.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/neutralize_interpolation.rs @@ -7,8 +7,8 @@ use crate::lint_text::_helpers::sites::{ }; use crate::lint_text::constants::LINT_SENTINEL_PREFIX; use crate::lint_text::errors::LintTextError; +use crate::lint_text::main::backtick_identifiers::backtick_identifiers; use crate::lint_text::models::Neutralized; -use crate::sql_lint::main::backtick_identifiers::backtick_identifiers; /// Neutralize references for lint, or every intrinsic spelling in any case for formatting. pub fn neutralize_interpolation( diff --git a/crates/sqlbuild-rules/src/sql_lint/main/preparation.rs b/crates/sqlbuild-rules/src/lint_text/main/preparation.rs similarity index 76% rename from crates/sqlbuild-rules/src/sql_lint/main/preparation.rs rename to crates/sqlbuild-rules/src/lint_text/main/preparation.rs index 71987caa78..680974de92 100644 --- a/crates/sqlbuild-rules/src/sql_lint/main/preparation.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/preparation.rs @@ -1,6 +1,6 @@ //! Native lexical preparation with Python-compatible ASCII offsets. -use crate::sql_lint::types::PreparedSql; +use crate::lint_text::types::PreparedSql; pub fn prepare( expanded: &str, @@ -8,7 +8,7 @@ pub fn prepare( prior_sites: &[usize], dialect: &str, ) -> Result, String> { - crate::sql_lint::_helpers::preparation::prepare( + crate::lint_text::_helpers::preparation::prepare( expanded, before_expansion, prior_sites, diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs index 0139da59b2..454ce07874 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs @@ -23,8 +23,8 @@ use crate::lint_text::tests::test_types::{ use sqlbuild_analysis::sql_macros::main::parse_sql_macro::parse_sql_macro; const SUPPRESSED_FILE: &str = - "-- sqb: ignore SQBRSQL005 because kept\n\n-- note\nselect 1\n-- sqb: ignore nope\nselect 2\n"; -const CROSS_JOIN_FILE: &str = "select o.order_id\nfrom orders as o\n-- sqb: ignore SQBRSQL038 because every order meets every policy\ncross join __seed(\"order_policy\") as p\n"; + "-- sqb: ignore SQLCLEAN005 because kept\n\n-- note\nselect 1\n-- sqb: ignore nope\nselect 2\n"; +const CROSS_JOIN_FILE: &str = "select o.order_id\nfrom orders as o\n-- sqb: ignore SQLCLEAN038 because every order meets every policy\ncross join __seed(\"order_policy\") as p\n"; #[test] fn given_site_patterns_when_text_opens_with_another_byte_then_no_site_starts_there() @@ -68,7 +68,7 @@ fn given_authored_sql_when_neutralizing_then_sites_use_character_offsets() description: "reference, macro call and template", body: "select @py.m(a, ')') from __ref('o') where x > ${v}", formatting: false, - expected_text: "select __sqb_lint_0__ from __sqb_lint_1__ where x > __sqb_lint_2__", + expected_text: "select __sqlclean_interpolation_0__ from __sqlclean_interpolation_1__ where x > __sqlclean_interpolation_2__", expected_sites: vec![ ("@py.m(a, ')')", 7, 20), ("__ref('o')", 26, 36), @@ -79,7 +79,7 @@ fn given_authored_sql_when_neutralizing_then_sites_use_character_offsets() description: "non-ASCII text before a site shifts character offsets only", body: "select 'é' as n, @@CTX:run.id", formatting: false, - expected_text: "select 'é' as n, __sqb_lint_0__", + expected_text: "select 'é' as n, __sqlclean_interpolation_0__", expected_sites: vec![("@@CTX:run.id", 17, 29)], }, NeutralizeTestCase { @@ -100,7 +100,7 @@ fn given_authored_sql_when_neutralizing_then_sites_use_character_offsets() description: "formatting protects every intrinsic spelling", body: "select __REF('a'), __ref ('b'), __cursor_start()", formatting: true, - expected_text: "select __sqb_lint_0__, __sqb_lint_1__, __sqb_lint_2__", + expected_text: "select __sqlclean_interpolation_0__, __sqlclean_interpolation_1__, __sqlclean_interpolation_2__", expected_sites: vec![ ("__REF('a')", 7, 17), ("__ref ('b')", 19, 30), @@ -140,31 +140,31 @@ fn given_generated_parameters_when_neutralizing_then_only_bare_parameters_are_re description: "quoted and raw audit parameters, but not calls or doubled at-signs", body: "select * from @this where @column < 0 and @'kind' = @py.f(1) and @@x", neutralize: neutralize_audit_parameters, - expected_text: "select * from __sqlbuild_audit_parameter_0__ where __sqlbuild_audit_parameter_1__ < 0 and '__sqlbuild_audit_parameter_2__' = @py.f(1) and @@x", + expected_text: "select * from __sqlclean_relation_parameter_0__ where __sqlclean_relation_parameter_1__ < 0 and '__sqlclean_relation_parameter_2__' = @py.f(1) and @@x", }, ParameterTestCase { description: "SQL macro parameters followed by words and a parenthesis", body: "@value IS NULL OR TRIM(@value) = ''", neutralize: |body| neutralize_sql_macro_parameters(body, &[]), - expected_text: "__sqlbuild_macro_parameter_0__ IS NULL OR TRIM(__sqlbuild_macro_parameter_1__) = ''", + expected_text: "__sqlclean_macro_parameter_0__ IS NULL OR TRIM(__sqlclean_macro_parameter_1__) = ''", }, ParameterTestCase { description: "SQL macro parameters before IN and a parenthesised predicate", body: "CASE WHEN @x IN ('a', 'b') THEN 1 END + CASE WHEN @x IS NULL AND (@y IS NULL) THEN 1 END", neutralize: |body| neutralize_sql_macro_parameters(body, &[]), - expected_text: "CASE WHEN __sqlbuild_macro_parameter_0__ IN ('a', 'b') THEN 1 END + CASE WHEN __sqlbuild_macro_parameter_1__ IS NULL AND (__sqlbuild_macro_parameter_2__ IS NULL) THEN 1 END", + expected_text: "CASE WHEN __sqlclean_macro_parameter_0__ IN ('a', 'b') THEN 1 END + CASE WHEN __sqlclean_macro_parameter_1__ IS NULL AND (__sqlclean_macro_parameter_2__ IS NULL) THEN 1 END", }, ParameterTestCase { description: "SQL macro slot reads hide the slot name, whole after an alias", body: "SELECT s.@src.amount AS amount, @src.id, @src.* FROM @src AS s", neutralize: |body| neutralize_sql_macro_parameters(body, &[]), - expected_text: "SELECT s.__sqlbuild_macro_parameter_0__ AS amount, __sqlbuild_macro_parameter_1__.__sqlbuild_macro_parameter_2__, __sqlbuild_macro_parameter_3__.* FROM __sqlbuild_macro_parameter_4__ AS s", + expected_text: "SELECT s.__sqlclean_macro_parameter_0__ AS amount, __sqlclean_macro_parameter_1__.__sqlclean_macro_parameter_2__, __sqlclean_macro_parameter_3__.* FROM __sqlclean_macro_parameter_4__ AS s", }, ParameterTestCase { description: "SQL macro calls, enum references and quoted text keep their at-signs", body: "@sql.other(@v) + @enum(\"s\").A + LENGTH('@v') + @@x", neutralize: |body| neutralize_sql_macro_parameters(body, &[]), - expected_text: "@sql.other(__sqlbuild_macro_parameter_0__) + @enum(\"s\").A + LENGTH('@v') + @@x", + expected_text: "@sql.other(__sqlclean_macro_parameter_0__) + @enum(\"s\").A + LENGTH('@v') + @@x", }, ParameterTestCase { description: "SQL test parameters, spaced or not, but not a malformed reference", @@ -176,7 +176,7 @@ fn given_generated_parameters_when_neutralizing_then_only_bare_parameters_are_re description: "context interpolation in hooks", body: "analyze @@CTX:relation; select @@CTX:a.b", neutralize: neutralize_context_interpolation, - expected_text: "analyze __sqlbuild_context_parameter_0__; select __sqlbuild_context_parameter_1__", + expected_text: "analyze __sqlclean_value_parameter_0__; select __sqlclean_value_parameter_1__", }, ]; @@ -199,17 +199,17 @@ fn given_relation_slot_reads_when_neutralizing_then_they_stand_as_their_expansio SlotReadTestCase { description: "a plain group read is one bare value", body: "SELECT @weekly.keys, orders FROM counted GROUP BY @weekly.keys", - expected_text: "SELECT __sqlbuild_macro_parameter_0__, orders FROM counted GROUP BY __sqlbuild_macro_parameter_1__", + expected_text: "SELECT __sqlclean_macro_parameter_0__, orders FROM counted GROUP BY __sqlclean_macro_parameter_1__", }, SlotReadTestCase { description: "plain single-slot and undeclared reads go through the one relation", body: "SELECT @weekly.refund_count, @weekly.week FROM @weekly WHERE @cutoff > @cutoff", - expected_text: "SELECT __sqlbuild_macro_parameter_0__.__sqlbuild_macro_parameter_1__, __sqlbuild_macro_parameter_0__.__sqlbuild_macro_parameter_2__ FROM __sqlbuild_macro_parameter_0__ WHERE __sqlbuild_macro_parameter_3__ > __sqlbuild_macro_parameter_4__", + expected_text: "SELECT __sqlclean_macro_parameter_0__.__sqlclean_macro_parameter_1__, __sqlclean_macro_parameter_0__.__sqlclean_macro_parameter_2__ FROM __sqlclean_macro_parameter_0__ WHERE __sqlclean_macro_parameter_3__ > __sqlclean_macro_parameter_4__", }, SlotReadTestCase { description: "aliased reads stay whole after their alias", body: "SELECT s.@weekly.keys, s.@weekly.refund_count FROM @weekly AS s", - expected_text: "SELECT s.__sqlbuild_macro_parameter_0__, s.__sqlbuild_macro_parameter_1__ FROM __sqlbuild_macro_parameter_2__ AS s", + expected_text: "SELECT s.__sqlclean_macro_parameter_0__, s.__sqlclean_macro_parameter_1__ FROM __sqlclean_macro_parameter_2__ AS s", }, ]; @@ -235,14 +235,14 @@ fn given_formatted_sql_when_restoring_then_each_sentinel_must_appear_once() RestoreTestCase { description: "sentinels in any case restore to the original text", body: "select @m, ${v}", - fixed: "SELECT __SQB_LINT_0__, __sqb_lint_1__", + fixed: "SELECT __SQLCLEAN_INTERPOLATION_0__, __sqlclean_interpolation_1__", expected_restored: Ok("SELECT @m, ${v}".to_string()), }, RestoreTestCase { description: "a duplicated sentinel is ambiguous", body: "select @m", - fixed: "select __sqb_lint_0__, __SQB_LINT_0__", - expected_restored: Err("formatted SQL contains 2 occurrences of sentinel '__sqb_lint_0__' standing in for '@m'; expected exactly 1".to_string()), + fixed: "select __sqlclean_interpolation_0__, __SQLCLEAN_INTERPOLATION_0__", + expected_restored: Err("formatted SQL contains 2 occurrences of sentinel '__sqlclean_interpolation_0__' standing in for '@m'; expected exactly 1".to_string()), }, ]; @@ -306,9 +306,9 @@ fn given_inline_directives_when_rejecting_then_each_prints_its_configured_except expected_findings: vec![ directive_finding( 1, - "SQBRSQL005", + "SQLCLEAN005", &[ - "rule = \"SQBRSQL005\"", + "rule = \"SQLCLEAN005\"", "path = \"models/orders.sql\"", "reason = \"kept\"", ], @@ -329,9 +329,9 @@ fn given_inline_directives_when_rejecting_then_each_prints_its_configured_except contents: CROSS_JOIN_FILE, expected_findings: vec![directive_finding( 3, - "SQBRSQL038", + "SQLCLEAN038", &[ - "rule = \"SQBRSQL038\"", + "rule = \"SQLCLEAN038\"", "path = \"models/orders.sql\"", "relation = \"order_policy\"", "reason = \"every order meets every policy\"", @@ -340,17 +340,17 @@ fn given_inline_directives_when_rejecting_then_each_prints_its_configured_except }, SuppressionTestCase { description: "a string literal mentioning a directive is not one", - contents: "select '-- sqb: ignore SQBRSQL004 because text' as note, 'sqb: ignore' as other\n", + contents: "select '-- sqb: ignore SQLCLEAN004 because text' as note, 'sqb: ignore' as other\n", expected_findings: vec![], }, SuppressionTestCase { description: "a trailing directive after code names the join on its line", - contents: "select o.order_id\nfrom orders as o\ncross join customers as c -- sqb: ignore SQBRSQL038 because deliberate\n", + contents: "select o.order_id\nfrom orders as o\ncross join customers as c -- sqb: ignore SQLCLEAN038 because deliberate\n", expected_findings: vec![directive_finding( 3, - "SQBRSQL038", + "SQLCLEAN038", &[ - "rule = \"SQBRSQL038\"", + "rule = \"SQLCLEAN038\"", "path = \"models/orders.sql\"", "relation = \"customers\"", "reason = \"deliberate\"", @@ -359,12 +359,12 @@ fn given_inline_directives_when_rejecting_then_each_prints_its_configured_except }, SuppressionTestCase { description: "a directive in a block comment", - contents: "select 1\n/* sqb: ignore SQBRSQL005 because kept */\nselect 2\n", + contents: "select 1\n/* sqb: ignore SQLCLEAN005 because kept */\nselect 2\n", expected_findings: vec![directive_finding( 2, - "SQBRSQL005", + "SQLCLEAN005", &[ - "rule = \"SQBRSQL005\"", + "rule = \"SQLCLEAN005\"", "path = \"models/orders.sql\"", "reason = \"kept\"", ], @@ -463,7 +463,7 @@ fn given_expanded_bodies_when_linting_then_findings_map_to_authored_lines() { .to_string(), ..LintSqlBody::default() }, - expected_findings: vec![("SQBRSQL005", 2, 6, Some((2, 10)))], + expected_findings: vec![("SQLCLEAN005", 2, 6, Some((2, 10)))], }, LintBodiesTestCase { description: "a parse failure is a fault at the mapped parser position", @@ -488,14 +488,14 @@ fn given_expanded_bodies_when_linting_then_findings_map_to_authored_lines() { }]], ..LintSqlBody::default() }, - expected_findings: vec![("SQBRSQL001", 1, 23, Some((1, 35)))], + expected_findings: vec![("SQLCLEAN001", 1, 23, Some((1, 35)))], }, ]; for test_case in test_cases { let options = LintSqlOptions { dialect: "duckdb".to_string(), - enabled_rules: Some(vec!["SQBRSQL001".to_string(), "SQBRSQL005".to_string()]), + enabled_rules: Some(vec!["SQLCLEAN001".to_string(), "SQLCLEAN005".to_string()]), max_literal_length: 100, max_ranking_order_by: 5, ..LintSqlOptions::default() @@ -579,7 +579,7 @@ fn given_sql_macro_bodies_when_linting_then_frames_and_generated_comments_are_no expression_frame: Some(null_comparison.frame), ..LintSqlBody::default() }, - expected_findings: vec![("SQBRSQL001", 1, 3, Some((1, 4)))], + expected_findings: vec![("SQLCLEAN001", 1, 3, Some((1, 4)))], }, LintBodiesTestCase { description: "comments a macro expansion generated are not judged in the caller", @@ -595,7 +595,7 @@ fn given_sql_macro_bodies_when_linting_then_frames_and_generated_comments_are_no }]], ..LintSqlBody::default() }, - expected_findings: vec![("SQBRSQL034", 1, 1, Some((1, 7)))], + expected_findings: vec![("SQLCLEAN034", 1, 1, Some((1, 7)))], }, LintBodiesTestCase { description: "comments a Python macro generated are judged in the caller", @@ -612,9 +612,9 @@ fn given_sql_macro_bodies_when_linting_then_frames_and_generated_comments_are_no ..LintSqlBody::default() }, expected_findings: vec![ - ("SQBRSQL034", 1, 1, Some((1, 7))), - ("SQBRSQL033", 1, 11, Some((1, 14))), - ("SQBRSQL033", 1, 15, Some((1, 23))), + ("SQLCLEAN034", 1, 1, Some((1, 7))), + ("SQLCLEAN033", 1, 11, Some((1, 14))), + ("SQLCLEAN033", 1, 15, Some((1, 23))), ], }, LintBodiesTestCase { @@ -626,9 +626,9 @@ fn given_sql_macro_bodies_when_linting_then_frames_and_generated_comments_are_no ..LintSqlBody::default() }, expected_findings: vec![ - ("SQBRSQL034", 1, 1, Some((1, 7))), - ("SQBRSQL033", 1, 11, Some((1, 14))), - ("SQBRSQL033", 2, 11, Some((2, 12))), + ("SQLCLEAN034", 1, 1, Some((1, 7))), + ("SQLCLEAN033", 1, 11, Some((1, 14))), + ("SQLCLEAN033", 2, 11, Some((2, 12))), ], }, ]; @@ -637,7 +637,7 @@ fn given_sql_macro_bodies_when_linting_then_frames_and_generated_comments_are_no let options = LintSqlOptions { dialect: "duckdb".to_string(), enabled_rules: Some( - ["SQBRSQL001", "SQBRSQL022", "SQBRSQL033", "SQBRSQL034"] + ["SQLCLEAN001", "SQLCLEAN022", "SQLCLEAN033", "SQLCLEAN034"] .map(str::to_string) .to_vec(), ), diff --git a/crates/sqlbuild-rules/src/lint_text/types.rs b/crates/sqlbuild-rules/src/lint_text/types.rs index 4e21467032..3ff5c816cb 100644 --- a/crates/sqlbuild-rules/src/lint_text/types.rs +++ b/crates/sqlbuild-rules/src/lint_text/types.rs @@ -1,4 +1,9 @@ -//! Schema values shared by fixture formatting and its object bridge. +//! Schema values shared by fixture formatting and its object bridge, and prepared lint text. + +use crate::lint_text::models::InterpolationSite; /// A fixture column name, optional type, and optional nullability. pub type FixtureFormatColumn = (String, Option, Option); + +/// Lint text with sentinels, its interpolation sites and the CTEs interpolation references. +pub type PreparedSql = (String, Vec, Vec); diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index c78de2bcf8..8e5981896b 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -7,7 +7,7 @@ use std::collections::BTreeMap; use crate::constants::{ API_VERSION, BUILT_IN_RULE_NAMESPACE, CUSTOM_RULE_NAMESPACE, RULE_CODE_NUMBER_LENGTH, - SCOPE_METADATA_SCHEMA_VERSION, + SCOPE_METADATA_SCHEMA_VERSION, SQL_RULE_PREFIX, }; #[derive(Clone, Copy, Debug, Default)] @@ -24,7 +24,12 @@ impl fensu_policy::policy::types::RuleCodeGrammar for RulesCodeGrammar { let (family, number) = remainder.split_at(remainder.len() - RULE_CODE_NUMBER_LENGTH); let family_valid = family.bytes().all(|value| value.is_ascii_uppercase()); let number_valid = number.bytes().all(|value| value.is_ascii_digit()); - number_valid && family_valid && (namespace == CUSTOM_RULE_NAMESPACE || !family.is_empty()) + let family_shape = match namespace { + CUSTOM_RULE_NAMESPACE => true, + SQL_RULE_PREFIX => family.is_empty(), + _ => !family.is_empty(), + }; + number_valid && family_valid && family_shape } fn rule_selector_is_valid(&self, value: &str) -> bool { @@ -39,12 +44,17 @@ impl fensu_policy::policy::types::RuleCodeGrammar for RulesCodeGrammar { impl RulesCodeGrammar { fn namespace_and_remainder(value: &str) -> Option<(&'static str, &str)> { - if let Some(remainder) = value.strip_prefix(CUSTOM_RULE_NAMESPACE) { - return Some((CUSTOM_RULE_NAMESPACE, remainder)); - } - value - .strip_prefix(BUILT_IN_RULE_NAMESPACE) - .map(|remainder| (BUILT_IN_RULE_NAMESPACE, remainder)) + [ + CUSTOM_RULE_NAMESPACE, + SQL_RULE_PREFIX, + BUILT_IN_RULE_NAMESPACE, + ] + .into_iter() + .find_map(|namespace| { + value + .strip_prefix(namespace) + .map(|remainder| (namespace, remainder)) + }) } } @@ -146,7 +156,8 @@ impl Default for LayoutConfig { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(default, deny_unknown_fields)] pub struct RulesConfig { - pub select: Vec, + /// Exact codes or family prefixes; `None` runs SQLClean's default SQL Rules and nothing else. + pub select: Option>, pub ignore: Vec, pub thresholds: BTreeMap, pub threshold_overrides: Vec, @@ -166,7 +177,7 @@ pub struct RulesConfig { impl Default for RulesConfig { fn default() -> Self { Self { - select: vec![], + select: None, ignore: vec![], thresholds: BTreeMap::new(), threshold_overrides: vec![], diff --git a/crates/sqlbuild-rules/src/sql_quality/_helpers/declared_keys.rs b/crates/sqlbuild-rules/src/relation_keys/_helpers/declared_keys.rs similarity index 94% rename from crates/sqlbuild-rules/src/sql_quality/_helpers/declared_keys.rs rename to crates/sqlbuild-rules/src/relation_keys/_helpers/declared_keys.rs index 02607a9bb2..54459b5ce5 100644 --- a/crates/sqlbuild-rules/src/sql_quality/_helpers/declared_keys.rs +++ b/crates/sqlbuild-rules/src/relation_keys/_helpers/declared_keys.rs @@ -1,10 +1,10 @@ //! Unique keys one relation declares through its incremental config and column audits. -use crate::sql_quality::constants::{ +use crate::relation_keys::constants::{ INCREMENTAL_MATERIALIZATION, KEY_PRESERVING_STRATEGIES, KEYED_DELETE_INSERT_STRATEGY, MICROBATCH_MODE, NOT_NULL_AUDIT, UNIQUE_AUDIT, }; -use crate::sql_quality::models::{DeclaredKeyColumn, DeclaredRelation}; +use crate::relation_keys::models::{DeclaredKeyColumn, DeclaredRelation}; /// `unique_key` only where the incremental strategy keeps one row per key. pub(crate) fn trusted_unique_key(relation: &DeclaredRelation) -> Vec> { diff --git a/crates/sqlbuild-rules/src/relation_keys/_helpers/mod.rs b/crates/sqlbuild-rules/src/relation_keys/_helpers/mod.rs new file mode 100644 index 0000000000..7fd6889894 --- /dev/null +++ b/crates/sqlbuild-rules/src/relation_keys/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod declared_keys; diff --git a/crates/sqlbuild-rules/src/relation_keys/constants.rs b/crates/sqlbuild-rules/src/relation_keys/constants.rs new file mode 100644 index 0000000000..b2894d1087 --- /dev/null +++ b/crates/sqlbuild-rules/src/relation_keys/constants.rs @@ -0,0 +1,8 @@ +pub(crate) const UNIQUE_AUDIT: &str = "unique"; +pub(crate) const NOT_NULL_AUDIT: &str = "not_null"; +pub(crate) const INCREMENTAL_MATERIALIZATION: &str = "incremental"; +pub(crate) const MICROBATCH_MODE: &str = "microbatch"; +pub(crate) const KEYED_DELETE_INSERT_STRATEGY: &str = "delete_insert"; +pub(crate) const KEY_PRESERVING_STRATEGIES: [&str; 2] = ["merge", KEYED_DELETE_INSERT_STRATEGY]; +pub(crate) const RELATION_KIND_REF: &str = "ref"; +pub(crate) const RELATION_KIND_SEED: &str = "seed"; diff --git a/crates/sqlbuild-rules/src/sql_quality/main/declared_relation_keys.rs b/crates/sqlbuild-rules/src/relation_keys/main/declared_relation_keys.rs similarity index 83% rename from crates/sqlbuild-rules/src/sql_quality/main/declared_relation_keys.rs rename to crates/sqlbuild-rules/src/relation_keys/main/declared_relation_keys.rs index f99921268b..c47bb7c943 100644 --- a/crates/sqlbuild-rules/src/sql_quality/main/declared_relation_keys.rs +++ b/crates/sqlbuild-rules/src/relation_keys/main/declared_relation_keys.rs @@ -1,8 +1,8 @@ use std::collections::BTreeMap; -use crate::sql_quality::constants::{RELATION_KIND_REF, RELATION_KIND_SEED}; -use crate::sql_quality::declared_keys::{non_null_unique_columns, trusted_unique_key}; -use crate::sql_quality::models::DeclaredRelation; +use crate::relation_keys::_helpers::declared_keys::{non_null_unique_columns, trusted_unique_key}; +use crate::relation_keys::constants::{RELATION_KIND_REF, RELATION_KIND_SEED}; +use crate::relation_keys::models::DeclaredRelation; /// Keys proving one row per key by `kind:name`; seeds are also refs, one-row relations key `[]`. pub fn declared_relation_keys( diff --git a/crates/sqlbuild-rules/src/sql_quality/main/mod.rs b/crates/sqlbuild-rules/src/relation_keys/main/mod.rs similarity index 60% rename from crates/sqlbuild-rules/src/sql_quality/main/mod.rs rename to crates/sqlbuild-rules/src/relation_keys/main/mod.rs index f8c958d3fb..414baa340b 100644 --- a/crates/sqlbuild-rules/src/sql_quality/main/mod.rs +++ b/crates/sqlbuild-rules/src/relation_keys/main/mod.rs @@ -1,2 +1 @@ pub mod declared_relation_keys; -pub mod diagnostics; diff --git a/crates/sqlbuild-rules/src/relation_keys/mod.rs b/crates/sqlbuild-rules/src/relation_keys/mod.rs new file mode 100644 index 0000000000..ad094fad60 --- /dev/null +++ b/crates/sqlbuild-rules/src/relation_keys/mod.rs @@ -0,0 +1,6 @@ +//! Unique keys project relations declare through incremental config and audits, which prove ranking determinism. + +pub(crate) mod _helpers; +mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-rules/src/sql_quality/models.rs b/crates/sqlbuild-rules/src/relation_keys/models.rs similarity index 60% rename from crates/sqlbuild-rules/src/sql_quality/models.rs rename to crates/sqlbuild-rules/src/relation_keys/models.rs index 7b91480b64..979bd81673 100644 --- a/crates/sqlbuild-rules/src/sql_quality/models.rs +++ b/crates/sqlbuild-rules/src/relation_keys/models.rs @@ -1,22 +1,3 @@ -use std::collections::{HashMap, HashSet}; - -use polyglot_sql::tokens::Token; -use polyglot_sql::{DialectType, Expression}; - -/// Inputs for one body's quality Rules, borrowed from the native lint request. -#[derive(Debug)] -pub(crate) struct QualityRequest<'a> { - pub(crate) statements: &'a [Expression], - pub(crate) tokens: &'a [Token], - pub(crate) dialect: DialectType, - pub(crate) enabled: &'a HashSet, - pub(crate) max_literal_length: usize, - pub(crate) max_ranking_order_by: usize, - pub(crate) relation_keys: &'a HashMap>>, - pub(crate) externally_referenced_ctes: &'a HashSet, - pub(crate) fixture_body: bool, -} - /// One project relation's declared key facts: its incremental config and audited columns. #[derive(Debug, Clone, Default)] pub struct DeclaredRelation { diff --git a/crates/sqlbuild-rules/src/rule_cache/tests/test_rule_cache.rs b/crates/sqlbuild-rules/src/rule_cache/tests/test_rule_cache.rs index 4d2c72069a..dd616aa6a2 100644 --- a/crates/sqlbuild-rules/src/rule_cache/tests/test_rule_cache.rs +++ b/crates/sqlbuild-rules/src/rule_cache/tests/test_rule_cache.rs @@ -25,7 +25,7 @@ fn given_models_when_hashing_rule_keys_together_then_each_key_matches_its_single expected_key_count: 64, }, ]; - let codes: Vec = vec!["SQBRSQL001".to_owned(), "SQBRSQL033".to_owned()]; + let codes: Vec = vec!["SQLCLEAN001".to_owned(), "SQLCLEAN033".to_owned()]; for test_case in test_cases { let inputs: Vec> = test_case diff --git a/crates/sqlbuild-rules/src/rule_catalogue/constants.rs b/crates/sqlbuild-rules/src/rule_catalogue/constants.rs index 1d2ae016fc..222049ed26 100644 --- a/crates/sqlbuild-rules/src/rule_catalogue/constants.rs +++ b/crates/sqlbuild-rules/src/rule_catalogue/constants.rs @@ -35,11 +35,11 @@ pub(crate) const GRAPH_EDGE_RULE_CODE: &str = "SQBRGRAPH101"; pub(crate) const REPLACEABLE_SKILL_STATES: [&str; 3] = ["fresh", "missing", "stale"]; pub(crate) const RULE_EXAMPLES: [(&str, &str); 28] = [ ( - "SQBRSQL040", + "SQLCLEAN040", "Compute category_key in an input CTE, then JOIN categories AS c ON o.category_key = c.category_key.", ), ( - "SQBRSQL041", + "SQLCLEAN041", "WITH orders AS (...), final AS (SELECT order_id FROM orders) SELECT order_id FROM final.", ), ( @@ -75,7 +75,7 @@ pub(crate) const RULE_EXAMPLES: [(&str, &str); 28] = [ "sales publishes sales__mart__orders and sales__mart__customers; the finance variant lives in finance or sales intermediate.", ), ( - "SQBRSQL038", + "SQLCLEAN038", "JOIN customers AS c ON o.customer_id = c.customer_id, or CROSS JOIN a provably one-row relation such as a one-row seed or an aggregate without GROUP BY.", ), ( diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/render_skills_body.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/render_skills_body.rs index 9882a1a4d0..0eae7c22a1 100644 --- a/crates/sqlbuild-rules/src/rule_catalogue/main/render_skills_body.rs +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/render_skills_body.rs @@ -15,7 +15,7 @@ pub fn render_skills_body(rules: &[RuleText], config: &RulesConfig) -> String { "# SQLBuild Rules", "", "Rules enforce SQL model architecture and never modifies SQL files.", - "Run `sqb rules run SQBR` before handing off model changes.", + "Run `sqb rules run SQBR` and `sqb rules run SQLCLEAN` before handing off model changes.", "Fix findings in SQL. A deliberate deviation is a reviewed [[rules.rule_exceptions]] entry with a reason in sqlbuild_project.toml; inline `-- sqb: ignore` comments are errors.", "", "## Active Rules", diff --git a/crates/sqlbuild-rules/src/rule_results/constants.rs b/crates/sqlbuild-rules/src/rule_results/constants.rs index 55a1a8d737..b053a8e161 100644 --- a/crates/sqlbuild-rules/src/rule_results/constants.rs +++ b/crates/sqlbuild-rules/src/rule_results/constants.rs @@ -1,6 +1,6 @@ //! Codes, placement folders and wording of rules results and literal hints. -pub(crate) const LONG_LITERAL_CODE: &str = "SQBRSQL044"; +pub(crate) const LONG_LITERAL_CODE: &str = "SQLCLEAN044"; pub(crate) const QUOTE: char = '\''; pub(crate) const LISTED_PATHS: usize = 5; pub(crate) const MIN_SHARED_FILES: usize = 2; diff --git a/crates/sqlbuild-rules/src/rule_results/main/default_rules_notice.rs b/crates/sqlbuild-rules/src/rule_results/main/default_rules_notice.rs new file mode 100644 index 0000000000..d5d609d4dc --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/main/default_rules_notice.rs @@ -0,0 +1,18 @@ +//! `default_rules_notice`: how to turn off configurable Rules, shown when unselected defaults report findings. + +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; + +use crate::constants::SQL_RULE_PREFIX; + +/// The notice when `select` is omitted and a finding comes from a default SQL Rule, otherwise `None`. +#[must_use] +pub fn default_rules_notice<'a>( + select_omitted: bool, + mut codes: impl Iterator, +) -> Option { + (select_omitted && codes.any(|code| code.starts_with(SQL_RULE_PREFIX))).then(|| { + format!( + "Using default rules. To disable configurable checks, add to {PROJECT_CONFIG_FILENAME}:\n\n [rules]\n select = []\n\nAlready have [rules]? Add select = [] there. Compiler validation stays enabled." + ) + }) +} diff --git a/crates/sqlbuild-rules/src/rule_results/main/duplicate_literal_hints.rs b/crates/sqlbuild-rules/src/rule_results/main/duplicate_literal_hints.rs index 42c0ea2189..e94214a49f 100644 --- a/crates/sqlbuild-rules/src/rule_results/main/duplicate_literal_hints.rs +++ b/crates/sqlbuild-rules/src/rule_results/main/duplicate_literal_hints.rs @@ -1,4 +1,4 @@ -//! `duplicate_literal_hints`: SQBRSQL044 help naming the other files that repeat a literal. +//! `duplicate_literal_hints`: SQLCLEAN044 help naming the other files that repeat a literal. use std::collections::{BTreeMap, BTreeSet, HashMap}; diff --git a/crates/sqlbuild-rules/src/rule_results/main/mod.rs b/crates/sqlbuild-rules/src/rule_results/main/mod.rs index a361ef467b..3cb9c4ec7c 100644 --- a/crates/sqlbuild-rules/src/rule_results/main/mod.rs +++ b/crates/sqlbuild-rules/src/rule_results/main/mod.rs @@ -1,3 +1,4 @@ +pub mod default_rules_notice; pub mod duplicate_literal_hints; pub mod format_result; pub mod format_rules_progress; diff --git a/crates/sqlbuild-rules/src/rule_results/tests/test_rule_results.rs b/crates/sqlbuild-rules/src/rule_results/tests/test_rule_results.rs index 97038120ff..1782bc251a 100644 --- a/crates/sqlbuild-rules/src/rule_results/tests/test_rule_results.rs +++ b/crates/sqlbuild-rules/src/rule_results/tests/test_rule_results.rs @@ -60,7 +60,7 @@ fn given_repeated_long_literals_when_hinting_then_matches_python_wording() { .paths .iter() .map(|path| RuleFinding { - code: "SQBRSQL044".to_owned(), + code: "SQLCLEAN044".to_owned(), path: (*path).to_owned(), line: 1, column: 8, @@ -89,16 +89,16 @@ fn given_rules_result_when_formatting_then_matches_python_text_and_json() { ResultTestCase { description: "text lists affected rules, the fix note and unevaluated resources", json_output: false, - expected_text: "models/o.sql:3:4 [SQBRSQL001] bad \u{e9}\n Remediation: Fix it.\n Affected Rules: A, B\n Fix available: `sqb format --fix` applies it after compiler verification\nFound 1 Rules findings\n2 models evaluated, 1 resources could not be evaluated", + expected_text: "models/o.sql:3:4 [SQLCLEAN001] bad \u{e9}\n Remediation: Fix it.\n Affected Rules: A, B\n Fix available: `sqb format --fix` applies it after compiler verification\nFound 1 Rules findings\n2 models evaluated, 1 resources could not be evaluated", }, ResultTestCase { description: "JSON is indented, key-sorted and ASCII-escaped", json_output: true, - expected_text: "{\n \"cache_hits\": 1,\n \"cache_misses\": 1,\n \"evaluated_models\": 2,\n \"finding_count\": 1,\n \"findings\": [\n {\n \"affected_rules\": [\n \"A\",\n \"B\"\n ],\n \"code\": \"SQBRSQL001\",\n \"column\": 4,\n \"fixable\": true,\n \"line\": 3,\n \"message\": \"bad \\u00e9\",\n \"path\": \"models/o.sql\",\n \"remediation\": \"Fix it.\"\n }\n ],\n \"unevaluated_resources\": 1\n}", + expected_text: "{\n \"cache_hits\": 1,\n \"cache_misses\": 1,\n \"evaluated_models\": 2,\n \"finding_count\": 1,\n \"findings\": [\n {\n \"affected_rules\": [\n \"A\",\n \"B\"\n ],\n \"code\": \"SQLCLEAN001\",\n \"column\": 4,\n \"fixable\": true,\n \"line\": 3,\n \"message\": \"bad \\u00e9\",\n \"path\": \"models/o.sql\",\n \"remediation\": \"Fix it.\"\n }\n ],\n \"unevaluated_resources\": 1\n}", }, ]; let finding = RuleFinding { - code: "SQBRSQL001".to_owned(), + code: "SQLCLEAN001".to_owned(), path: "models/o.sql".to_owned(), line: 3, column: 4, diff --git a/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs b/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs index d1353c4aab..b0bd8f7873 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs @@ -1,4 +1,4 @@ -use crate::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; +use crate::constants::{API_VERSION, NATIVE_BUILD_IDENTITY, SQL_RULE_PREFIX}; use crate::models::RulesCodeGrammar; use crate::models::{CustomRule, ResolveRulesRequest, RuleGuidance, RuleMetadata, RuleScope}; use fensu_policy::policy::errors::PolicyError; @@ -373,8 +373,9 @@ pub(crate) fn catalogue() -> Vec { "Add RuleCase values evaluated by evaluate_rule under tests/.", ), ]; + let defaults: Vec<&str> = sqlclean::sql_lint::main::default_rules::default_rules(); rules.extend( - crate::sql_lint::main::catalogue::catalogue() + sqlclean::sql_lint::main::catalogue::catalogue() .into_iter() .map(|rule| RuleMetadata { code: rule.code.into(), @@ -384,7 +385,7 @@ pub(crate) fn catalogue() -> Vec { remediation: rule.remediation.into(), guidance: None, implementation_fingerprint: env!("CARGO_PKG_VERSION").into(), - enabled_by_default: true, + enabled_by_default: defaults.contains(&rule.code), project_wide: false, custom: false, scope: RuleScope::SqlLint, @@ -452,19 +453,21 @@ pub(crate) fn with_custom(custom: &[CustomRule]) -> Result, St pub(crate) fn select<'a>( catalogue: &'a [RuleMetadata], - selected: &[String], + selected: Option<&[String]>, ignored: &[String], ) -> Result, String> { let grammar = RulesCodeGrammar; let references: Vec<&RuleMetadata> = catalogue.iter().collect(); + let selected_or_none: &[String] = selected.unwrap_or_default(); let configured = PolicySelectors { - select: selected.to_vec(), + select: selected_or_none.to_vec(), warn: Vec::new(), ignore: Vec::new(), }; let _ = resolve_policy(&references, &(), &configured, &grammar).map_err(rules_error)?; let (mut effective_select, custom_selected) = - activate_explicit_custom_matches(catalogue, selected, &grammar); + activate_explicit_custom_matches(catalogue, selected_or_none, &grammar); + effective_select.extend(sql_lint_selection(catalogue, selected, &grammar)); if custom_selected && !effective_select .iter() @@ -482,6 +485,25 @@ pub(crate) fn select<'a>( .map_err(rules_error) } +/// SQLClean's selection: its default Rules without a selection, and every Rule a selector names, whatever its default. +pub(crate) fn sql_lint_selection( + catalogue: &[RuleMetadata], + selected: Option<&[String]>, + grammar: &RulesCodeGrammar, +) -> Vec { + catalogue + .iter() + .filter(|rule| rule.code.starts_with(SQL_RULE_PREFIX)) + .filter(|rule| match selected { + None => rule.enabled_by_default, + Some(selectors) => selectors + .iter() + .any(|selector| grammar.code_matches_selector(&rule.code, selector)), + }) + .map(|rule| rule.code.clone()) + .collect() +} + fn activate_explicit_custom_matches( catalogue: &[RuleMetadata], selected: &[String], @@ -517,7 +539,11 @@ pub(crate) fn selected_codes_json(request_json: &str) -> Result } crate::configuration::main::validate::validate(&request.config)?; let rules = with_custom(&request.custom_rules)?; - let selected = select(&rules, &request.config.select, &request.config.ignore)?; + let selected = select( + &rules, + request.config.select.as_deref(), + &request.config.ignore, + )?; let codes: Vec<&str> = selected.iter().map(|rule| rule.code.as_str()).collect(); serde_json::to_string(&codes).map_err(|error| error.to_string()) } diff --git a/crates/sqlbuild-rules/src/rules/_helpers/dynamic_contracts.rs b/crates/sqlbuild-rules/src/rules/_helpers/dynamic_contracts.rs index 4143496870..acf4c1e5ef 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/dynamic_contracts.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/dynamic_contracts.rs @@ -72,7 +72,7 @@ pub(crate) fn allows_output_star(query: &QueryFacts, root: &QueryFacts, model: & fn rules_need_no_query(model: &Model, selected: &BTreeMap) -> bool { selected.values().all(|rule| { rule.custom - || rule.code.starts_with("SQBRSQL") + || rule.code.starts_with("SQLCLEAN") || rule.code.starts_with("SQBRCONTRACT10") || (model.bare_dynamic_pivot && rule.code == SELECT_STAR_MODEL_RULE_CODE) || matches!( diff --git a/crates/sqlbuild-rules/src/rules/main/select.rs b/crates/sqlbuild-rules/src/rules/main/select.rs index a1ca2e502a..45f8eef747 100644 --- a/crates/sqlbuild-rules/src/rules/main/select.rs +++ b/crates/sqlbuild-rules/src/rules/main/select.rs @@ -2,7 +2,7 @@ use crate::models::RuleMetadata; pub(crate) fn select<'a>( catalogue: &'a [RuleMetadata], - selected: &[String], + selected: Option<&[String]>, ignored: &[String], ) -> Result, String> { crate::rules::_helpers::catalogue::select(catalogue, selected, ignored) diff --git a/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs b/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs index c9778e61d1..9bf552da84 100644 --- a/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs +++ b/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs @@ -1,29 +1,21 @@ -//! `selected_sql_codes`: the built-in SQL rule codes a `select`/`ignore` pair enables. +//! `selected_sql_codes`: the built-in SQL Rule codes a `select`/`ignore` pair enables. -use crate::constants::SQL_RULE_PREFIX; -use crate::rules::main::catalogue::catalogue; +use fensu_policy::policy::types::RuleCodeGrammar; -fn matches_any(code: &str, selectors: &[String]) -> bool { - for selector in selectors { - if code.starts_with(selector.as_str()) { - return true; - } - } - false -} +use crate::models::RulesCodeGrammar; +use crate::rules::_helpers::catalogue::sql_lint_selection; +use crate::rules::main::catalogue::catalogue; -/// Catalogue SQL rule codes matching a `select` prefix and no `ignore` prefix, in catalogue order. +/// SQL Rule codes `select` enables (SQLClean's defaults when it is `None`) that no `ignore` selector matches, in catalogue order. #[must_use] -pub fn selected_sql_codes(select: &[String], ignore: &[String]) -> Vec { - let mut codes: Vec = Vec::new(); - for rule in catalogue() { - let code: String = rule.code; - if code.starts_with(SQL_RULE_PREFIX) - && matches_any(&code, select) - && !matches_any(&code, ignore) - { - codes.push(code); - } - } - codes +pub fn selected_sql_codes(select: Option<&[String]>, ignore: &[String]) -> Vec { + let grammar = RulesCodeGrammar; + sql_lint_selection(&catalogue(), select, &grammar) + .into_iter() + .filter(|code| { + !ignore + .iter() + .any(|selector| grammar.code_matches_selector(code, selector)) + }) + .collect() } diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs deleted file mode 100644 index f5649d280e..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs +++ /dev/null @@ -1,1873 +0,0 @@ -use std::collections::HashSet; - -use polyglot_sql::tokens::{Span, Token, TokenType}; - -use crate::constants::{EMPTY_FIXTURE_QUERY_TOKEN_COUNT, ENCLOSING_CTE_TOKEN_COUNT}; -use crate::lint_text::constants::SQL_MACRO_PARAMETER_SENTINEL_PREFIX; -use crate::sql_lint::_helpers::engine::{ - direct_indices, is_comment, is_layout, is_query_from, query_end, significant_after, - significant_before, token_depths, -}; -use crate::sql_lint::constants::RELATION_MODIFIER_KEYWORDS; -use crate::sql_lint::models::{AdditionalFactOptions, AdditionalQueryFacts, SqlMacroOutput}; -use rustc_hash::{FxHashMap, FxHashSet}; -use sqlbuild_core::constants::{CLOSE_PAREN, OPEN_PAREN, SQL_WILDCARD}; - -const COUNT_ONE_LITERAL: &str = "1"; -const QUALIFIED_REFERENCE_LENGTH: usize = 3; -const QUALIFIER_PAIR_LENGTH: usize = 2; -const IMPLICIT_ALIAS_MINIMUM_LENGTH: usize = 2; -const GENERATED_IDENTIFIER_PREFIXES: [&str; 3] = [ - "__sqb_lint_", - "__sqlbuild_audit_parameter_", - "__sqlbuild_context_parameter_", -]; - -struct QuerySlice<'a> { - tokens: &'a [Token], - depths: &'a [usize], - direct: &'a [usize], - start: usize, - end: usize, - depth: usize, -} - -struct ReferenceFactContext<'a> { - query: &'a QuerySlice<'a>, - outer_relations: &'a FxHashSet, - external_identifiers: &'a HashSet, - sql_macro_outputs: &'a [SqlMacroOutput], - query_start: usize, - query_end: usize, -} - -struct ReferenceCandidateContext<'a> { - tokens: &'a [Token], - references: &'a [usize], - external_identifiers: &'a HashSet, - projection_aliases: &'a FxHashMap, -} - -struct SelectFactContext<'a> { - tokens: &'a [Token], - depths: &'a [usize], - external_identifiers: &'a HashSet, - sql_macro_outputs: &'a [SqlMacroOutput], - dependency_identifiers: &'a HashSet, - allows_ceremonial_select: bool, - allows_dynamic_output_star: bool, - allows_empty_fixture_star: bool, -} - -struct ProjectedStarContext<'a> { - tokens: &'a [Token], - depths: &'a [usize], - direct: &'a [usize], - query_end: usize, -} - -pub(super) fn collect_additional_facts( - tokens: &[Token], - options: &AdditionalFactOptions<'_>, -) -> AdditionalQueryFacts { - let mut facts = AdditionalQueryFacts::default(); - let depths = token_depths(tokens); - let significant: Vec = (0..tokens.len()) - .filter(|&index| !is_layout(&tokens[index]) && !is_comment(&tokens[index])) - .collect(); - for (position, &index) in significant.iter().enumerate() { - let token = &tokens[index]; - if token.token_type == TokenType::Union - && significant.get(position + 1).is_none_or(|&next| { - !matches!( - tokens[next].token_type, - TokenType::All | TokenType::Distinct | TokenType::By - ) - }) - { - facts.bare_unions.push(token.span); - } - if token.token_type == TokenType::Else - && significant - .get(position + 1) - .is_some_and(|&next| tokens[next].token_type == TokenType::Null) - { - let next = significant[position + 1]; - facts.redundant_else_nulls.push(Span { - start: token.span.start, - end: tokens[next].span.end, - line: token.span.line, - column: token.span.column, - }); - } - if token.token_type == TokenType::Distinct - && significant - .get(position + 1) - .is_some_and(|&next| tokens[next].token_type == TokenType::LParen) - && let Some(span) = - parenthesized_distinct_span(tokens, &depths, significant[position + 1]) - { - facts.parenthesized_distincts.push(span); - } - if token.token_type == TokenType::Semicolon - && significant - .get(position.wrapping_sub(1)) - .is_some_and(|&previous| tokens[previous].token_type == TokenType::Semicolon) - { - facts.consecutive_semicolons.push(token.span); - } - if is_identifier(token) - && let Some((&as_index, &alias_index)) = significant - .get(position + 1) - .zip(significant.get(position + 2)) - && tokens[as_index].token_type == TokenType::As - && is_identifier(&tokens[alias_index]) - && token.token_type == tokens[alias_index].token_type - && token.text == tokens[alias_index].text - && is_simple_projection_reference(tokens, &depths, &significant, position) - { - facts.redundant_self_aliases.push(Span { - start: token.span.end, - end: tokens[alias_index].span.end, - line: tokens[as_index].span.line, - column: tokens[as_index].span.column, - }); - } - if token.text.eq_ignore_ascii_case("COUNT") - && let Some(span) = count_one_span(tokens, &significant, position) - { - facts.count_one_literals.push(span); - } - if token.text.eq_ignore_ascii_case("ROW_NUMBER") - && let Some(span) = unstable_row_number_span(tokens, &depths, index) - { - facts.unstable_row_numbers.push(span); - } - if token.token_type == TokenType::Not - && let Some(span) = null_not_in_span(tokens, &depths, &significant, position) - { - facts.null_not_in_predicates.push(span); - } - if token.token_type == TokenType::Case - && let Some(span) = simple_boolean_case_span(tokens, &significant, position) - { - facts.simple_boolean_cases.push(span); - } - if token.token_type == TokenType::Else - && significant - .get(position + 1) - .is_some_and(|&next| tokens[next].token_type == TokenType::Case) - { - facts - .nested_else_cases - .push(tokens[significant[position + 1]].span); - } - if position + 2 < significant.len() - && constant_predicate(tokens, &significant[position..position + 3]) - { - facts - .constant_predicates - .push(tokens[significant[position + 1]].span); - } - } - let select_facts = collect_select_additional_facts(&SelectFactContext { - tokens, - depths: &depths, - external_identifiers: options.external_identifiers, - sql_macro_outputs: options.sql_macro_outputs, - dependency_identifiers: options.dependency_identifiers, - allows_ceremonial_select: options.allows_ceremonial_select, - allows_dynamic_output_star: options.allows_dynamic_output_star, - allows_empty_fixture_star: options.allows_empty_fixture_star, - }); - facts.duplicate_output_aliases = select_facts.duplicate_output_aliases; - facts.unaliased_calculations = select_facts.unaliased_calculations; - facts.projected_stars = select_facts.projected_stars; - facts.duplicate_table_aliases = select_facts.duplicate_table_aliases; - facts.unused_table_aliases = select_facts.unused_table_aliases; - facts.null_rejected_left_joins = select_facts.null_rejected_left_joins; - facts.implicit_inner_joins = select_facts.implicit_inner_joins; - facts.unqualified_multi_source_columns = select_facts.unqualified_multi_source_columns; - facts.inconsistent_single_source_qualification = - select_facts.inconsistent_single_source_qualification; - facts.unknown_relation_qualifiers = select_facts.unknown_relation_qualifiers; - facts.unused_joined_relations = select_facts.unused_joined_relations; - facts.mixed_group_order_references = select_facts.mixed_group_order_references; - facts.ambiguous_order_directions = select_facts.ambiguous_order_directions; - facts.set_arity_mismatches = set_arity_mismatch_spans(tokens, &depths); - facts -} - -fn is_simple_projection_reference( - tokens: &[Token], - depths: &[usize], - significant: &[usize], - position: usize, -) -> bool { - let index = significant[position]; - let Some(boundary) = (0..index).rev().find(|&candidate| { - depths[candidate] == depths[index] - && (tokens[candidate].token_type == TokenType::Select - || is_query_from(tokens, candidate) - || tokens[candidate].token_type == TokenType::Join) - }) else { - return false; - }; - if tokens[boundary].token_type != TokenType::Select { - return false; - } - let item_boundary = significant[..position] - .iter() - .rposition(|&candidate| { - candidate > boundary - && depths[candidate] == depths[index] - && tokens[candidate].token_type == TokenType::Comma - }) - .map_or(boundary, |boundary_position| significant[boundary_position]); - let prefix: Vec = significant[..position] - .iter() - .copied() - .filter(|&candidate| candidate > item_boundary && depths[candidate] == depths[index]) - .collect(); - if is_bare_star_prefix(tokens, &prefix) { - return true; - } - let mut qualifier_start = prefix.len(); - while qualifier_start >= QUALIFIER_PAIR_LENGTH - && tokens[prefix[qualifier_start - 1]].token_type == TokenType::Dot - && is_identifier(&tokens[prefix[qualifier_start - 2]]) - { - qualifier_start -= 2; - } - qualifier_start < prefix.len() && is_bare_star_prefix(tokens, &prefix[..qualifier_start]) -} - -fn collect_select_additional_facts(context: &SelectFactContext<'_>) -> AdditionalQueryFacts { - let tokens = context.tokens; - let depths = context.depths; - let mut facts = AdditionalQueryFacts::default(); - for (select_index, token) in tokens.iter().enumerate() { - if token.token_type != TokenType::Select { - continue; - } - let depth = depths[select_index]; - let end = query_end(tokens, depths, select_index, depth); - let direct: Vec = direct_indices(depths, select_index + 1, end, depth) - .into_iter() - .filter(|&index| !is_layout(&tokens[index]) && !is_comment(&tokens[index])) - .collect(); - let from_position = direct - .iter() - .position(|&index| is_query_from(tokens, index)) - .unwrap_or(direct.len()); - facts - .duplicate_output_aliases - .extend(duplicate_alias_spans(tokens, &direct[..from_position])); - if !is_exists_subquery(tokens, select_index) - && !is_set_continuation_select(tokens, select_index) - && !is_scalar_subquery(tokens, depths, select_index) - && !has_enclosing_cte_column_list(tokens, depths, select_index) - && !is_ceremonial_select( - tokens, - &direct[..from_position], - context.allows_ceremonial_select, - ) - { - facts - .unaliased_calculations - .extend(unaliased_calculation_spans( - tokens, - &direct[..from_position], - )); - } - if !is_dependency_import_select(context, select_index, &direct, from_position) - && !is_compiler_proven_dynamic_output_select( - context, - select_index, - &direct, - from_position, - ) - && !is_canonical_empty_input_fixture(context, select_index, end) - && !has_irreducible_projected_star( - &ProjectedStarContext { - tokens, - depths, - direct: &direct, - query_end: end, - }, - from_position, - ) - { - facts - .projected_stars - .extend(projected_star_spans(tokens, &direct[..from_position])); - } - if from_position < direct.len() { - let clause_end = direct[from_position + 1..] - .iter() - .position(|&index| is_after_relation_clause(tokens[index].token_type)) - .map_or(direct.len(), |offset| from_position + 1 + offset); - facts.duplicate_table_aliases.extend(duplicate_alias_spans( - tokens, - &direct[from_position + 1..clause_end], - )); - let query_slice = QuerySlice { - tokens, - depths, - direct: &direct[from_position + 1..clause_end], - start: select_index, - end, - depth, - }; - facts - .unused_table_aliases - .extend(unused_alias_spans(&query_slice)); - facts - .null_rejected_left_joins - .extend(null_rejected_left_join_spans(tokens, &direct, clause_end)); - facts.implicit_inner_joins.extend(implicit_inner_join_spans( - tokens, - &direct[from_position + 1..clause_end], - )); - let reference_slice = QuerySlice { - tokens, - depths, - direct: &direct, - start: from_position, - end: clause_end, - depth, - }; - let outer_relations = outer_relation_names(tokens, depths, select_index, depth); - let reference_context = ReferenceFactContext { - query: &reference_slice, - outer_relations: &outer_relations, - external_identifiers: context.external_identifiers, - sql_macro_outputs: context.sql_macro_outputs, - query_start: select_index, - query_end: end, - }; - let reference_facts = collect_reference_facts(&reference_context); - facts - .unqualified_multi_source_columns - .extend(reference_facts.unqualified_multi_source_columns); - facts - .inconsistent_single_source_qualification - .extend(reference_facts.inconsistent_single_source_qualification); - facts - .unknown_relation_qualifiers - .extend(reference_facts.unknown_relation_qualifiers); - facts - .unused_joined_relations - .extend(reference_facts.unused_joined_relations); - } - facts - .mixed_group_order_references - .extend(mixed_reference_clause_spans(tokens, &direct)); - facts - .ambiguous_order_directions - .extend(ambiguous_order_direction_spans(tokens, &direct)); - } - facts -} - -fn is_canonical_empty_input_fixture( - context: &SelectFactContext<'_>, - select_index: usize, - query_end: usize, -) -> bool { - if !context.allows_empty_fixture_star { - return false; - } - let significant_query: Vec<&str> = context.tokens[select_index..query_end] - .iter() - .filter(|token| !is_layout(token) && !is_comment(token)) - .map(|token| token.text.as_str()) - .collect(); - if significant_query.len() != EMPTY_FIXTURE_QUERY_TOKEN_COUNT - || !significant_query[0].eq_ignore_ascii_case("select") - || significant_query[1] != SQL_WILDCARD - || !significant_query[2].eq_ignore_ascii_case("from") - || !significant_query[3].eq_ignore_ascii_case("__empty_fixture") - || significant_query[4] != OPEN_PAREN - || significant_query[5] != CLOSE_PAREN - { - return false; - } - let previous: Vec<&Token> = context.tokens[..select_index] - .iter() - .filter(|token| !is_layout(token) && !is_comment(token)) - .rev() - .take(ENCLOSING_CTE_TOKEN_COUNT) - .collect(); - if previous.len() != ENCLOSING_CTE_TOKEN_COUNT - || previous[0].token_type != TokenType::LParen - || !previous[1].text.eq_ignore_ascii_case("as") - { - return false; - } - let cte_name = previous[2].text.trim_matches(['"', '`']); - ["__ref__", "__source__", "__seed__"] - .iter() - .any(|prefix| cte_name.starts_with(prefix)) -} - -fn is_compiler_proven_dynamic_output_select( - context: &SelectFactContext<'_>, - select_index: usize, - direct: &[usize], - from_position: usize, -) -> bool { - if !context.allows_dynamic_output_star - || context.depths[select_index] != 0 - || direct[..from_position].len() != 1 - || context.tokens[direct[0]].token_type != TokenType::Star - { - return false; - } - !context.tokens[select_index + 1..] - .iter() - .zip(&context.depths[select_index + 1..]) - .any(|(token, depth)| token.token_type == TokenType::Select && *depth == 0) -} - -fn has_irreducible_projected_star( - context: &ProjectedStarContext<'_>, - from_position: usize, -) -> bool { - let ProjectedStarContext { - tokens, - depths, - direct, - query_end, - } = context; - let projection = &direct[..from_position]; - if projection.len() != 1 || tokens[projection[0]].token_type != TokenType::Star { - return false; - } - - let Some(relation_start) = from_position - .checked_add(1) - .filter(|&start| start < direct.len()) - else { - return false; - }; - let relation_end = direct[relation_start..] - .iter() - .position(|&index| is_after_relation_clause(tokens[index].token_type)) - .map_or(direct.len(), |offset| relation_start + offset); - let relation = &direct[relation_start..relation_end]; - if relation - .iter() - .any(|&index| matches!(tokens[index].token_type, TokenType::Join | TokenType::Comma)) - { - return false; - } - if is_lone_audit_parameter_relation(tokens, relation) { - return true; - } - - let mut pivots = relation - .iter() - .copied() - .filter(|&index| tokens[index].text.eq_ignore_ascii_case("PIVOT")); - let Some(pivot_index) = pivots.next() else { - return false; - }; - if pivots.next().is_some() { - return false; - } - dynamic_pivot_uses_any(tokens, depths, pivot_index, *query_end) -} - -fn is_lone_audit_parameter_relation(tokens: &[Token], relation: &[usize]) -> bool { - let Some(&relation_index) = relation.first() else { - return false; - }; - let normalized = tokens[relation_index].text.to_ascii_lowercase(); - let Some(index) = normalized - .strip_prefix("__sqlbuild_audit_parameter_") - .and_then(|value| value.strip_suffix("__")) - else { - return false; - }; - if index.is_empty() || !index.chars().all(|character| character.is_ascii_digit()) { - return false; - } - match relation.get(1..) { - Some([]) => true, - Some([alias]) => is_identifier(&tokens[*alias]), - Some([as_index, alias]) => { - tokens[*as_index].token_type == TokenType::As && is_identifier(&tokens[*alias]) - } - _ => false, - } -} - -fn dynamic_pivot_uses_any( - tokens: &[Token], - depths: &[usize], - pivot_index: usize, - query_end: usize, -) -> bool { - let Some(pivot_open) = significant_after(tokens, pivot_index) else { - return false; - }; - if tokens[pivot_open].token_type != TokenType::LParen { - return false; - } - let pivot_depth = depths[pivot_open]; - let Some(pivot_close) = (pivot_open + 1..query_end).find(|&index| { - depths[index] == pivot_depth + 1 && tokens[index].token_type == TokenType::RParen - }) else { - return false; - }; - (pivot_open + 1..pivot_close).any(|in_index| { - depths[in_index] == pivot_depth + 1 - && tokens[in_index].token_type == TokenType::In - && significant_after(tokens, in_index).is_some_and(|any_open| { - any_open < pivot_close - && tokens[any_open].token_type == TokenType::LParen - && significant_after(tokens, any_open).is_some_and(|any_index| { - any_index < pivot_close && tokens[any_index].token_type == TokenType::Any - }) - }) - }) -} - -fn has_enclosing_cte_column_list(tokens: &[Token], depths: &[usize], select_index: usize) -> bool { - let select_depth = depths[select_index]; - let Some(body_open) = (0..select_index).rev().find(|&index| { - tokens[index].token_type == TokenType::LParen - && depths[index].saturating_add(1) == select_depth - }) else { - return false; - }; - let Some(as_index) = significant_before(tokens, body_open) else { - return false; - }; - if tokens[as_index].token_type != TokenType::As { - return false; - } - significant_before(tokens, as_index) - .is_some_and(|index| tokens[index].token_type == TokenType::RParen) -} - -fn is_dependency_import_select( - context: &SelectFactContext<'_>, - select_index: usize, - direct: &[usize], - from_position: usize, -) -> bool { - let tokens = context.tokens; - let depths = context.depths; - if from_position != 1 - || tokens[direct[0]].token_type != TokenType::Star - || !is_top_level_cte_body(tokens, depths, select_index) - { - return false; - } - if direct.len() <= from_position + 1 || has_cte_body_set_operation(tokens, depths, select_index) - { - return false; - } - let relation = &direct[from_position + 1..]; - let Some(&relation_index) = relation.first() else { - return false; - }; - if !context - .dependency_identifiers - .contains(&tokens[relation_index].text.to_ascii_lowercase()) - { - return false; - } - match relation { - [_] => true, - [_, alias] => is_identifier(&tokens[*alias]), - [_, explicit_as, alias] => { - tokens[*explicit_as].token_type == TokenType::As && is_identifier(&tokens[*alias]) - } - _ => false, - } -} - -fn has_cte_body_set_operation(tokens: &[Token], depths: &[usize], select_index: usize) -> bool { - let select_depth = depths[select_index]; - let body_end = (select_index + 1..tokens.len()) - .find(|&index| { - depths[index] == select_depth && tokens[index].token_type == TokenType::RParen - }) - .unwrap_or(tokens.len()); - (select_index + 1..body_end).any(|index| { - depths[index] == select_depth - && matches!( - tokens[index].token_type, - TokenType::Union | TokenType::Intersect | TokenType::Except - ) - }) -} - -fn is_top_level_cte_body(tokens: &[Token], depths: &[usize], select_index: usize) -> bool { - let Some(open_index) = significant_before(tokens, select_index) else { - return false; - }; - if tokens[open_index].token_type != TokenType::LParen || depths[open_index] != 0 { - return false; - } - significant_before(tokens, open_index) - .is_some_and(|index| tokens[index].token_type == TokenType::As) -} - -fn collect_reference_facts(context: &ReferenceFactContext<'_>) -> AdditionalQueryFacts { - let query = context.query; - let QuerySlice { - tokens, - direct, - start: from_position, - end: relation_end, - .. - } = query; - let from_position = *from_position; - let relation_end = *relation_end; - let mut facts = AdditionalQueryFacts::default(); - let relations = &direct[from_position + 1..relation_end]; - let relation_count = if is_values_relation(tokens, relations) { - 1 - } else { - 1 + relations - .iter() - .filter(|&&index| { - matches!(tokens[index].token_type, TokenType::Join | TokenType::Comma) - }) - .count() - }; - let mut known = relation_names(tokens, relations); - known.extend(context.outer_relations.iter().cloned()); - let references: Vec = direct[..from_position] - .iter() - .chain(direct[relation_end..].iter()) - .copied() - .collect(); - let projection_aliases = projection_alias_positions(tokens, &direct[..from_position]); - let candidate_context = ReferenceCandidateContext { - tokens, - references: &references, - external_identifiers: context.external_identifiers, - projection_aliases: &projection_aliases, - }; - let qualified: Vec = references - .iter() - .enumerate() - .filter_map(|(position, &index)| simple_qualifier(tokens, &references, position, index)) - .collect(); - let unqualified: Vec = references - .iter() - .enumerate() - .filter_map(|(position, &index)| unqualified_reference(&candidate_context, position, index)) - .collect(); - if relation_count > 1 { - facts - .unqualified_multi_source_columns - .extend(unqualified.iter().map(|&index| tokens[index].span)); - } else { - facts - .inconsistent_single_source_qualification - .extend(inconsistent_qualification( - tokens, - (&qualified, &unqualified), - context.sql_macro_outputs, - )); - } - facts.unknown_relation_qualifiers.extend( - qualified - .iter() - .filter(|&&index| !known.contains(&tokens[index].text.to_ascii_lowercase())) - .map(|&index| tokens[index].span), - ); - let projects_bare_star = has_bare_projected_star(tokens, &direct[..from_position]); - if !projects_bare_star { - facts.unused_joined_relations.extend(unused_join_spans( - query, - context.query_start, - context.query_end, - )); - } - facts -} - -/// The unqualified references of a single-relation scope that also qualifies some, each judged as it was written. In a macro file a plain slot read such as `@orders.amount` lints as a parameter sentinel pair and counts as unqualified. In a call's output, a read qualified by a relation the call passes takes neither style: the slot decides how it expands, not the caller or the macro author. A scope whose every reference a macro call generated is judged once, in the macro file. -fn inconsistent_qualification( - tokens: &[Token], - (qualified, unqualified): (&[usize], &[usize]), - outputs: &[SqlMacroOutput], -) -> Vec { - let output_of = |index: usize| -> Option { - let start: usize = tokens[index].span.start; - outputs - .iter() - .position(|output| output.start <= start && start < output.end) - }; - let mut references = qualified.iter().chain(unqualified).copied(); - if let Some(first) = references.next() - && let Some(output) = output_of(first) - && references.all(|index| output_of(index) == Some(output)) - { - return Vec::new(); - } - let passed_relation_read = |index: &usize| { - output_of(*index).is_some_and(|output| { - let qualifier: String = unquoted_name(&tokens[*index].text); - outputs[output] - .relations - .iter() - .any(|relation| relation.eq_ignore_ascii_case(&qualifier)) - }) - }; - let (plain_reads, qualified): (Vec, Vec) = qualified - .iter() - .copied() - .filter(|index| !passed_relation_read(index)) - .partition(|&index| is_plain_slot_read(tokens, index)); - if qualified.is_empty() || (unqualified.is_empty() && plain_reads.is_empty()) { - return Vec::new(); - } - let mut unqualified: Vec = unqualified.to_vec(); - unqualified.extend(plain_reads); - unqualified.sort_unstable(); - unqualified - .iter() - .map(|&index| tokens[index].span) - .collect() -} - -/// An identifier's name without its quotes. -fn unquoted_name(text: &str) -> String { - text.trim_matches(|character| matches!(character, '"' | '`' | '[' | ']')) - .to_owned() -} - -/// Whether the qualified reference at `index` is a plain SQL macro slot read, both its relation and its slot a macro parameter. -fn is_plain_slot_read(tokens: &[Token], index: usize) -> bool { - let is_parameter = |token: &Token| { - token - .text - .to_ascii_lowercase() - .starts_with(SQL_MACRO_PARAMETER_SENTINEL_PREFIX) - }; - is_parameter(&tokens[index]) && tokens.get(index + 2).is_some_and(is_parameter) -} - -fn simple_qualifier( - tokens: &[Token], - references: &[usize], - position: usize, - index: usize, -) -> Option { - let previous = position.checked_sub(1).map(|value| references[value]); - let dot = references.get(position + 1).copied(); - let column = references.get(position + 2).copied(); - let following = references.get(position + 3).copied(); - (is_identifier(&tokens[index]) - && !is_generated_identifier(&tokens[index]) - && previous.is_none_or(|candidate| { - !matches!( - tokens[candidate].token_type, - TokenType::Dot | TokenType::Colon | TokenType::DColon | TokenType::DotColon - ) - }) - && dot.is_some_and(|candidate| tokens[candidate].token_type == TokenType::Dot) - && column.is_some_and(|candidate| is_identifier(&tokens[candidate])) - && following.is_none_or(|candidate| tokens[candidate].token_type != TokenType::Dot)) - .then_some(index) -} - -fn unqualified_reference( - context: &ReferenceCandidateContext<'_>, - position: usize, - index: usize, -) -> Option { - let ReferenceCandidateContext { - tokens, - references, - external_identifiers, - projection_aliases, - } = context; - let previous = position.checked_sub(1).map(|value| references[value]); - let next = references.get(position + 1).copied(); - (is_identifier(&tokens[index]) - && !is_generated_identifier(&tokens[index]) - && !is_context_value(&tokens[index]) - && !external_identifiers.contains(&tokens[index].text.to_ascii_lowercase()) - && !projection_aliases - .get(&tokens[index].text.to_ascii_lowercase()) - .is_some_and(|alias_index| *alias_index < index) - && previous.is_none_or(|candidate| { - !matches!( - tokens[candidate].token_type, - TokenType::Dot - | TokenType::As - | TokenType::Colon - | TokenType::DColon - | TokenType::DotColon - ) - }) - && next.is_none_or(|candidate| { - !matches!( - tokens[candidate].token_type, - TokenType::Dot - | TokenType::LParen - | TokenType::Colon - | TokenType::DColon - | TokenType::DotColon - ) - })) - .then_some(index) -} - -fn relation_names(tokens: &[Token], relation_clause: &[usize]) -> FxHashSet { - let mut names: FxHashSet = FxHashSet::default(); - for (position, &index) in relation_clause.iter().enumerate() { - let starts_relation = position == 0 - || matches!( - tokens[relation_clause[position - 1]].token_type, - TokenType::Join | TokenType::Comma - ); - if starts_relation && is_identifier(&tokens[index]) { - names.insert(tokens[index].text.to_ascii_lowercase()); - } - if tokens[index].token_type == TokenType::As - && let Some(&alias) = relation_clause.get(position + 1) - && is_identifier(&tokens[alias]) - { - names.insert(tokens[alias].text.to_ascii_lowercase()); - } - } - let mut segment_start = 0_usize; - for segment_end in relation_clause - .iter() - .enumerate() - .filter_map(|(position, &index)| { - matches!(tokens[index].token_type, TokenType::Join | TokenType::Comma) - .then_some(position) - }) - .chain(std::iter::once(relation_clause.len())) - { - let segment = &relation_clause[segment_start..segment_end]; - segment_start = segment_end + 1; - let relation_end = segment - .iter() - .position(|&index| is_relation_condition_start(&tokens[index])) - .unwrap_or(segment.len()); - if let Some(&implicit_name) = segment[..relation_end] - .iter() - .rev() - .find(|&&index| is_identifier(&tokens[index])) - { - names.insert(tokens[implicit_name].text.to_ascii_lowercase()); - } - } - names -} - -fn outer_relation_names( - tokens: &[Token], - depths: &[usize], - select_index: usize, - depth: usize, -) -> FxHashSet { - let mut names: FxHashSet = FxHashSet::default(); - for outer_select in (0..select_index).filter(|&index| { - tokens[index].token_type == TokenType::Select - && depths[index] < depth - && query_end(tokens, depths, index, depths[index]) > select_index - }) { - let outer_depth = depths[outer_select]; - let outer_end = query_end(tokens, depths, outer_select, outer_depth); - let direct: Vec = direct_indices(depths, outer_select + 1, outer_end, outer_depth) - .into_iter() - .filter(|&index| !is_layout(&tokens[index]) && !is_comment(&tokens[index])) - .collect(); - let Some(from_position) = direct - .iter() - .position(|&index| is_query_from(tokens, index)) - else { - continue; - }; - let clause_end = direct[from_position + 1..] - .iter() - .position(|&index| is_after_relation_clause(tokens[index].token_type)) - .map_or(direct.len(), |offset| from_position + 1 + offset); - names.extend(relation_names( - tokens, - &direct[from_position + 1..clause_end], - )); - } - names -} - -fn projected_star_spans(tokens: &[Token], projection: &[usize]) -> Vec { - let mut spans: Vec = Vec::new(); - for (position, &index) in projection.iter().enumerate() { - if tokens[index].token_type != TokenType::Star { - continue; - } - let previous = position.checked_sub(1).map(|value| projection[value]); - let qualified = - previous.is_some_and(|candidate| tokens[candidate].token_type == TokenType::Dot); - let mut item_start = 0_usize; - for prior_position in (0..position).rev() { - if tokens[projection[prior_position]].token_type == TokenType::Comma { - item_start = prior_position + 1; - break; - } - } - let bare = is_bare_star_prefix(tokens, &projection[item_start..position]); - if qualified || bare { - spans.push(tokens[index].span); - } - } - spans -} - -fn has_bare_projected_star(tokens: &[Token], projection: &[usize]) -> bool { - projection.iter().enumerate().any(|(position, &index)| { - if tokens[index].token_type != TokenType::Star { - return false; - } - let mut item_start = 0_usize; - for prior_position in (0..position).rev() { - if tokens[projection[prior_position]].token_type == TokenType::Comma { - item_start = prior_position + 1; - break; - } - } - is_bare_star_prefix(tokens, &projection[item_start..position]) - }) -} - -fn is_exists_subquery(tokens: &[Token], select_index: usize) -> bool { - let Some(open_index) = significant_before(tokens, select_index) else { - return false; - }; - tokens[open_index].token_type == TokenType::LParen - && significant_before(tokens, open_index) - .is_some_and(|index| tokens[index].token_type == TokenType::Exists) -} - -fn is_set_continuation_select(tokens: &[Token], select_index: usize) -> bool { - let mut previous = significant_before(tokens, select_index); - if previous.is_some_and(|index| { - matches!( - tokens[index].token_type, - TokenType::All | TokenType::Distinct - ) - }) { - previous = previous.and_then(|index| significant_before(tokens, index)); - } - previous.is_some_and(|index| { - matches!( - tokens[index].token_type, - TokenType::Union | TokenType::Intersect | TokenType::Except - ) - }) -} - -fn is_bare_star_prefix(tokens: &[Token], prefix: &[usize]) -> bool { - let mut position = 0_usize; - while prefix.get(position).is_some_and(|&index| { - matches!( - tokens[index].token_type, - TokenType::Distinct | TokenType::All - ) - }) { - position += 1; - } - if position == prefix.len() { - return true; - } - if tokens[prefix[position]].token_type != TokenType::Top { - return false; - } - position += 1; - if !prefix - .get(position) - .is_some_and(|&index| tokens[index].token_type == TokenType::Number) - { - return false; - } - position += 1; - if prefix - .get(position) - .is_some_and(|&index| tokens[index].token_type == TokenType::Percent) - { - position += 1; - } - if prefix - .get(position) - .is_some_and(|&index| tokens[index].token_type == TokenType::With) - && prefix - .get(position + 1) - .is_some_and(|&index| tokens[index].token_type == TokenType::Ties) - { - position += 2; - } - position == prefix.len() -} - -fn is_scalar_subquery(tokens: &[Token], depths: &[usize], select_index: usize) -> bool { - let Some(open_index) = significant_before(tokens, select_index) else { - return false; - }; - if tokens[open_index].token_type != TokenType::LParen { - return false; - } - let Some(previous) = significant_before(tokens, open_index) else { - return false; - }; - if matches!( - tokens[previous].token_type, - TokenType::From | TokenType::Join | TokenType::As | TokenType::Exists | TokenType::Lateral - ) { - return false; - } - if tokens[previous].token_type != TokenType::Comma { - return true; - } - let parent_depth = depths[previous]; - let query_start = (0..previous) - .rev() - .find(|&index| { - tokens[index].token_type == TokenType::Select && depths[index] < parent_depth - }) - .map_or(0, |index| index + 1); - let relation_boundary = (query_start..previous).rev().find(|&index| { - depths[index] == parent_depth - && (tokens[index].token_type == TokenType::Select - || is_query_from(tokens, index) - || matches!( - tokens[index].token_type, - TokenType::Join | TokenType::Semicolon - )) - }); - !relation_boundary.is_some_and(|index| { - is_query_from(tokens, index) || tokens[index].token_type == TokenType::Join - }) -} - -fn is_ceremonial_select(tokens: &[Token], projection: &[usize], allowed: bool) -> bool { - allowed - && projection.len() == 1 - && tokens[projection[0]].token_type == TokenType::Number - && tokens[projection[0]].text == COUNT_ONE_LITERAL -} - -fn is_values_relation(tokens: &[Token], relation_clause: &[usize]) -> bool { - relation_clause - .first() - .is_some_and(|&index| tokens[index].token_type == TokenType::Values) -} - -fn is_generated_identifier(token: &Token) -> bool { - let normalized = token.text.to_ascii_lowercase(); - GENERATED_IDENTIFIER_PREFIXES - .iter() - .any(|prefix| normalized.starts_with(prefix)) -} - -fn is_relation_condition_start(token: &Token) -> bool { - matches!( - token.token_type, - TokenType::On | TokenType::Using | TokenType::MatchCondition - ) || token.text.eq_ignore_ascii_case("MATCH_CONDITION") -} - -fn is_context_value(token: &Token) -> bool { - matches!( - token.text.to_ascii_uppercase().as_str(), - "CURRENT_CATALOG" - | "CURRENT_DATE" - | "CURRENT_DATETIME" - | "CURRENT_ROLE" - | "CURRENT_SCHEMA" - | "CURRENT_TIME" - | "CURRENT_TIMESTAMP" - | "CURRENT_USER" - ) -} - -fn projection_alias_positions(tokens: &[Token], projection: &[usize]) -> FxHashMap { - let mut aliases: FxHashMap = FxHashMap::default(); - for window in projection.windows(2) { - if tokens[window[0]].token_type == TokenType::As && is_identifier(&tokens[window[1]]) { - aliases.insert(tokens[window[1]].text.to_ascii_lowercase(), window[1]); - } - } - aliases -} - -fn unused_join_spans(query: &QuerySlice<'_>, query_start: usize, query_end: usize) -> Vec { - let QuerySlice { - tokens, - direct, - start: from_position, - end: relation_end, - .. - } = query; - let from_position = *from_position; - let relation_end = *relation_end; - let relation_start_index = direct.get(from_position + 1).copied().unwrap_or(query_end); - let relation_end_index = direct.get(relation_end).copied().unwrap_or(query_end); - let references: Vec = (query_start..relation_start_index) - .chain(relation_end_index..query_end) - .filter(|&index| !is_layout(&tokens[index]) && !is_comment(&tokens[index])) - .collect(); - if contains_possible_unqualified_reference(tokens, &references) { - return Vec::new(); - } - let mut spans: Vec = Vec::new(); - for position in from_position + 1..relation_end { - let index = direct[position]; - if tokens[index].token_type != TokenType::Join { - continue; - } - let Some(alias) = relation_alias_after_join(tokens, direct, position) else { - continue; - }; - let used_outside_relation_clause = - contains_qualified_reference(tokens, &references, &alias); - let downstream_relation_start = direct[position + 1..relation_end] - .iter() - .position(|&candidate| { - matches!( - tokens[candidate].token_type, - TokenType::Join | TokenType::Comma - ) - }) - .map(|offset| position + 1 + offset); - let used_by_downstream_relation = downstream_relation_start.is_some_and(|start| { - let downstream_start_index = direct[start]; - let downstream_end_index = direct.get(relation_end).copied().unwrap_or(query_end); - let downstream_references: Vec = (downstream_start_index..downstream_end_index) - .filter(|&candidate| { - !is_layout(&tokens[candidate]) && !is_comment(&tokens[candidate]) - }) - .collect(); - contains_qualified_reference(tokens, &downstream_references, &alias) - }); - let used = used_outside_relation_clause || used_by_downstream_relation; - if !used { - spans.push(tokens[index].span); - } - } - spans -} - -fn contains_possible_unqualified_reference(tokens: &[Token], references: &[usize]) -> bool { - references.iter().enumerate().any(|(position, &index)| { - if !is_identifier(&tokens[index]) - || is_generated_identifier(&tokens[index]) - || is_context_value(&tokens[index]) - { - return false; - } - let previous = if position == 0 { - None - } else { - Some(references[position - 1]) - }; - let next = references.get(position + 1).copied(); - previous.is_none_or(|candidate| { - !matches!(tokens[candidate].token_type, TokenType::Dot | TokenType::As) - }) && next.is_none_or(|candidate| { - !matches!( - tokens[candidate].token_type, - TokenType::Dot | TokenType::LParen - ) - }) - }) -} - -fn contains_qualified_reference(tokens: &[Token], references: &[usize], alias: &str) -> bool { - references.windows(2).any(|window| { - tokens[window[0]].text.eq_ignore_ascii_case(alias) - && tokens[window[1]].token_type == TokenType::Dot - }) -} - -fn is_after_relation_clause(token_type: TokenType) -> bool { - matches!( - token_type, - TokenType::Where - | TokenType::Group - | TokenType::Having - | TokenType::Qualify - | TokenType::Order - | TokenType::Limit - | TokenType::Offset - | TokenType::Fetch - | TokenType::Union - | TokenType::Intersect - | TokenType::Except - | TokenType::Semicolon - ) -} - -fn unaliased_calculation_spans(tokens: &[Token], projection: &[usize]) -> Vec { - let mut spans: Vec = Vec::new(); - let mut start = 0_usize; - let ends: Vec = projection - .iter() - .enumerate() - .filter_map(|(position, &index)| { - (tokens[index].token_type == TokenType::Comma).then_some(position) - }) - .chain(std::iter::once(projection.len())) - .collect(); - for end in ends { - let item = &projection[start..end]; - start = end + 1; - if item.is_empty() - || item - .iter() - .any(|&index| tokens[index].token_type == TokenType::As) - || item - .iter() - .any(|&index| is_generated_identifier(&tokens[index])) - || simple_projection(tokens, item) - || implicit_alias_projection(tokens, item) - { - continue; - } - spans.push(tokens[item[0]].span); - } - spans -} - -fn simple_projection(tokens: &[Token], item: &[usize]) -> bool { - let trimmed: Vec = item - .iter() - .copied() - .skip_while(|&index| { - matches!( - tokens[index].token_type, - TokenType::Distinct | TokenType::All - ) - }) - .collect(); - let wildcard = trimmed.first().is_some_and(|&index| { - tokens[index].token_type == TokenType::Star - || (trimmed.len() >= QUALIFIED_REFERENCE_LENGTH - && is_identifier(&tokens[index]) - && tokens[trimmed[1]].token_type == TokenType::Dot - && tokens[trimmed[2]].token_type == TokenType::Star) - }); - wildcard - || trimmed.len() == 1 - && (is_bare_column_token(&tokens[trimmed[0]]) - || tokens[trimmed[0]].token_type == TokenType::Star) - || trimmed.len() == QUALIFIED_REFERENCE_LENGTH - && is_identifier(&tokens[trimmed[0]]) - && tokens[trimmed[1]].token_type == TokenType::Dot - && (is_bare_column_token(&tokens[trimmed[2]]) - || tokens[trimmed[2]].token_type == TokenType::Star) -} - -fn is_bare_column_token(token: &Token) -> bool { - is_identifier(token) - || (!matches!( - token.token_type, - TokenType::Null - | TokenType::True - | TokenType::False - | TokenType::Number - | TokenType::String - ) && token - .text - .chars() - .next() - .is_some_and(|character| character.is_alphabetic() || character == '_') - && token - .text - .chars() - .all(|character| character.is_alphanumeric() || character == '_')) -} - -fn implicit_alias_projection(tokens: &[Token], item: &[usize]) -> bool { - item.len() >= IMPLICIT_ALIAS_MINIMUM_LENGTH - && is_identifier(&tokens[*item.last().unwrap_or(&item[0])]) - && !matches!( - tokens[item[item.len() - 2]].token_type, - TokenType::Dot | TokenType::Plus | TokenType::Dash | TokenType::Star | TokenType::Slash - ) -} - -fn unused_alias_spans(query: &QuerySlice<'_>) -> Vec { - let QuerySlice { - tokens, - depths, - direct: relation_clause, - start: query_start, - end: query_end, - depth, - } = query; - let query_start = *query_start; - let query_end = *query_end; - let depth = *depth; - let mut spans: Vec = Vec::new(); - for (span_start, alias_index) in relation_alias_positions(tokens, relation_clause) { - if significant_after(tokens, alias_index) - .is_some_and(|index| tokens[index].token_type == TokenType::LParen) - { - continue; - } - let alias = &tokens[alias_index].text; - let relation_name = significant_before(tokens, span_start); - let used = (query_start..query_end).any(|index| { - index != alias_index - && Some(index) != relation_name - && depths[index] >= depth - && is_identifier(&tokens[index]) - && tokens[index].text.eq_ignore_ascii_case(alias) - && significant_before(tokens, index).is_none_or(|previous| { - !matches!(tokens[previous].token_type, TokenType::Dot | TokenType::As) - }) - && significant_after(tokens, index) - .is_none_or(|next| tokens[next].token_type != TokenType::LParen) - }); - if !used { - spans.push(Span { - start: tokens[span_start].span.start, - end: tokens[alias_index].span.end, - line: tokens[span_start].span.line, - column: tokens[span_start].span.column, - }); - } - } - spans -} - -fn relation_alias_positions(tokens: &[Token], relation_clause: &[usize]) -> Vec<(usize, usize)> { - let mut aliases: Vec<(usize, usize)> = Vec::new(); - let mut segment_start = 0_usize; - let mut in_condition = false; - for (position, &index) in relation_clause.iter().enumerate() { - if matches!(tokens[index].token_type, TokenType::Join | TokenType::Comma) { - segment_start = position + 1; - in_condition = false; - continue; - } - if in_condition || is_relation_condition_start(&tokens[index]) { - in_condition = true; - continue; - } - if tokens[index].token_type == TokenType::As { - if let Some(&alias) = relation_clause.get(position + 1) - && is_identifier(&tokens[alias]) - { - aliases.push((index, alias)); - } - continue; - } - if is_implicit_relation_alias( - tokens, - &relation_clause[segment_start..], - position - segment_start, - ) { - aliases.push((index, index)); - } - } - aliases -} - -fn is_implicit_relation_alias(tokens: &[Token], segment: &[usize], position: usize) -> bool { - let candidate = &tokens[segment[position]]; - if position == 0 - || !is_identifier(candidate) - || candidate.token_type != TokenType::QuotedIdentifier - && RELATION_MODIFIER_KEYWORDS - .iter() - .any(|keyword| candidate.text.eq_ignore_ascii_case(keyword)) - { - return false; - } - let previous = &tokens[segment[position - 1]]; - if previous.token_type == TokenType::LParen { - return true; - } - is_identifier(previous) - && position - .checked_sub(QUALIFIER_PAIR_LENGTH) - .is_none_or(|before| tokens[segment[before]].token_type == TokenType::Dot) -} - -fn null_rejected_left_join_spans( - tokens: &[Token], - direct: &[usize], - relation_end: usize, -) -> Vec { - let Some(where_position) = direct - .iter() - .position(|&index| tokens[index].token_type == TokenType::Where) - else { - return Vec::new(); - }; - if direct[where_position + 1..] - .iter() - .any(|&index| tokens[index].token_type == TokenType::Or) - { - return Vec::new(); - } - let mut spans: Vec = Vec::new(); - for position in 0..relation_end { - let index = direct[position]; - if tokens[index].token_type != TokenType::Left { - continue; - } - let Some(join_position) = direct[position + 1..relation_end] - .iter() - .position(|&candidate| tokens[candidate].token_type == TokenType::Join) - .map(|offset| position + 1 + offset) - else { - continue; - }; - let Some(alias) = relation_alias_after_join(tokens, direct, join_position) else { - continue; - }; - if right_side_null_rejected(tokens, &direct[where_position + 1..], &alias) { - spans.push(tokens[index].span); - } - } - spans -} - -fn right_side_null_rejected(tokens: &[Token], predicate: &[usize], alias: &str) -> bool { - for window in predicate.windows(4) { - if !tokens[window[0]].text.eq_ignore_ascii_case(alias) - || tokens[window[1]].token_type != TokenType::Dot - || !is_identifier(&tokens[window[2]]) - { - continue; - } - if matches!( - tokens[window[3]].token_type, - TokenType::Eq - | TokenType::Neq - | TokenType::Lt - | TokenType::Lte - | TokenType::Gt - | TokenType::Gte - | TokenType::Like - | TokenType::In - | TokenType::Between - ) { - return true; - } - } - for window in predicate.windows(6) { - if tokens[window[0]].text.eq_ignore_ascii_case(alias) - && tokens[window[1]].token_type == TokenType::Dot - && is_identifier(&tokens[window[2]]) - && tokens[window[3]].token_type == TokenType::Is - && tokens[window[4]].token_type == TokenType::Not - && tokens[window[5]].token_type == TokenType::Null - { - return true; - } - } - false -} - -fn relation_alias_after_join( - tokens: &[Token], - direct: &[usize], - join_position: usize, -) -> Option { - let relation = &direct[join_position + 1..]; - let relation_end = relation - .iter() - .position(|&index| { - is_relation_condition_start(&tokens[index]) - || is_after_relation_clause(tokens[index].token_type) - || matches!(tokens[index].token_type, TokenType::Join | TokenType::Comma) - }) - .unwrap_or(relation.len()); - relation[..relation_end] - .iter() - .rev() - .find(|&&index| is_identifier(&tokens[index])) - .map(|&index| tokens[index].text.to_ascii_lowercase()) -} - -fn implicit_inner_join_spans(tokens: &[Token], relation_clause: &[usize]) -> Vec { - let mut spans: Vec = Vec::new(); - for (position, &index) in relation_clause.iter().enumerate() { - if is_plain_conditioned_join(tokens, relation_clause, position, index) { - spans.push(tokens[index].span); - } - } - spans -} - -fn is_plain_conditioned_join( - tokens: &[Token], - relation_clause: &[usize], - position: usize, - index: usize, -) -> bool { - if tokens[index].token_type != TokenType::Join - || position > 0 - && matches!( - tokens[relation_clause[position - 1]].token_type, - TokenType::Inner - | TokenType::Left - | TokenType::Right - | TokenType::Full - | TokenType::Outer - | TokenType::Cross - | TokenType::Natural - | TokenType::AsOf - | TokenType::Semi - | TokenType::Anti - ) - { - return false; - } - for &candidate in &relation_clause[position + 1..] { - if tokens[candidate].token_type == TokenType::Join { - break; - } - if matches!( - tokens[candidate].token_type, - TokenType::On | TokenType::Using - ) { - return true; - } - } - false -} - -fn ambiguous_order_direction_spans(tokens: &[Token], direct: &[usize]) -> Vec { - let mut spans: Vec = Vec::new(); - for (position, &index) in direct.iter().enumerate() { - if tokens[index].token_type != TokenType::Order - || direct - .get(position + 1) - .is_none_or(|&by| tokens[by].token_type != TokenType::By) - { - continue; - } - let end = direct[position + 2..] - .iter() - .position(|&candidate| is_clause_boundary(tokens[candidate].token_type)) - .map_or(direct.len(), |offset| position + 2 + offset); - let clause = &direct[position + 2..end]; - let item_ends: Vec = clause - .iter() - .enumerate() - .filter_map(|(item_position, &candidate)| { - (tokens[candidate].token_type == TokenType::Comma).then_some(item_position) - }) - .chain(std::iter::once(clause.len())) - .collect(); - let mut item_start = 0_usize; - let mut explicit = false; - let mut implicit = false; - for item_end in item_ends { - let item = &clause[item_start..item_end]; - item_start = item_end + 1; - let has_direction = item.last().is_some_and(|&candidate| { - matches!( - tokens[candidate].token_type, - TokenType::Asc | TokenType::Desc - ) - }); - explicit |= has_direction; - implicit |= !has_direction && !item.is_empty(); - } - if explicit && implicit { - spans.push(tokens[index].span); - } - } - spans -} - -fn duplicate_alias_spans(tokens: &[Token], indices: &[usize]) -> Vec { - let mut seen: FxHashSet = FxHashSet::default(); - let mut duplicates: Vec = Vec::new(); - for window in indices.windows(2) { - if tokens[window[0]].token_type != TokenType::As || !is_identifier(&tokens[window[1]]) { - continue; - } - let normalized = tokens[window[1]].text.to_ascii_lowercase(); - if !seen.insert(normalized) { - duplicates.push(tokens[window[1]].span); - } - } - duplicates -} - -fn mixed_reference_clause_spans(tokens: &[Token], direct: &[usize]) -> Vec { - let mut spans: Vec = Vec::new(); - for (position, &index) in direct.iter().enumerate() { - if !matches!( - tokens[index].token_type, - TokenType::Group | TokenType::Order - ) || direct - .get(position + 1) - .is_none_or(|&by| tokens[by].token_type != TokenType::By) - { - continue; - } - let end = direct[position + 2..] - .iter() - .position(|&candidate| is_clause_boundary(tokens[candidate].token_type)) - .map_or(direct.len(), |offset| position + 2 + offset); - let clause = &direct[position + 2..end]; - let mut has_number = false; - let mut has_name = false; - for (item_position, &candidate) in clause.iter().enumerate() { - if item_position == 0 - || tokens[clause[item_position - 1]].token_type == TokenType::Comma - { - has_number |= tokens[candidate].token_type == TokenType::Number; - has_name |= tokens[candidate].token_type != TokenType::Number; - } - } - if has_number && has_name { - spans.push(tokens[index].span); - } - } - spans -} - -fn is_clause_boundary(token_type: TokenType) -> bool { - matches!( - token_type, - TokenType::Having - | TokenType::Qualify - | TokenType::Order - | TokenType::Limit - | TokenType::Offset - | TokenType::Fetch - | TokenType::Union - | TokenType::Intersect - | TokenType::Except - | TokenType::Semicolon - ) -} - -fn parenthesized_distinct_span( - tokens: &[Token], - depths: &[usize], - open_index: usize, -) -> Option { - let close_depth = depths[open_index] + 1; - let close_index = (open_index + 1..tokens.len()).find(|&index| { - tokens[index].token_type == TokenType::RParen && depths[index] == close_depth - })?; - Some(Span { - start: tokens[significant_before(tokens, open_index)?].span.start, - end: tokens[close_index].span.end, - line: tokens[significant_before(tokens, open_index)?].span.line, - column: tokens[significant_before(tokens, open_index)?].span.column, - }) -} - -fn count_one_span(tokens: &[Token], significant: &[usize], position: usize) -> Option { - let open = *significant.get(position + 1)?; - let one = *significant.get(position + 2)?; - let close = *significant.get(position + 3)?; - (tokens[open].token_type == TokenType::LParen - && tokens[one].token_type == TokenType::Number - && tokens[one].text == COUNT_ONE_LITERAL - && tokens[close].token_type == TokenType::RParen) - .then_some(tokens[one].span) -} - -fn simple_boolean_case_span( - tokens: &[Token], - significant: &[usize], - position: usize, -) -> Option { - let tail = &significant[position + 1..]; - let end_offset = tail - .iter() - .position(|&index| tokens[index].token_type == TokenType::End)?; - let body = &tail[..=end_offset]; - if body - .iter() - .any(|&index| tokens[index].token_type == TokenType::Case) - { - return None; - } - let when_position = body - .iter() - .position(|&index| tokens[index].token_type == TokenType::When)?; - let then_position = body - .iter() - .position(|&index| tokens[index].token_type == TokenType::Then)?; - let else_position = body - .iter() - .position(|&index| tokens[index].token_type == TokenType::Else)?; - let then_value = *body.get(then_position + 1)?; - let else_value = *body.get(else_position + 1)?; - if when_position != 0 - || tokens[then_value].token_type != TokenType::True - || tokens[else_value].token_type != TokenType::False - || else_position + 2 != body.len() - 1 - { - return None; - } - let start = tokens[significant[position]].span; - let end = tokens[*body.last()?].span; - Some(Span { - start: start.start, - end: end.end, - line: start.line, - column: start.column, - }) -} - -fn unstable_row_number_span( - tokens: &[Token], - depths: &[usize], - function_index: usize, -) -> Option { - let over_index = (function_index + 1..tokens.len()).find(|&index| { - depths[index] == depths[function_index] && tokens[index].token_type == TokenType::Over - })?; - let open_index = significant_after(tokens, over_index)?; - if tokens[open_index].token_type != TokenType::LParen { - return None; - } - let close_depth = depths[open_index] + 1; - let close_index = (open_index + 1..tokens.len()).find(|&index| { - tokens[index].token_type == TokenType::RParen && depths[index] == close_depth - })?; - (!tokens[open_index + 1..close_index] - .iter() - .any(|token| matches!(token.token_type, TokenType::Order | TokenType::OrderBy))) - .then_some(tokens[function_index].span) -} - -fn null_not_in_span( - tokens: &[Token], - depths: &[usize], - significant: &[usize], - position: usize, -) -> Option { - let in_index = *significant.get(position + 1)?; - let open_index = *significant.get(position + 2)?; - if tokens[in_index].token_type != TokenType::In - || tokens[open_index].token_type != TokenType::LParen - { - return None; - } - let close_depth = depths[open_index] + 1; - let close_index = (open_index + 1..tokens.len()).find(|&index| { - tokens[index].token_type == TokenType::RParen && depths[index] == close_depth - })?; - let list_depth = depths[open_index] + 1; - let list = &tokens[open_index + 1..close_index]; - (!list - .iter() - .any(|token| token.token_type == TokenType::Select) - && list.iter().enumerate().any(|(offset, token)| { - token.token_type == TokenType::Null && depths[open_index + 1 + offset] == list_depth - })) - .then_some(tokens[significant[position]].span) -} - -fn constant_predicate(tokens: &[Token], indices: &[usize]) -> bool { - let left = &tokens[indices[0]]; - let operator = &tokens[indices[1]]; - let right = &tokens[indices[2]]; - matches!( - left.token_type, - TokenType::Number | TokenType::String | TokenType::True | TokenType::False - ) && left.token_type == right.token_type - && left.text == right.text - && matches!(operator.token_type, TokenType::Eq | TokenType::Neq) -} - -fn set_arity_mismatch_spans(tokens: &[Token], depths: &[usize]) -> Vec { - let mut spans: Vec = Vec::new(); - for (index, token) in tokens.iter().enumerate() { - if !matches!( - token.token_type, - TokenType::Union | TokenType::Intersect | TokenType::Except - ) { - continue; - } - let depth = depths[index]; - let Some(left_select) = (0..index).rev().find(|&candidate| { - depths[candidate] == depth && tokens[candidate].token_type == TokenType::Select - }) else { - continue; - }; - let Some(right_select) = (index + 1..tokens.len()).find(|&candidate| { - depths[candidate] == depth && tokens[candidate].token_type == TokenType::Select - }) else { - continue; - }; - let left_query = QuerySlice { - tokens, - depths, - direct: &[], - start: left_select, - end: index, - depth, - }; - let left_count = projection_arity(&left_query); - let right_end = query_end(tokens, depths, right_select, depth); - let right_query = QuerySlice { - tokens, - depths, - direct: &[], - start: right_select, - end: right_end, - depth, - }; - let right_count = projection_arity(&right_query); - if left_count - .zip(right_count) - .is_some_and(|(left, right)| left != right) - { - spans.push(token.span); - } - } - spans -} - -fn projection_arity(query: &QuerySlice<'_>) -> Option { - let QuerySlice { - tokens, - depths, - start: select, - end, - depth, - .. - } = query; - let select = *select; - let end = *end; - let depth = *depth; - let projection_end = (select + 1..end) - .find(|&index| depths[index] == depth && is_query_from(tokens, index)) - .unwrap_or(end); - if (select + 1..projection_end) - .any(|index| depths[index] == depth && tokens[index].token_type == TokenType::Star) - { - return None; - } - Some( - 1 + (select + 1..projection_end) - .filter(|&index| depths[index] == depth && tokens[index].token_type == TokenType::Comma) - .count(), - ) -} - -fn is_identifier(token: &Token) -> bool { - matches!( - token.token_type, - TokenType::Identifier | TokenType::QuotedIdentifier | TokenType::Var - ) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs deleted file mode 100644 index 18a586fc5f..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs +++ /dev/null @@ -1,1999 +0,0 @@ -use std::borrow::Cow; -use std::collections::{HashMap, HashSet, VecDeque}; -use std::str::FromStr; - -use polyglot_sql::expressions::With; -use polyglot_sql::parser::ParserConfig; -use polyglot_sql::tokens::{Span, Token, TokenType}; -use polyglot_sql::{ - ComplexityGuardOptions, Dialect, DialectType, Expression, ExpressionWalk, Parser, -}; - -use crate::sql_lint::constants::LINT_API_VERSION; -use crate::sql_lint::models::{ - AdditionalFactOptions, CrossJoinRequest, LintDiagnostic, LintEdit, LintRequest, LintResponse, - LintRuleMetadata, QueryFacts, SqlMacroOutput, -}; - -use crate::sql_lint::_helpers::additional::collect_additional_facts; -use crate::sql_lint::_helpers::inline_relations::collect_inline_query_relation_spans; -use crate::sql_lint::_helpers::join_predicates::cross_join_diagnostics; -use crate::sql_lint::_helpers::terminal_shape::collect_terminal_shape_facts; - -const NULL_COMPARISON: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL001", - message: "Comparison with NULL is never true", - remediation: "Use IS NULL or IS NOT NULL when testing for NULL.", -}; -const MAX_LINT_FUNCTION_CALL_DEPTH: usize = 128; -const CEREMONIAL_CTE_PREFIXES: [&str; 11] = [ - "__expected__", - "__assert__", - "__ref__", - "__source__", - "__seed__", - "__dbt_ref__", - "__macro_actual__", - "__macro_expected__", - "__udf_actual__", - "__udf_expected__", - "__table_fn_", -]; - -const IMPLICIT_CARTESIAN_JOIN: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL002", - message: "Comma-separated sources create an implicit cartesian join", - remediation: "Replace comma-separated sources with an explicit keyed join, or use CROSS JOIN when the cartesian product is intentional.", -}; -const JOIN_WITHOUT_CONDITION: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL003", - message: "Non-cross join has no meaningful condition", - remediation: "Add a meaningful ON or USING condition, or declare an intentional cartesian product with CROSS JOIN.", -}; -const UNORDERED_LIMIT: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL004", - message: "Row selection is nondeterministic", - remediation: "Add ORDER BY with a deterministic tie-breaker before LIMIT or OFFSET.", -}; -const UNUSED_CTE: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL005", - message: "CTE is unreachable from the final query", - remediation: "Reference the CTE from the final query or another reachable CTE, or remove it.", -}; -const REDUNDANT_DISTINCT: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL006", - message: "DISTINCT is redundant with the grouped output", - remediation: "Remove DISTINCT; the equivalent GROUP BY already determines the output groups.", -}; -const POSITIONAL_SET_STAR: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL007", - message: "Positional set operation is vulnerable to column-order drift", - remediation: "Enumerate columns in the same order in every set-operation branch.", -}; -const EXPLICIT_UNION: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL008", - message: "Bare UNION leaves duplicate handling implicit", - remediation: "Use UNION DISTINCT or UNION ALL to state duplicate handling explicitly.", -}; -const DUPLICATE_TABLE_ALIAS: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL009", - message: "Table alias is duplicated in the same query scope", - remediation: "Give every relation in this query scope a unique alias.", -}; -const DUPLICATE_OUTPUT_ALIAS: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL010", - message: "Output alias is duplicated in the same select list", - remediation: "Give every projected expression a unique output alias.", -}; -const MIXED_GROUP_ORDER_REFERENCES: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL011", - message: "Clause mixes positional and named references", - remediation: "Use column names or positions consistently throughout this clause.", -}; -const REDUNDANT_ELSE_NULL: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL012", - message: "ELSE NULL is redundant", - remediation: "Remove ELSE NULL; CASE expressions return NULL when ELSE is omitted.", -}; -const PARENTHESIZED_DISTINCT: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL013", - message: "DISTINCT is written as though it were a function", - remediation: "Write DISTINCT as a SELECT modifier without function-like parentheses.", -}; -const CONSTANT_PREDICATE: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL014", - message: "Predicate compares identical constant values", - remediation: "Remove the scaffold predicate or replace it with the intended condition.", -}; -const CONSECUTIVE_SEMICOLON: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL015", - message: "Empty statement follows another terminator", - remediation: "Remove the redundant statement terminator.", -}; -const REDUNDANT_SELF_ALIAS: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL016", - message: "Expression is aliased to its existing name", - remediation: "Remove the self-alias when the inferred output identity remains stable.", -}; -const COUNT_ONE: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL017", - message: "Row count uses a constant expression", - remediation: "Use COUNT(*) to state that rows, rather than a nullable expression, are counted.", -}; -const UNSTABLE_ROW_NUMBER: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL018", - message: "ROW_NUMBER has no deterministic ordering", - remediation: "Add ORDER BY with a stable tie-breaker inside the window specification.", -}; -const NULL_NOT_IN: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL019", - message: "NOT IN list contains NULL and can reject every row", - remediation: "Remove NULL from the list or express the anti-join with a NULL-safe NOT EXISTS.", -}; -const SET_ARITY_MISMATCH: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL020", - message: "Set-operation branches project different column counts", - remediation: "Project the same number of columns in every set-operation branch.", -}; -const PROJECTED_STAR: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL021", - message: "Query output has an uncontrolled column shape", - remediation: "Enumerate the intended output columns explicitly.", -}; -const UNALIASED_CALCULATION: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL022", - message: "Calculated projection has no stable output name", - remediation: "Add an explicit AS alias that describes the projected value.", -}; -const UNUSED_TABLE_ALIAS: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL023", - message: "Table alias is never referenced", - remediation: "Use the alias to qualify references or remove it.", -}; -const NULL_REJECTED_LEFT_JOIN: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL024", - message: "WHERE predicate rejects NULL rows from a LEFT JOIN", - remediation: "Move the right-side predicate into ON or use INNER JOIN when row rejection is intentional.", -}; -const IMPLICIT_INNER_JOIN: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL025", - message: "JOIN type is implicit", - remediation: "Write INNER JOIN explicitly.", -}; -const AMBIGUOUS_ORDER_DIRECTIONS: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL026", - message: "ORDER BY mixes explicit and implicit directions", - remediation: "State ASC or DESC for every ordering expression in this clause.", -}; -const UNQUALIFIED_MULTI_SOURCE_COLUMN: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL027", - message: "Column reference is unqualified in a multi-relation query", - remediation: "Qualify the column with its relation alias.", -}; -const INCONSISTENT_SINGLE_SOURCE_QUALIFICATION: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL028", - message: "Single-relation query mixes qualified and unqualified columns", - remediation: "Use one consistent qualification style in this query scope.", -}; -const UNKNOWN_RELATION_QUALIFIER: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL029", - message: "Qualified reference uses a relation absent from this query scope", - remediation: "Use an available relation alias or add the intended relation to FROM.", -}; -const SIMPLE_BOOLEAN_CASE: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL030", - message: "CASE expression only converts a condition to TRUE or FALSE", - remediation: "Use a NULL-safe boolean expression directly.", -}; -const NESTED_ELSE_CASE: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL031", - message: "CASE expression is nested directly inside ELSE", - remediation: "Flatten the nested branch into the surrounding CASE expression.", -}; -const UNUSED_JOINED_RELATION: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL032", - message: "Joined relation contributes no referenced values", - remediation: "Use the joined relation or remove the join. When the join intentionally filters rows, record it as a reviewed [[rules.rule_exceptions]] entry with a reason in sqlbuild_project.toml.", -}; -const COMMENT_ATTACHMENT: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL033", - message: "Comment is not attached to the syntax node it explains", - remediation: "Use a standalone -- comment block immediately above the relevant CTE, JOIN, filter, predicate, selected expression, CASE branch, grouping, or ordering node.", -}; -const CTE_ONLY_BODY: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL034", - message: "Query keeps transformation logic outside top-level CTEs", - remediation: "Move transformation logic into named top-level CTEs before the terminal SELECT.", -}; -const TERMINAL_SELECT: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL035", - message: "Terminal SELECT does not read plainly from the final top-level CTE", - remediation: "Name the final logical CTE and leave one terminal SELECT from that CTE.", -}; -const NESTED_CTE: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL036", - message: "CTE is nested inside another query", - remediation: "Hoist the nested CTE into the statement's top-level WITH list.", -}; -const RECURSIVE_CTE: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL037", - message: "Recursive CTE is not permitted", - remediation: "Replace recursion with an explicit upstream relation or a bounded non-recursive query shape.", -}; -const CROSS_JOIN: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL038", - message: "CROSS JOIN creates an explicit cartesian product", - remediation: "Use a keyed JOIN, or make one side provably at most one row (LIMIT 1, an aggregate without GROUP BY, or a one-row seed). When the cartesian product is intentional, record it as a reviewed [[rules.rule_exceptions]] entry with rule, path, relation and reason in sqlbuild_project.toml.", -}; -const INLINE_QUERY_RELATION: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL039", - message: "Inline query relation obscures data flow", - remediation: "Extract the derived relation into a named top-level CTE; use explicit LATERAL only when the relation must remain correlated.", -}; -const TRIVIAL_EQUALITY_TOKEN_COUNT: usize = 3; -const JOIN_EXPRESSION: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL040", - message: "JOIN ON contains a computed expression", - remediation: "Compute keys, including variant path access, in input CTEs and join on plain columns. Literal-only predicates such as ON TRUE are allowed.", -}; -const FINAL_CTE_NAME: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL041", - message: "Only the last top-level CTE may be named final, and it must use that name", - remediation: "Name the last top-level CTE final and rename any other final CTE; SQBRSQL035 checks the terminal SELECT.", -}; - -const DEFAULT_RULES: [&str; 13] = [ - NULL_COMPARISON.code, - IMPLICIT_CARTESIAN_JOIN.code, - JOIN_WITHOUT_CONDITION.code, - UNORDERED_LIMIT.code, - UNUSED_CTE.code, - REDUNDANT_DISTINCT.code, - POSITIONAL_SET_STAR.code, - DUPLICATE_TABLE_ALIAS.code, - DUPLICATE_OUTPUT_ALIAS.code, - UNSTABLE_ROW_NUMBER.code, - NULL_NOT_IN.code, - SET_ARITY_MISMATCH.code, - INLINE_QUERY_RELATION.code, -]; - -const ALL_RULE_METADATA: [&LintRuleMetadata; 45] = [ - &NULL_COMPARISON, - &IMPLICIT_CARTESIAN_JOIN, - &JOIN_WITHOUT_CONDITION, - &UNORDERED_LIMIT, - &UNUSED_CTE, - &REDUNDANT_DISTINCT, - &POSITIONAL_SET_STAR, - &EXPLICIT_UNION, - &DUPLICATE_TABLE_ALIAS, - &DUPLICATE_OUTPUT_ALIAS, - &MIXED_GROUP_ORDER_REFERENCES, - &REDUNDANT_ELSE_NULL, - &PARENTHESIZED_DISTINCT, - &CONSTANT_PREDICATE, - &CONSECUTIVE_SEMICOLON, - &REDUNDANT_SELF_ALIAS, - &COUNT_ONE, - &UNSTABLE_ROW_NUMBER, - &NULL_NOT_IN, - &SET_ARITY_MISMATCH, - &PROJECTED_STAR, - &UNALIASED_CALCULATION, - &UNUSED_TABLE_ALIAS, - &NULL_REJECTED_LEFT_JOIN, - &IMPLICIT_INNER_JOIN, - &AMBIGUOUS_ORDER_DIRECTIONS, - &UNQUALIFIED_MULTI_SOURCE_COLUMN, - &INCONSISTENT_SINGLE_SOURCE_QUALIFICATION, - &UNKNOWN_RELATION_QUALIFIER, - &SIMPLE_BOOLEAN_CASE, - &NESTED_ELSE_CASE, - &UNUSED_JOINED_RELATION, - &COMMENT_ATTACHMENT, - &CTE_ONLY_BODY, - &TERMINAL_SELECT, - &NESTED_CTE, - &RECURSIVE_CTE, - &CROSS_JOIN, - &INLINE_QUERY_RELATION, - &JOIN_EXPRESSION, - &FINAL_CTE_NAME, - &crate::sql_quality::constants::UNUSED_CTE_OUTPUT, - &crate::sql_quality::constants::RANKING_SORT_CAP, - &crate::sql_quality::constants::LONG_LITERAL, - &crate::sql_quality::constants::REPEATED_JSON_PARSE, -]; - -fn is_ceremonial_cte_name(name: &str) -> bool { - CEREMONIAL_CTE_PREFIXES - .iter() - .any(|prefix| name.starts_with(prefix)) -} - -struct QueryTokenContext<'a> { - tokens: &'a [Token], - depths: &'a [usize], - direct: &'a [usize], - query_end: usize, - depth: usize, -} - -struct ExpressionRange { - start: usize, - end: usize, - depth: usize, - strip_alias: bool, -} - -struct DiagnosticContext<'a> { - sql: &'a str, - dialect: DialectType, - facts: &'a QueryFacts, - tokens: &'a [Token], - enabled: &'a HashSet, -} - -struct FactBuildOptions<'a> { - external_identifiers: &'a HashSet, - sql_macro_outputs: &'a [SqlMacroOutput], - dependency_identifiers: &'a HashSet, - externally_referenced_ctes: &'a HashSet, - allows_ceremonial_select: bool, - allows_dynamic_output_star: bool, - allows_empty_fixture_star: bool, -} - -pub(crate) fn lint_json_impl(request_json: &str) -> Result { - let request: LintRequest = - serde_json::from_str(request_json).map_err(|error| error.to_string())?; - let response = lint(request)?; - serde_json::to_string(&response).map_err(|error| error.to_string()) -} - -pub(crate) fn rule_metadata() -> Vec { - ALL_RULE_METADATA.iter().map(|rule| **rule).collect() -} - -pub(crate) fn lint(request: LintRequest) -> Result { - if request.version != LINT_API_VERSION { - return Err(format!( - "unsupported native lint request version {}; expected {LINT_API_VERSION}", - request.version - )); - } - let enabled = enabled_rules(request.enabled_rules, &request.ignored_rules)?; - let dialect_type = - DialectType::from_str(&request.dialect).map_err(|error| error.to_string())?; - let dialect = Dialect::get(dialect_type); - let tokens = dialect - .tokenize(&request.sql) - .map_err(|error| error.to_string())?; - let mut parser = Parser::with_config( - tokens.clone(), - ParserConfig { - dialect: Some(dialect_type), - complexity_guard: ComplexityGuardOptions { - max_function_call_depth: Some(MAX_LINT_FUNCTION_CALL_DEPTH), - ..Default::default() - }, - ..Default::default() - }, - ); - let statements = parser.parse().map_err(|error| error.to_string())?; - for statement in &statements { - if statement - .dfs() - .any(|node| matches!(node, Expression::Command(_) | Expression::Raw(_))) - { - return Err("unsupported syntax: native parser retained an opaque SQL node".to_owned()); - } - } - let external_identifiers: HashSet = request - .external_identifiers - .iter() - .map(|identifier| identifier.to_ascii_lowercase()) - .collect(); - let dependency_identifiers: HashSet = request - .dependency_identifiers - .iter() - .map(|identifier| identifier.to_ascii_lowercase()) - .collect(); - let externally_referenced_ctes: HashSet = request - .externally_referenced_ctes - .iter() - .map(|name| name.to_ascii_lowercase()) - .collect(); - let options = FactBuildOptions { - external_identifiers: &external_identifiers, - sql_macro_outputs: &request.sql_macro_outputs, - dependency_identifiers: &dependency_identifiers, - externally_referenced_ctes: &externally_referenced_ctes, - allows_ceremonial_select: request.allows_ceremonial_select, - allows_dynamic_output_star: request.allows_dynamic_output_star, - allows_empty_fixture_star: request.allows_empty_fixture_star, - }; - let facts = build_facts(&statements, &tokens, &request.sql, &options); - let context = DiagnosticContext { - sql: &request.sql, - dialect: dialect_type, - facts: &facts, - tokens: &tokens, - enabled: &enabled, - }; - let mut diagnostics = diagnostics(&context); - if enabled.contains(CROSS_JOIN.code) { - let cross_joins = CrossJoinRequest { - statements: &statements, - tokens: &tokens, - cross_spans: &facts.cross_joins, - relation_keys: &request.relation_keys, - relation_names: &request.relation_names, - }; - diagnostics.extend(cross_join_diagnostics(&cross_joins, &CROSS_JOIN)); - } - let depths = token_depths(&tokens); - if enabled.contains(JOIN_EXPRESSION.code) { - diagnostics.extend(diagnostics_for_spans( - &JOIN_EXPRESSION, - &crate::sql_lint::_helpers::join_predicates::join_expression_spans( - &tokens, - &depths, - dialect_type, - ), - Some("computing a join key requires an authored CTE rewrite"), - )); - } - if enabled.contains(FINAL_CTE_NAME.code) && !request.allows_ceremonial_select { - diagnostics.extend(diagnostics_for_spans( - &FINAL_CTE_NAME, - &crate::sql_lint::_helpers::terminal_shape::final_cte_name_spans(&tokens, &depths), - Some("renaming a CTE requires updating its references"), - )); - } - diagnostics.extend(crate::sql_quality::main::diagnostics::diagnostics( - &crate::sql_quality::models::QualityRequest { - statements: &statements, - tokens: &tokens, - dialect: dialect_type, - enabled: &enabled, - max_literal_length: request - .max_literal_length - .unwrap_or(crate::sql_quality::constants::DEFAULT_MAX_LITERAL_LENGTH), - max_ranking_order_by: request - .max_ranking_order_by - .unwrap_or(crate::sql_quality::constants::DEFAULT_MAX_RANKING_ORDER_BY), - relation_keys: &request.relation_keys, - externally_referenced_ctes: &externally_referenced_ctes, - fixture_body: request.allows_ceremonial_select, - }, - )?); - diagnostics.sort_by_key(|diagnostic| (diagnostic.start, diagnostic.end, diagnostic.code)); - Ok(LintResponse { - version: LINT_API_VERSION, - diagnostics, - }) -} - -fn enabled_rules( - requested: Option>, - ignored: &[String], -) -> Result, String> { - let selectors = - requested.unwrap_or_else(|| DEFAULT_RULES.iter().map(ToString::to_string).collect()); - let mut rules = expand_rule_selectors(&selectors)?; - for code in expand_rule_selectors(ignored)? { - rules.remove(&code); - } - Ok(rules) -} - -fn expand_rule_selectors(selectors: &[String]) -> Result, String> { - let mut selected: HashSet = HashSet::new(); - for selector in selectors { - let matches: Vec<&str> = ALL_RULE_METADATA - .iter() - .map(|rule| rule.code) - .filter(|code| code.starts_with(selector)) - .collect(); - if matches.is_empty() { - return Err(format!("unknown native lint rule '{selector}'")); - } - selected.extend(matches.into_iter().map(ToString::to_string)); - } - Ok(selected) -} - -fn build_facts( - expressions: &[Expression], - tokens: &[Token], - sql: &str, - options: &FactBuildOptions<'_>, -) -> QueryFacts { - let mut facts: QueryFacts = - collect_token_query_facts(tokens, sql, options.allows_ceremonial_select); - facts.null_comparisons = null_comparison_spans(tokens); - facts.additional = collect_additional_facts( - tokens, - &AdditionalFactOptions { - external_identifiers: options.external_identifiers, - sql_macro_outputs: options.sql_macro_outputs, - dependency_identifiers: options.dependency_identifiers, - allows_ceremonial_select: options.allows_ceremonial_select, - allows_dynamic_output_star: options.allows_dynamic_output_star, - allows_empty_fixture_star: options.allows_empty_fixture_star, - }, - ); - for expression in expressions { - facts - .unused_cte_names - .extend(collect_unused_ctes(expression, options)); - } - facts -} - -fn collect_token_query_facts( - tokens: &[Token], - sql: &str, - allows_ceremonial_select: bool, -) -> QueryFacts { - let depths = token_depths(tokens); - let mut facts = collect_rule_migration_facts(tokens, &depths, sql, allows_ceremonial_select); - for (select_index, token) in tokens.iter().enumerate() { - if token.token_type != TokenType::Select { - continue; - } - let depth = depths[select_index]; - let end = query_end(tokens, &depths, select_index, depth); - let direct = direct_indices(&depths, select_index + 1, end, depth); - let order = first_type(tokens, &direct, TokenType::Order); - let limit = first_row_selection_clause(tokens, &direct); - if order.is_none() - && let Some(index) = limit - && !limits_to_no_rows(tokens, index) - { - facts.unordered_limits.push(tokens[index].span); - } - let context = QueryTokenContext { - tokens, - depths: &depths, - direct: &direct, - query_end: end, - depth, - }; - if let Some(span) = redundant_distinct_span(&context) { - facts.redundant_distincts.push(span); - } - let (implicit, missing_conditions) = collect_from_and_join_facts(tokens, &direct); - facts.implicit_cartesian_joins.extend(implicit); - facts.joins_without_condition.extend(missing_conditions); - } - facts.positional_set_stars = collect_set_operation_facts(tokens, &depths); - facts -} - -fn collect_rule_migration_facts( - tokens: &[Token], - depths: &[usize], - sql: &str, - allows_ceremonial_select: bool, -) -> QueryFacts { - let mut facts = QueryFacts::default(); - let significant: Vec = (0..tokens.len()) - .filter(|&index| !is_layout(&tokens[index]) && !is_comment(&tokens[index])) - .collect(); - - let root_with = crate::sql_lint::_helpers::terminal_shape::root_with_indices(tokens, depths); - - for (position, &index) in significant.iter().enumerate() { - let text = tokens[index].text.as_str(); - if text.eq_ignore_ascii_case("cross") - && significant - .get(position + 1) - .is_some_and(|&next| tokens[next].token_type == TokenType::Join) - { - facts.cross_joins.push(tokens[index].span); - } - if tokens[index].token_type == TokenType::With { - let starts_hierarchy = position.checked_sub(1).is_some_and(|previous| { - tokens[significant[previous]].token_type == TokenType::Start - }); - if !root_with.contains(&index) && !starts_hierarchy { - facts.nested_ctes.push(tokens[index].span); - } - if !starts_hierarchy - && significant - .get(position + 1) - .is_some_and(|&next| tokens[next].text.eq_ignore_ascii_case("recursive")) - { - facts.recursive_ctes.push(tokens[index].span); - } - } - } - facts - .inline_query_relations - .extend(collect_inline_query_relation_spans( - tokens, - depths, - &significant, - )); - - let (cte_only_bodies, terminal_selects) = - collect_terminal_shape_facts(tokens, depths, &significant, allows_ceremonial_select); - facts.cte_only_bodies.extend(cte_only_bodies); - facts.terminal_selects.extend(terminal_selects); - facts - .invalid_comment_attachments - .extend(collect_comment_attachment_facts(tokens, sql)); - facts -} - -fn collect_comment_attachment_facts(tokens: &[Token], sql: &str) -> Vec { - let mut invalid: Vec = Vec::new(); - for (index, token) in tokens.iter().enumerate() { - if !token.trailing_comments.is_empty() { - invalid.push(token.span); - } - if token.comments.is_empty() { - continue; - } - let gap_start = index - .checked_sub(1) - .map_or(0, |previous| tokens[previous].span.end); - let gap = char_slice(sql, gap_start, token.span.start).unwrap_or_default(); - let block_comment = gap.contains("/*"); - let detached = gap.replace("\r\n", "\n").ends_with("\n\n"); - let unsupported_target = matches!( - token.token_type, - TokenType::Comma - | TokenType::RParen - | TokenType::Semicolon - | TokenType::Then - | TokenType::Else - | TokenType::From - ); - if block_comment || detached || unsupported_target { - invalid.push(token.span); - } - } - invalid -} - -pub(super) fn token_depths(tokens: &[Token]) -> Vec { - let mut depth = 0_usize; - tokens - .iter() - .map(|token| { - let current = depth; - if matches!( - token.token_type, - TokenType::LParen | TokenType::LBracket | TokenType::LBrace - ) { - depth += 1; - } else if matches!( - token.token_type, - TokenType::RParen | TokenType::RBracket | TokenType::RBrace - ) { - depth = depth.saturating_sub(1); - } - current - }) - .collect() -} - -pub(super) fn query_end(tokens: &[Token], depths: &[usize], start: usize, depth: usize) -> usize { - (start + 1..tokens.len()) - .find(|&index| { - depths[index] < depth - || depths[index] == depth - && matches!( - tokens[index].token_type, - TokenType::RParen - | TokenType::Semicolon - | TokenType::Union - | TokenType::Intersect - | TokenType::Except - ) - }) - .unwrap_or(tokens.len()) -} - -pub(super) fn direct_indices( - depths: &[usize], - start: usize, - end: usize, - depth: usize, -) -> Vec { - (start..end) - .filter(|&index| depths[index] == depth) - .collect() -} - -fn first_type(tokens: &[Token], indices: &[usize], token_type: TokenType) -> Option { - indices - .iter() - .copied() - .find(|&index| tokens[index].token_type == token_type) -} - -fn first_row_selection_clause(tokens: &[Token], direct: &[usize]) -> Option { - let from = direct - .iter() - .copied() - .find(|&index| is_query_from(tokens, index)); - direct.iter().copied().find(|&index| { - matches!( - tokens[index].token_type, - TokenType::Limit | TokenType::Offset - ) && from.is_none_or(|from_index| index > from_index) - && significant_before(tokens, index).is_none_or(|previous| { - !matches!( - tokens[previous].token_type, - TokenType::Colon | TokenType::DColon | TokenType::Dot | TokenType::DotColon - ) - }) - }) -} - -/// Whether a row-selection clause is `LIMIT 0`, which selects no rows in any order. -fn limits_to_no_rows(tokens: &[Token], index: usize) -> bool { - tokens[index].token_type == TokenType::Limit - && tokens.get(index + 1).is_some_and(|count| { - count.token_type == TokenType::Number && count.text.parse::() == Ok(0) - }) -} - -pub(super) fn is_query_from(tokens: &[Token], index: usize) -> bool { - tokens[index].token_type == TokenType::From - && significant_before(tokens, index) - .is_none_or(|previous| tokens[previous].token_type != TokenType::Distinct) -} - -fn redundant_distinct_span(context: &QueryTokenContext<'_>) -> Option { - let QueryTokenContext { - tokens, - depths, - direct, - query_end, - depth, - } = context; - let group_position = group_by_position(tokens, direct)?; - let group_index = direct[group_position]; - let projection_end = direct - .iter() - .copied() - .find(|&index| is_query_from(tokens, index)) - .filter(|index| *index < group_index) - .unwrap_or(group_index); - let distinct_position = direct.iter().position(|&index| { - index < projection_end && tokens[index].token_type == TokenType::Distinct - })?; - if direct - .get(distinct_position + 1) - .is_some_and(|&index| tokens[index].token_type == TokenType::On) - { - return None; - } - let projection_start = direct[distinct_position] + 1; - let group_start = *direct.get(group_position + 2)?; - let group_end = direct[group_position + 2..] - .iter() - .copied() - .find(|&index| { - matches!( - tokens[index].token_type, - TokenType::Having - | TokenType::Qualify - | TokenType::Order - | TokenType::Limit - | TokenType::Offset - ) - }) - .unwrap_or(*query_end); - let projection_signatures = expression_signatures( - tokens, - depths, - &ExpressionRange { - start: projection_start, - end: projection_end, - depth: *depth, - strip_alias: true, - }, - ); - let group_signatures = expression_signatures( - tokens, - depths, - &ExpressionRange { - start: group_start, - end: group_end, - depth: *depth, - strip_alias: false, - }, - ); - if group_signatures.is_empty() - || !group_signatures - .iter() - .all(|group| projection_signatures.contains(group)) - { - return None; - } - Some(tokens[direct[distinct_position]].span) -} - -fn group_by_position(tokens: &[Token], direct: &[usize]) -> Option { - direct.windows(2).position(|window| { - tokens[window[0]].token_type == TokenType::Group - && tokens[window[1]].text.eq_ignore_ascii_case("BY") - }) -} - -fn expression_signatures( - tokens: &[Token], - depths: &[usize], - range: &ExpressionRange, -) -> Vec { - let mut boundaries: Vec = (range.start..range.end) - .filter(|&index| { - depths[index] == range.depth && tokens[index].token_type == TokenType::Comma - }) - .collect(); - boundaries.push(range.end); - let mut segment_start = range.start; - let mut signatures: Vec = Vec::new(); - for segment_end in boundaries { - let effective_end = alias_start(tokens, depths, range, segment_start..segment_end); - let parts: Vec = tokens[segment_start..effective_end] - .iter() - .map(|token| token.text.to_ascii_lowercase()) - .collect(); - let signature = parts.join(" "); - if !signature.is_empty() { - signatures.push(signature); - } - segment_start = segment_end + 1; - } - signatures -} - -fn alias_start( - tokens: &[Token], - depths: &[usize], - range: &ExpressionRange, - segment: std::ops::Range, -) -> usize { - if !range.strip_alias { - return segment.end; - } - segment - .clone() - .find(|&index| depths[index] == range.depth && tokens[index].token_type == TokenType::As) - .unwrap_or(segment.end) -} - -fn collect_from_and_join_facts(tokens: &[Token], direct: &[usize]) -> (Vec, Vec) { - let mut implicit: Vec = Vec::new(); - let mut missing_conditions: Vec = Vec::new(); - let Some(from_position) = direct - .iter() - .position(|&index| is_query_from(tokens, index)) - else { - return (implicit, missing_conditions); - }; - let clause_end = direct[from_position + 1..] - .iter() - .position(|&index| is_post_from_clause(tokens[index].token_type)) - .map_or(direct.len(), |offset| from_position + 1 + offset); - let relation_clause = &direct[from_position + 1..clause_end]; - let values_relation = relation_clause - .first() - .is_some_and(|&index| tokens[index].token_type == TokenType::Values); - for (position, &index) in relation_clause.iter().enumerate() { - if tokens[index].token_type == TokenType::Comma - && !values_relation - && relation_clause - .get(position + 1) - .is_none_or(|&next| tokens[next].token_type != TokenType::Lateral) - { - implicit.push(tokens[index].span); - } - } - for (position, &index) in direct.iter().enumerate() { - if tokens[index].token_type != TokenType::Join - || is_conditionless_join(tokens, direct, position) - { - continue; - } - let condition_end = direct[position + 1..] - .iter() - .position(|&candidate| { - tokens[candidate].token_type == TokenType::Join - || is_post_from_clause(tokens[candidate].token_type) - }) - .map_or(direct.len(), |offset| position + 1 + offset); - let condition = &direct[position + 1..condition_end]; - let on_position = condition - .iter() - .position(|&candidate| tokens[candidate].token_type == TokenType::On); - let has_using = condition - .iter() - .any(|&candidate| tokens[candidate].token_type == TokenType::Using); - if !has_using - && on_position - .is_none_or(|offset| trivial_on_condition(tokens, &condition[offset + 1..])) - { - missing_conditions.push(tokens[index].span); - } - } - (implicit, missing_conditions) -} - -fn is_post_from_clause(token_type: TokenType) -> bool { - matches!( - token_type, - TokenType::Join - | TokenType::Where - | TokenType::Group - | TokenType::Having - | TokenType::Qualify - | TokenType::Order - | TokenType::Limit - | TokenType::Offset - | TokenType::Fetch - ) -} - -fn is_conditionless_join(tokens: &[Token], direct: &[usize], join_position: usize) -> bool { - direct[..join_position] - .iter() - .rev() - .take_while(|&&index| { - !matches!( - tokens[index].token_type, - TokenType::From | TokenType::Join | TokenType::Comma - ) - }) - .any(|&index| { - matches!( - tokens[index].text.to_ascii_uppercase().as_str(), - "CROSS" | "NATURAL" | "APPLY" | "POSITIONAL" | "PASTE" - ) - }) -} - -fn trivial_on_condition(tokens: &[Token], condition: &[usize]) -> bool { - let significant: Vec<&Token> = condition - .iter() - .map(|&index| &tokens[index]) - .filter(|token| !matches!(token.token_type, TokenType::LParen | TokenType::RParen)) - .collect(); - significant.is_empty() - || significant.len() == 1 && significant[0].token_type == TokenType::True - || significant.len() == TRIVIAL_EQUALITY_TOKEN_COUNT - && significant[1].token_type == TokenType::Eq - && matches!( - significant[0].token_type, - TokenType::Number | TokenType::String | TokenType::True | TokenType::False - ) - && significant[0].token_type == significant[2].token_type - && significant[0].text == significant[2].text -} - -fn collect_set_operation_facts(tokens: &[Token], depths: &[usize]) -> Vec { - let mut spans: Vec = Vec::new(); - for (index, token) in tokens.iter().enumerate() { - if !matches!( - token.token_type, - TokenType::Union | TokenType::Intersect | TokenType::Except - ) { - continue; - } - let depth = depths[index]; - let by_name = tokens[index + 1..] - .iter() - .take_while(|candidate| candidate.token_type != TokenType::Select) - .any(|candidate| candidate.text.eq_ignore_ascii_case("BY")) - && tokens[index + 1..] - .iter() - .take_while(|candidate| candidate.token_type != TokenType::Select) - .any(|candidate| candidate.text.eq_ignore_ascii_case("NAME")); - if !by_name - && (select_before_projects_star(tokens, depths, index, depth) - || select_after_projects_star(tokens, depths, index, depth)) - { - spans.push(token.span); - } - } - spans -} - -fn select_before_projects_star( - tokens: &[Token], - depths: &[usize], - end: usize, - depth: usize, -) -> bool { - let Some(start) = (0..end) - .rev() - .find(|&index| depths[index] == depth && tokens[index].token_type == TokenType::Select) - .or_else(|| { - (0..end) - .rev() - .filter(|&index| tokens[index].token_type == TokenType::Select) - .min_by_key(|&index| depths[index]) - }) - else { - return false; - }; - projection_projects_star( - tokens, - depths, - &ExpressionRange { - start, - end, - depth: depths[start], - strip_alias: false, - }, - ) -} - -fn select_after_projects_star( - tokens: &[Token], - depths: &[usize], - start: usize, - depth: usize, -) -> bool { - let Some(select) = (start + 1..tokens.len()) - .find(|&index| depths[index] == depth && tokens[index].token_type == TokenType::Select) - .or_else(|| { - (start + 1..tokens.len()) - .filter(|&index| tokens[index].token_type == TokenType::Select) - .min_by_key(|&index| depths[index]) - }) - else { - return false; - }; - let select_depth = depths[select]; - projection_projects_star( - tokens, - depths, - &ExpressionRange { - start: select, - end: query_end(tokens, depths, select, select_depth), - depth: select_depth, - strip_alias: false, - }, - ) -} - -fn projection_projects_star(tokens: &[Token], depths: &[usize], range: &ExpressionRange) -> bool { - (range.start + 1..range.end) - .take_while(|&index| depths[index] != range.depth || !is_query_from(tokens, index)) - .any(|index| depths[index] == range.depth && tokens[index].token_type == TokenType::Star) -} - -fn collect_unused_ctes(expression: &Expression, options: &FactBuildOptions<'_>) -> Vec { - let mut unused: Vec = Vec::new(); - for node in expression.dfs() { - match node { - Expression::Select(select) if select.with.is_some() => { - let mut root = select.clone(); - if let Some(with) = root.with.take() { - unused.extend(collect_unused_for_with( - &with, - &Expression::Select(root), - options, - )); - } - } - Expression::Union(operation) if operation.with.is_some() => { - let mut root = operation.clone(); - if let Some(with) = root.with.take() { - unused.extend(collect_unused_for_with( - &with, - &Expression::Union(root), - options, - )); - } - } - Expression::Intersect(operation) if operation.with.is_some() => { - let mut root = operation.clone(); - if let Some(with) = root.with.take() { - unused.extend(collect_unused_for_with( - &with, - &Expression::Intersect(root), - options, - )); - } - } - Expression::Except(operation) if operation.with.is_some() => { - let mut root = operation.clone(); - if let Some(with) = root.with.take() { - unused.extend(collect_unused_for_with( - &with, - &Expression::Except(root), - options, - )); - } - } - _ => {} - }; - } - unused -} - -fn collect_unused_for_with( - with: &With, - root: &Expression, - options: &FactBuildOptions<'_>, -) -> Vec { - let names: HashSet = with - .ctes - .iter() - .map(|cte| cte.alias.name.to_ascii_lowercase()) - .collect(); - let dependencies: HashMap> = with - .ctes - .iter() - .map(|cte| { - ( - cte.alias.name.to_ascii_lowercase(), - table_names(&cte.this) - .intersection(&names) - .cloned() - .collect(), - ) - }) - .collect(); - let mut reachable: HashSet = table_names(root).intersection(&names).cloned().collect(); - reachable.extend( - options - .externally_referenced_ctes - .intersection(&names) - .cloned(), - ); - if options.allows_ceremonial_select { - reachable.extend( - names - .iter() - .filter(|name| is_ceremonial_cte_name(name)) - .cloned(), - ); - } - let mut queue: VecDeque = reachable.iter().cloned().collect(); - while let Some(name) = queue.pop_front() { - if let Some(required) = dependencies.get(&name) { - for dependency in required { - if reachable.insert(dependency.clone()) { - queue.push_back(dependency.clone()); - } - } - } - } - with.ctes - .iter() - .filter(|cte| !reachable.contains(&cte.alias.name.to_ascii_lowercase())) - .map(|cte| cte.alias.name.clone()) - .collect() -} - -fn table_names(expression: &Expression) -> HashSet { - expression - .dfs() - .filter_map(|node| match node { - Expression::Table(table) => Some(table.name.name.to_ascii_lowercase()), - _ => None, - }) - .collect() -} - -fn null_comparison_fix(tokens: &[Token], span: Span) -> Option { - let index = token_index_for_span(tokens, span)?; - let token = &tokens[index]; - if !adjacent_null(tokens, index, true) { - return None; - } - let replacement = match token.token_type { - TokenType::Eq => "IS", - TokenType::Neq => "IS NOT", - _ => return None, - }; - Some(LintEdit { - start: span.start, - end: span.end, - replacement: replacement.to_string(), - }) -} - -fn redundant_distinct_fix(tokens: &[Token], span: Span) -> LintEdit { - let end = token_index_for_span(tokens, span) - .and_then(|index| significant_after(tokens, index)) - .filter(|&index| !is_comment(&tokens[index])) - .map_or(span.end, |index| tokens[index].span.start); - LintEdit { - start: span.start, - end, - replacement: String::new(), - } -} - -fn implicit_cartesian_fix(sql: &str, tokens: &[Token], span: Span) -> Option { - let comma_index = token_index_for_span(tokens, span)?; - let depths = token_depths(tokens); - let depth = depths[comma_index]; - let from_index = (0..comma_index).rev().find(|&index| { - depths[index] == depth - && (is_query_from(tokens, index) || tokens[index].token_type == TokenType::Semicolon) - })?; - if !is_query_from(tokens, from_index) { - return None; - } - let clause_end = (comma_index + 1..tokens.len()) - .find(|&index| depths[index] == depth && is_post_from_clause(tokens[index].token_type)) - .unwrap_or(tokens.len()); - if tokens[from_index + 1..clause_end] - .iter() - .any(|token| token.token_type == TokenType::Join) - { - return None; - } - let next_index = significant_after(tokens, comma_index)?; - let end = tokens[next_index].span.start; - let replaced = char_slice(sql, span.start, end)?; - if contains_comment(replaced) { - return None; - } - Some(LintEdit { - start: span.start, - end, - replacement: " CROSS JOIN ".to_string(), - }) -} - -fn conditionless_join_fix(tokens: &[Token], span: Span) -> Option { - let join_index = token_index_for_span(tokens, span)?; - let depths = token_depths(tokens); - let depth = depths[join_index]; - if significant_before(tokens, join_index).is_some_and(|index| { - matches!( - tokens[index].text.to_ascii_uppercase().as_str(), - "INNER" - | "LEFT" - | "RIGHT" - | "FULL" - | "OUTER" - | "CROSS" - | "NATURAL" - | "ASOF" - | "SEMI" - | "ANTI" - ) - }) { - return None; - } - let condition_end = (join_index + 1..tokens.len()) - .find(|&index| { - depths[index] == depth - && (tokens[index].token_type == TokenType::Join - || is_post_from_clause(tokens[index].token_type)) - }) - .unwrap_or(tokens.len()); - if tokens[join_index + 1..condition_end] - .iter() - .any(|token| matches!(token.token_type, TokenType::On | TokenType::Using)) - { - return None; - } - Some(LintEdit { - start: span.start, - end: span.end, - replacement: "CROSS JOIN".to_string(), - }) -} - -fn unused_cte_fix(sql: &str, tokens: &[Token], name_span: Span) -> Option { - let name_index = token_index_for_span(tokens, name_span)?; - if tokens[name_index].text.starts_with("__") { - return None; - } - let as_index = significant_after(tokens, name_index)?; - if tokens[as_index].token_type != TokenType::As { - return None; - } - let open_index = significant_after(tokens, as_index)?; - if tokens[open_index].token_type != TokenType::LParen { - return None; - } - let depths = token_depths(tokens); - let close_depth = depths[open_index] + 1; - let close_index = (open_index + 1..tokens.len()).find(|&index| { - tokens[index].token_type == TokenType::RParen && depths[index] == close_depth - })?; - let body_tokens = &tokens[open_index + 1..close_index]; - let first_body = body_tokens.iter().find(|token| !is_layout(token))?; - if !matches!(first_body.token_type, TokenType::Select | TokenType::With) - || body_tokens.iter().any(|token| { - matches!( - token.text.to_ascii_uppercase().as_str(), - "INSERT" | "UPDATE" | "DELETE" | "MERGE" - ) - }) - { - return None; - } - let previous_index = significant_before(tokens, name_index)?; - let next_index = significant_after(tokens, close_index)?; - let (start, end) = if tokens[next_index].token_type == TokenType::Comma { - let following_index = significant_after(tokens, next_index)?; - (name_span.start, tokens[following_index].span.start) - } else if tokens[previous_index].token_type == TokenType::Comma { - ( - tokens[previous_index].span.start, - tokens[close_index].span.end, - ) - } else if tokens[previous_index].token_type == TokenType::With { - ( - tokens[previous_index].span.start, - tokens[next_index].span.start, - ) - } else { - return None; - }; - if contains_comment(char_slice(sql, start, end)?) { - return None; - } - Some(LintEdit { - start, - end, - replacement: String::new(), - }) -} - -fn token_index_for_span(tokens: &[Token], span: Span) -> Option { - tokens.iter().position(|token| token.span == span) -} - -pub(super) fn significant_before(tokens: &[Token], index: usize) -> Option { - (0..index) - .rev() - .find(|&candidate| !is_layout(&tokens[candidate])) -} - -pub(super) fn significant_after(tokens: &[Token], index: usize) -> Option { - (index + 1..tokens.len()).find(|&candidate| !is_layout(&tokens[candidate])) -} - -pub(super) fn is_layout(token: &Token) -> bool { - matches!(token.token_type, TokenType::Space | TokenType::Break) -} - -pub(super) fn is_comment(token: &Token) -> bool { - matches!( - token.token_type, - TokenType::LineComment | TokenType::BlockComment - ) -} - -fn diagnostics(context: &DiagnosticContext<'_>) -> Vec { - let mut output: Vec = core_diagnostics(context); - output.extend(additional_style_diagnostics(context)); - output.extend(additional_relation_diagnostics(context)); - output.extend(rule_migration_diagnostics(context)); - output -} - -fn core_diagnostics(context: &DiagnosticContext<'_>) -> Vec { - let DiagnosticContext { - sql, - dialect, - facts, - tokens, - enabled, - } = context; - let mut diagnostics: Vec = Vec::new(); - if enabled.contains(NULL_COMPARISON.code) { - diagnostics.extend(facts.null_comparisons.iter().map(|span| { - let fix = null_comparison_fix(tokens, *span); - diagnostic( - &NULL_COMPARISON, - Some(*span), - fix, - Some("only equality and inequality comparisons have a deterministic repair"), - ) - })); - } - if enabled.contains(IMPLICIT_CARTESIAN_JOIN.code) { - diagnostics.extend(facts.implicit_cartesian_joins.iter().map(|span| { - let fix = implicit_cartesian_fix(sql, tokens, *span); - diagnostic( - &IMPLICIT_CARTESIAN_JOIN, - Some(*span), - fix, - Some("mixed joins, comments, or generated SQL require an explicit authored repair"), - ) - })); - } - if enabled.contains(JOIN_WITHOUT_CONDITION.code) { - diagnostics.extend(facts.joins_without_condition.iter().map(|span| { - let fix = conditionless_join_fix(tokens, *span); - diagnostic( - &JOIN_WITHOUT_CONDITION, - Some(*span), - fix, - Some("qualified joins or joins with placeholder conditions require user intent"), - ) - })); - } - if enabled.contains(UNORDERED_LIMIT.code) { - diagnostics.extend(diagnostics_for_spans( - &UNORDERED_LIMIT, - &facts.unordered_limits, - Some("deterministic ordering columns and tie-breakers require user intent"), - )); - } - if enabled.contains(UNUSED_CTE.code) { - for span in unused_cte_spans(tokens, &facts.unused_cte_names) { - let fix = unused_cte_fix(sql, tokens, span); - diagnostics.push(diagnostic( - &UNUSED_CTE, - Some(span), - fix, - Some("only contiguous authored, comment-free, select-only CTEs can be removed"), - )); - } - } - if enabled.contains(REDUNDANT_DISTINCT.code) { - diagnostics.extend(facts.redundant_distincts.iter().map(|span| { - diagnostic( - &REDUNDANT_DISTINCT, - Some(*span), - Some(redundant_distinct_fix(tokens, *span)), - None, - ) - })); - } - if enabled.contains(POSITIONAL_SET_STAR.code) { - diagnostics.extend(diagnostics_for_spans( - &POSITIONAL_SET_STAR, - &facts.positional_set_stars, - Some( - "column names and their intended positional order require resolved schema evidence", - ), - )); - } - if enabled.contains(EXPLICIT_UNION.code) { - diagnostics.extend(facts.additional.bare_unions.iter().map(|span| { - let fix = explicit_union_fix(*dialect, *span); - diagnostic( - &EXPLICIT_UNION, - Some(*span), - fix, - Some("the active dialect does not accept explicit UNION DISTINCT"), - ) - })); - } - diagnostics -} - -fn additional_style_diagnostics(context: &DiagnosticContext<'_>) -> Vec { - let DiagnosticContext { - sql, - facts, - enabled, - .. - } = context; - let mut diagnostics: Vec = Vec::new(); - if enabled.contains(DUPLICATE_TABLE_ALIAS.code) { - diagnostics.extend(diagnostics_for_spans( - &DUPLICATE_TABLE_ALIAS, - &facts.additional.duplicate_table_aliases, - Some("choosing a new relation alias requires user intent"), - )); - } - if enabled.contains(DUPLICATE_OUTPUT_ALIAS.code) { - diagnostics.extend(diagnostics_for_spans( - &DUPLICATE_OUTPUT_ALIAS, - &facts.additional.duplicate_output_aliases, - Some("choosing a stable output name requires user intent"), - )); - } - if enabled.contains(MIXED_GROUP_ORDER_REFERENCES.code) { - diagnostics.extend(diagnostics_for_spans( - &MIXED_GROUP_ORDER_REFERENCES, - &facts.additional.mixed_group_order_references, - Some("normalization requires reliable projection-name resolution"), - )); - } - if enabled.contains(REDUNDANT_ELSE_NULL.code) { - diagnostics.extend(facts.additional.redundant_else_nulls.iter().map(|span| { - diagnostic( - &REDUNDANT_ELSE_NULL, - Some(*span), - Some(deletion(*span)), - None, - ) - })); - } - if enabled.contains(PARENTHESIZED_DISTINCT.code) { - diagnostics.extend(facts.additional.parenthesized_distincts.iter().map(|span| { - diagnostic( - &PARENTHESIZED_DISTINCT, - Some(*span), - parenthesized_distinct_fix(sql, *span), - Some("comments or malformed parentheses prevent a lossless rewrite"), - ) - })); - } - if enabled.contains(CONSTANT_PREDICATE.code) { - diagnostics.extend(diagnostics_for_spans( - &CONSTANT_PREDICATE, - &facts.additional.constant_predicates, - Some("removing or replacing a scaffold predicate requires user intent"), - )); - } - if enabled.contains(CONSECUTIVE_SEMICOLON.code) { - diagnostics.extend(facts.additional.consecutive_semicolons.iter().map(|span| { - diagnostic( - &CONSECUTIVE_SEMICOLON, - Some(*span), - Some(deletion(*span)), - None, - ) - })); - } - if enabled.contains(REDUNDANT_SELF_ALIAS.code) { - diagnostics.extend(facts.additional.redundant_self_aliases.iter().map(|span| { - diagnostic( - &REDUNDANT_SELF_ALIAS, - Some(*span), - None, - Some("removing the alias may change statically inferred output identity"), - ) - })); - } - if enabled.contains(COUNT_ONE.code) { - diagnostics.extend(facts.additional.count_one_literals.iter().map(|span| { - diagnostic( - &COUNT_ONE, - Some(*span), - Some(LintEdit { - start: span.start, - end: span.end, - replacement: "*".to_string(), - }), - None, - ) - })); - } - if enabled.contains(UNSTABLE_ROW_NUMBER.code) { - diagnostics.extend(diagnostics_for_spans( - &UNSTABLE_ROW_NUMBER, - &facts.additional.unstable_row_numbers, - Some("ordering columns and a stable tie-breaker require user intent"), - )); - } - diagnostics -} - -fn additional_relation_diagnostics(context: &DiagnosticContext<'_>) -> Vec { - let DiagnosticContext { - sql, - facts, - tokens, - enabled, - .. - } = context; - let mut diagnostics: Vec = Vec::new(); - if enabled.contains(NULL_NOT_IN.code) { - diagnostics.extend(diagnostics_for_spans( - &NULL_NOT_IN, - &facts.additional.null_not_in_predicates, - Some("the intended NULL and anti-join semantics require user intent"), - )); - } - if enabled.contains(SET_ARITY_MISMATCH.code) { - diagnostics.extend(diagnostics_for_spans( - &SET_ARITY_MISMATCH, - &facts.additional.set_arity_mismatches, - Some("the intended branch schema requires user intent"), - )); - } - if enabled.contains(PROJECTED_STAR.code) { - diagnostics.extend(diagnostics_for_spans( - &PROJECTED_STAR, - &facts.additional.projected_stars, - Some("column enumeration requires reliable resolved schema evidence"), - )); - } - if enabled.contains(UNALIASED_CALCULATION.code) { - diagnostics.extend(diagnostics_for_spans( - &UNALIASED_CALCULATION, - &facts.additional.unaliased_calculations, - Some("a meaningful stable output name requires user intent"), - )); - } - if enabled.contains(UNUSED_TABLE_ALIAS.code) { - diagnostics.extend(facts.additional.unused_table_aliases.iter().map(|span| { - diagnostic( - &UNUSED_TABLE_ALIAS, - Some(*span), - Some(deletion(*span)), - None, - ) - })); - } - if enabled.contains(NULL_REJECTED_LEFT_JOIN.code) { - diagnostics.extend(diagnostics_for_spans( - &NULL_REJECTED_LEFT_JOIN, - &facts.additional.null_rejected_left_joins, - Some("moving the predicate or changing join type requires user intent"), - )); - } - if enabled.contains(IMPLICIT_INNER_JOIN.code) { - diagnostics.extend(facts.additional.implicit_inner_joins.iter().map(|span| { - diagnostic( - &IMPLICIT_INNER_JOIN, - Some(*span), - Some(LintEdit { - start: span.start, - end: span.end, - replacement: "INNER JOIN".to_string(), - }), - None, - ) - })); - } - if enabled.contains(AMBIGUOUS_ORDER_DIRECTIONS.code) { - diagnostics.extend(diagnostics_for_spans( - &AMBIGUOUS_ORDER_DIRECTIONS, - &facts.additional.ambiguous_order_directions, - Some("normalizing every expression requires a multi-range authored rewrite"), - )); - } - if enabled.contains(UNQUALIFIED_MULTI_SOURCE_COLUMN.code) { - diagnostics.extend(diagnostics_for_spans( - &UNQUALIFIED_MULTI_SOURCE_COLUMN, - &facts.additional.unqualified_multi_source_columns, - Some("choosing the correct relation qualifier requires user intent"), - )); - } - if enabled.contains(INCONSISTENT_SINGLE_SOURCE_QUALIFICATION.code) { - diagnostics.extend(diagnostics_for_spans( - &INCONSISTENT_SINGLE_SOURCE_QUALIFICATION, - &facts.additional.inconsistent_single_source_qualification, - Some("qualification style is normalized by the canonical formatter only when safe"), - )); - } - if enabled.contains(UNKNOWN_RELATION_QUALIFIER.code) { - diagnostics.extend(diagnostics_for_spans( - &UNKNOWN_RELATION_QUALIFIER, - &facts.additional.unknown_relation_qualifiers, - Some("the intended relation or alias requires user intent"), - )); - } - if enabled.contains(SIMPLE_BOOLEAN_CASE.code) { - diagnostics.extend(facts.additional.simple_boolean_cases.iter().map(|span| { - diagnostic( - &SIMPLE_BOOLEAN_CASE, - Some(*span), - simple_boolean_case_fix(sql, tokens, *span), - Some("comments or a non-canonical CASE shape prevent a lossless rewrite"), - ) - })); - } - if enabled.contains(NESTED_ELSE_CASE.code) { - diagnostics.extend(diagnostics_for_spans( - &NESTED_ELSE_CASE, - &facts.additional.nested_else_cases, - Some("flattening nested branches requires a multi-range authored rewrite"), - )); - } - if enabled.contains(UNUSED_JOINED_RELATION.code) { - diagnostics.extend(diagnostics_for_spans( - &UNUSED_JOINED_RELATION, - &facts.additional.unused_joined_relations, - Some("the join may intentionally filter or multiply rows"), - )); - } - diagnostics -} - -fn rule_migration_diagnostics(context: &DiagnosticContext<'_>) -> Vec { - let DiagnosticContext { facts, enabled, .. } = context; - let mut diagnostics: Vec = Vec::new(); - if enabled.contains(COMMENT_ATTACHMENT.code) { - diagnostics.extend(diagnostics_for_spans( - &COMMENT_ATTACHMENT, - &facts.invalid_comment_attachments, - Some("comment attachment requires an authored placement decision"), - )); - } - if enabled.contains(CTE_ONLY_BODY.code) { - diagnostics.extend(diagnostics_for_spans( - &CTE_ONLY_BODY, - &facts.cte_only_bodies, - Some("extracting transformation logic into named CTEs requires user intent"), - )); - } - if enabled.contains(TERMINAL_SELECT.code) { - diagnostics.extend(diagnostics_for_spans( - &TERMINAL_SELECT, - &facts.terminal_selects, - Some("choosing the final logical CTE and projection requires user intent"), - )); - } - if enabled.contains(NESTED_CTE.code) { - diagnostics.extend(diagnostics_for_spans( - &NESTED_CTE, - &facts.nested_ctes, - Some("hoisting a nested CTE requires an authored query rewrite"), - )); - } - if enabled.contains(RECURSIVE_CTE.code) { - diagnostics.extend(diagnostics_for_spans( - &RECURSIVE_CTE, - &facts.recursive_ctes, - Some("replacing recursion requires an authored query design"), - )); - } - if enabled.contains(INLINE_QUERY_RELATION.code) { - diagnostics.extend(diagnostics_for_spans( - &INLINE_QUERY_RELATION, - &facts.inline_query_relations, - Some("extracting a derived relation can change correlation and scope"), - )); - } - diagnostics -} - -fn deletion(span: Span) -> LintEdit { - LintEdit { - start: span.start, - end: span.end, - replacement: String::new(), - } -} - -fn parenthesized_distinct_fix(sql: &str, span: Span) -> Option { - let source = char_slice(sql, span.start, span.end)?; - if contains_comment(source) { - return None; - } - let open = source.find('(')?; - let close = source.rfind(')')?; - (open < close).then(|| LintEdit { - start: span.start, - end: span.end, - replacement: format!("DISTINCT {}", &source[open + 1..close]), - }) -} - -fn simple_boolean_case_fix(sql: &str, tokens: &[Token], span: Span) -> Option { - let case_index = token_index_for_span(tokens, span).or_else(|| { - tokens - .iter() - .position(|token| token.span.start == span.start && token.token_type == TokenType::Case) - })?; - let end_index = tokens.iter().position(|token| token.span.end == span.end)?; - let source = char_slice(sql, span.start, span.end)?; - if contains_comment(source) { - return None; - } - let case_tokens = &tokens[case_index..=end_index]; - if case_tokens - .iter() - .filter(|token| token.token_type == TokenType::When) - .count() - != 1 - || case_tokens - .iter() - .filter(|token| token.token_type == TokenType::Then) - .count() - != 1 - || case_tokens - .iter() - .filter(|token| token.token_type == TokenType::Else) - .count() - != 1 - { - return None; - } - let when_index = - (case_index + 1..end_index).find(|&index| tokens[index].token_type == TokenType::When)?; - let then_index = - (when_index + 1..end_index).find(|&index| tokens[index].token_type == TokenType::Then)?; - let condition = char_slice( - sql, - tokens[when_index].span.end, - tokens[then_index].span.start, - )? - .trim(); - (!condition.is_empty()).then(|| LintEdit { - start: span.start, - end: span.end, - replacement: format!("COALESCE({condition}, FALSE)"), - }) -} - -fn explicit_union_fix(dialect: DialectType, span: Span) -> Option { - matches!( - dialect, - DialectType::Generic - | DialectType::PostgreSQL - | DialectType::MySQL - | DialectType::BigQuery - | DialectType::Snowflake - | DialectType::DuckDB - | DialectType::Hive - | DialectType::Spark - | DialectType::Trino - | DialectType::Presto - | DialectType::Redshift - | DialectType::ClickHouse - | DialectType::Databricks - | DialectType::Athena - ) - .then(|| LintEdit { - start: span.start, - end: span.end, - replacement: "UNION DISTINCT".to_string(), - }) -} - -fn char_slice(sql: &str, start: usize, end: usize) -> Option<&str> { - if start > end { - return None; - } - let byte_start = char_offset_to_byte_offset(sql, start)?; - let byte_end = char_offset_to_byte_offset(sql, end)?; - sql.get(byte_start..byte_end) -} - -fn char_offset_to_byte_offset(sql: &str, offset: usize) -> Option { - if offset == sql.chars().count() { - return Some(sql.len()); - } - sql.char_indices() - .nth(offset) - .map(|(byte_offset, _)| byte_offset) -} - -fn contains_comment(source: &str) -> bool { - source.contains("--") || source.contains("/*") -} - -fn diagnostics_for_spans( - rule: &LintRuleMetadata, - spans: &[Span], - fix_unavailable_reason: Option<&'static str>, -) -> Vec { - spans - .iter() - .map(|span| diagnostic(rule, Some(*span), None, fix_unavailable_reason)) - .collect() -} - -fn diagnostic( - rule: &LintRuleMetadata, - span: Option, - fix: Option, - fix_unavailable_reason: Option<&'static str>, -) -> LintDiagnostic { - let span = span.unwrap_or_default(); - LintDiagnostic { - code: rule.code, - message: Cow::Borrowed(rule.message), - remediation: Cow::Borrowed(rule.remediation), - start: span.start, - end: span.end, - fix_unavailable_reason: fix.is_none().then_some(fix_unavailable_reason).flatten(), - fix, - } -} - -fn unused_cte_spans(tokens: &[Token], unused_names: &[String]) -> Vec { - let mut remaining: Vec = unused_names - .iter() - .map(|name| name.to_ascii_lowercase()) - .collect(); - let mut spans: Vec = Vec::new(); - for (index, token) in tokens.iter().enumerate() { - let name = token.text.to_ascii_lowercase(); - let Some(position) = remaining.iter().position(|candidate| candidate == &name) else { - continue; - }; - if tokens - .get(index + 1) - .is_some_and(|next| next.token_type == TokenType::As) - { - remaining.remove(position); - spans.push(token.span); - } - } - spans -} - -fn null_comparison_spans(tokens: &[Token]) -> Vec { - let comparison_types = [ - TokenType::Eq, - TokenType::Neq, - TokenType::Lt, - TokenType::Lte, - TokenType::Gt, - TokenType::Gte, - ]; - let depths = token_depths(tokens); - tokens - .iter() - .enumerate() - .filter(|(_, token)| comparison_types.contains(&token.token_type)) - .filter(|(index, _)| !is_assignment_operator(tokens, &depths, *index)) - .filter(|(index, _)| { - adjacent_null(tokens, *index, false) || adjacent_null(tokens, *index, true) - }) - .map(|(_, token)| token.span) - .collect() -} - -fn is_assignment_operator(tokens: &[Token], depths: &[usize], operator_index: usize) -> bool { - let depth = depths[operator_index]; - for index in (0..operator_index).rev() { - if depths[index] != depth { - continue; - } - match tokens[index].token_type { - TokenType::Where | TokenType::Having | TokenType::Qualify | TokenType::On => { - return false; - } - TokenType::Set => return true, - TokenType::Select | TokenType::Semicolon => return false, - _ => {} - } - } - false -} - -fn adjacent_null(tokens: &[Token], operator_index: usize, forward: bool) -> bool { - let mut index = operator_index as isize + if forward { 1 } else { -1 }; - let mut wrappers = 0_usize; - loop { - if index < 0 { - return false; - } - let Some(token) = tokens.get(index as usize) else { - return false; - }; - if token.token_type == TokenType::Null { - break; - } - let wrapper = if forward { - TokenType::LParen - } else { - TokenType::RParen - }; - if token.token_type != wrapper { - return false; - } - wrappers += 1; - index += if forward { 1 } else { -1 }; - } - index += if forward { 1 } else { -1 }; - let closing_wrapper = if forward { - TokenType::RParen - } else { - TokenType::LParen - }; - for _ in 0..wrappers { - if index < 0 { - return false; - } - let Some(token) = tokens.get(index as usize) else { - return false; - }; - if token.token_type != closing_wrapper { - return false; - } - index += if forward { 1 } else { -1 }; - } - true -} diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/formatter.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/formatter.rs deleted file mode 100644 index 88ddd0a4e1..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/formatter.rs +++ /dev/null @@ -1,343 +0,0 @@ -use std::collections::{BTreeMap, HashMap}; -use std::str::FromStr; - -use polyglot_sql::tokens::{Token, TokenType}; -use polyglot_sql::{ComplexityGuardOptions, Dialect, DialectType, ParseOptions}; -use rayon::iter::{IntoParallelIterator, ParallelIterator}; - -use crate::lint_text::constants::LINT_SENTINEL_PREFIX; -use crate::sql_lint::_helpers::formatter_syntax::{cte_macro_indices, protect_syntax}; -use crate::sql_lint::_helpers::printed_layout::{PrintedSql, WrapOptions, wrap_lines}; -use crate::sql_lint::_helpers::token_layout::{ - AuthoredSql, KeywordRecase, OracleLayout, TokenTexts, comments_in, neutralize_comments, - oracle_layout, print_authored_tokens, tokenize, verify_printed_tokens, -}; -use crate::sql_lint::constants::{ - CAST_TYPE_SEPARATOR_KEYWORD, CLOSE_PARENTHESIS, LINT_API_VERSION, OPEN_PARENTHESIS, -}; -use crate::sql_lint::models::{FormatRequest, FormatResponse}; - -#[cfg(test)] -thread_local! { - /// Idempotency re-prints on this thread, for tests that assert which bodies are re-printed. - pub(crate) static RELAYOUTS: std::cell::Cell = const { std::cell::Cell::new(0) }; -} - -const NOT_IDEMPOTENT_FAILURE: &str = "native formatter output is not idempotent"; -const UNSUPPORTED_SQL_FAILURE: &str = - "native formatter could not safely restore literal or cast spelling"; - -pub(crate) fn format_json_impl(request_json: &str) -> Result { - let request: FormatRequest = - serde_json::from_str(request_json).map_err(|error| error.to_string())?; - let response = format_sql(request)?; - serde_json::to_string(&response).map_err(|error| error.to_string()) -} - -/// Format requests in parallel, in order; a request that fails is declined with its reason. -pub(crate) fn format_requests(requests: Vec) -> Vec { - requests - .into_par_iter() - .map(|request| { - let original: String = request.sql.clone(); - format_sql(request).unwrap_or_else(|reason| FormatResponse { - version: LINT_API_VERSION, - sql: original, - changed: false, - formatted: false, - reason: Some(reason), - }) - }) - .collect() -} - -fn format_sql(request: FormatRequest) -> Result { - let original = request.sql.clone(); - match try_format_sql(request) { - Ok(value) => Ok(value), - Err(reason) => Ok(FormatResponse { - version: LINT_API_VERSION, - sql: original, - changed: false, - formatted: false, - reason: Some(reason), - }), - } -} - -fn try_format_sql(request: FormatRequest) -> Result { - if request.version != LINT_API_VERSION { - return Err(format!( - "unsupported native format request version {}; expected {LINT_API_VERSION}", - request.version - )); - } - let original = request.sql; - let dialect_type = - DialectType::from_str(&request.dialect).map_err(|error| error.to_string())?; - let dialect = Dialect::get(dialect_type); - let parse_options = ParseOptions { - complexity_guard: Some(ComplexityGuardOptions { - max_function_call_depth: request.max_function_call_depth, - ..Default::default() - }), - }; - let wrap = request.line_width.map(|line_width| WrapOptions { - line_width, - token_widths: &request.token_widths, - }); - let layout = Layout { - dialect: &dialect, - wrap: wrap.as_ref(), - }; - let (formatted, formatted_tokens, oracle) = format_layout(&original, &layout, &parse_options)?; - let changed = formatted != original; - if changed && relayout((&formatted, &formatted_tokens), oracle.as_ref(), &layout)? != formatted - { - return Err(NOT_IDEMPOTENT_FAILURE.to_string()); - } - Ok(FormatResponse { - version: LINT_API_VERSION, - changed, - sql: formatted, - formatted: true, - reason: None, - }) -} - -/// The printed output of [`format_layout`], its tokens, and the oracle it was printed against. -type FormattedLayout = (String, Vec, Option); - -/// Print the tokens of `sql` with parse-tree whitespace; return the output and its oracle. -fn format_layout( - sql: &str, - layout: &Layout<'_>, - parse_options: &ParseOptions, -) -> Result { - let dialect = layout.dialect; - let tokens = dialect - .tokenize(sql) - .map_err(|error| format!("native formatter tokenization failed: {error}"))?; - if tokens.is_empty() { - return Ok((sql.to_string(), tokens, None)); - } - let comments = comments_in(sql, &tokens); - let neutral = neutralize_comments(sql, &comments); - let validation_sql = protect_syntax(&neutral, &tokens, &[], false)?; - dialect - .parse_with_options(&validation_sql.sql, parse_options) - .map_err(|error| format!("native formatter could not parse SQL: {error}"))?; - let semantic_tokens = without_statement_terminators(&tokens); - let oracle = oracle_layout( - &layout_oracle(&neutral, &semantic_tokens, dialect, parse_options)?, - dialect, - )?; - let authored = AuthoredSql { - sql, - tokens: &tokens, - comments: &comments, - }; - let (printed, recases, authored_texts) = layout.print(&authored, &oracle)?; - if printed == sql { - return Ok((printed, tokens, Some(oracle))); - } - let printed_tokens = tokenize(&printed, dialect)?; - verify_printed_tokens( - (sql, &tokens, &authored_texts), - (&printed, &printed_tokens), - dialect, - &recases, - )?; - Ok((printed, printed_tokens, Some(oracle))) -} - -/// Re-print formatted SQL against its oracle; unchanged tokens imply an unchanged layout. -fn relayout( - (formatted, tokens): (&str, &[Token]), - oracle: Option<&OracleLayout>, - layout: &Layout<'_>, -) -> Result { - #[cfg(test)] - RELAYOUTS.with(|relayouts| relayouts.set(relayouts.get() + 1)); - let Some(oracle) = oracle.filter(|_| !tokens.is_empty()) else { - return Ok(formatted.to_string()); - }; - let comments = comments_in(formatted, tokens); - let authored = AuthoredSql { - sql: formatted, - tokens, - comments: &comments, - }; - Ok(layout.print(&authored, oracle)?.0) -} - -/// The dialect and optional line wrapping every printed layout uses. -struct Layout<'a> { - dialect: &'a Dialect, - wrap: Option<&'a WrapOptions<'a>>, -} - -impl Layout<'_> { - fn print( - &self, - authored: &AuthoredSql<'_>, - oracle: &OracleLayout, - ) -> Result<(String, BTreeMap, TokenTexts), String> { - let (printed, recases, texts) = print_authored_tokens(authored, oracle, self.dialect)?; - let no_widths: HashMap = HashMap::new(); - let token_widths = self.wrap.map_or(&no_widths, |wrap| wrap.token_widths); - let Some(mut sql) = PrintedSql::parse(&printed, self.dialect, token_widths)? else { - return Ok((printed, recases, texts)); - }; - sql.arrange_clauses(self.wrap.map_or(usize::MAX, |wrap| wrap.line_width)); - if let Some(wrap) = self.wrap { - wrap_lines(&mut sql, wrap.line_width); - } - sql.lead_operators_of_multiline_operands(); - Ok((sql.render(), recases, texts)) - } -} - -/// Return the parse-tree layout of `neutral_sql`; only its whitespace is ever printed. -fn layout_oracle( - neutral_sql: &str, - semantic_tokens: &[Token], - dialect: &Dialect, - parse_options: &ParseOptions, -) -> Result { - let protected = protect_syntax( - neutral_sql, - semantic_tokens, - &cast_type_token_ranges(neutral_sql, semantic_tokens)?, - true, - )?; - let expressions = dialect - .parse_with_options(&protected.sql, parse_options) - .map_err(|error| error.to_string())?; - let statements: Result, String> = expressions - .iter() - .map(|expression| { - dialect - .generate_pretty(expression) - .map_err(|error| error.to_string()) - }) - .collect(); - let restored = protected.restore(statements?.join(";\n"), dialect)?; - layout_cte_macros(restored, dialect) -} - -fn layout_cte_macros(mut formatted: String, dialect: &Dialect) -> Result { - if !formatted.contains(LINT_SENTINEL_PREFIX) { - return Ok(formatted); - } - let tokens = dialect - .tokenize(&formatted) - .map_err(|error| error.to_string())?; - let macros = cte_macro_indices(&tokens); - if macros.is_empty() { - return Ok(formatted); - } - let depths = crate::sql_lint::_helpers::engine::token_depths(&tokens); - let chars: Vec = formatted.chars().collect(); - let mut gaps: BTreeMap<(usize, usize), String> = BTreeMap::new(); - for index in macros { - let with = (0..index) - .rev() - .find(|&previous| { - tokens[previous].token_type == TokenType::With && depths[previous] == depths[index] - }) - .ok_or_else(|| "native formatter lost a CTE macro scope".to_string())?; - let indentation = line_indentation(&chars, tokens[with].span.start); - let newline = format!("\n{}", " ".repeat(indentation)); - gaps.insert( - (tokens[index - 1].span.end, tokens[index].span.start), - newline.clone(), - ); - if tokens - .get(index + 1) - .is_some_and(|token| token.token_type == TokenType::Comma) - && let Some(next) = tokens.get(index + 2) - { - gaps.insert((tokens[index + 1].span.end, next.span.start), newline); - } - } - for ((start, end), replacement) in gaps.into_iter().rev() { - let start = char_to_byte(&formatted, start)?; - let end = char_to_byte(&formatted, end)?; - formatted.replace_range(start..end, &replacement); - } - Ok(formatted) -} - -fn line_indentation(characters: &[char], offset: usize) -> usize { - let line_start = characters[..offset] - .iter() - .rposition(|character| *character == '\n') - .map_or(0, |position| position + 1); - characters[line_start..offset] - .iter() - .take_while(|character| **character == ' ' || **character == '\t') - .count() -} - -fn cast_type_token_ranges(sql: &str, tokens: &[Token]) -> Result, String> { - let characters: Vec = sql.chars().collect(); - let mut ranges: Vec<(usize, usize)> = Vec::new(); - let mut cast_type_starts: Vec> = Vec::new(); - let mut previous: Option = None; - for (index, token) in tokens.iter().enumerate() { - let text: String = characters - .get(token.span.start..token.span.end) - .ok_or_else(|| UNSUPPORTED_SQL_FAILURE.to_string())? - .iter() - .collect(); - let normalized = text.to_ascii_uppercase(); - if normalized == OPEN_PARENTHESIS { - let opens_cast = previous - .as_deref() - .is_some_and(|value| matches!(value, "CAST" | "TRY_CAST")); - cast_type_starts.push(opens_cast.then_some(usize::MAX)); - } else if normalized == CLOSE_PARENTHESIS { - if let Some(Some(type_start)) = cast_type_starts.pop() - && type_start != usize::MAX - && type_start < index - { - ranges.push((type_start, index - 1)); - } - } else if normalized == CAST_TYPE_SEPARATOR_KEYWORD - && let Some(Some(type_start)) = cast_type_starts.last_mut() - && *type_start == usize::MAX - { - *type_start = index + 1; - } - previous = Some(normalized); - } - ranges.sort_unstable(); - Ok(ranges) -} - -fn without_statement_terminators(tokens: &[Token]) -> Vec { - tokens - .iter() - .enumerate() - .filter(|(index, token)| { - token.token_type != TokenType::Semicolon - && !(token.token_type == TokenType::Comma - && tokens - .get(index + 1) - .is_some_and(|next| next.token_type == TokenType::From)) - }) - .map(|(_, token)| token.clone()) - .collect() -} - -fn char_to_byte(value: &str, char_offset: usize) -> Result { - if char_offset == value.chars().count() { - return Ok(value.len()); - } - value - .char_indices() - .nth(char_offset) - .map(|(offset, _)| offset) - .ok_or_else(|| "native formatter produced an invalid character offset".to_string()) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/formatter_syntax.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/formatter_syntax.rs deleted file mode 100644 index f99ce7bcaa..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/formatter_syntax.rs +++ /dev/null @@ -1,459 +0,0 @@ -use crate::lint_text::constants::LINT_SENTINEL_PREFIX; -use crate::sql_lint::_helpers::engine::token_depths; -use crate::sql_lint::constants::{ - CTE_MACRO_PLACEHOLDER_LITERAL, POSITION_ARGUMENT_SEPARATOR, POSTFIX_CAST_PREFIX_TOKEN_COUNT, -}; -use polyglot_sql::Dialect; -use polyglot_sql::tokens::{Token, TokenType}; -use std::collections::{HashMap, HashSet}; - -pub(super) struct ProtectedSql { - pub sql: String, - /// The lower-case prefix every sentinel name starts with. - prefix: String, - fragments: Vec<(String, String)>, - in_functions: HashSet, -} - -impl ProtectedSql { - pub(super) fn restore(&self, formatted: String, dialect: &Dialect) -> Result { - let tokens = dialect - .tokenize(&formatted) - .map_err(|error| error.to_string())?; - let mut positions: HashMap> = HashMap::new(); - for (index, token) in tokens.iter().enumerate() { - let prefixed = token - .text - .as_bytes() - .get(..self.prefix.len()) - .is_some_and(|start| start.eq_ignore_ascii_case(self.prefix.as_bytes())); - if !prefixed { - continue; - } - positions - .entry(token.text.to_ascii_lowercase()) - .or_default() - .push(index); - } - let mut edits: Vec<(usize, usize, String)> = Vec::new(); - for (placeholder, original) in &self.fragments { - let key = cte_placeholder_sentinel(placeholder) - .unwrap_or(placeholder) - .trim_start_matches(':') - .trim_start_matches('.') - .trim_matches('\'') - .to_ascii_lowercase(); - let found = positions - .get(&key) - .filter(|found| found.len() == 1) - .ok_or_else(|| { - "native formatter could not restore a protected SQL fragment".to_string() - })?; - let index = found[0]; - if self.in_functions.contains(&key) { - let mut depth = 0; - let separator = (index + 1..tokens.len()) - .find(|&position| { - match tokens[position].token_type { - TokenType::LParen => depth += 1, - TokenType::RParen => depth -= 1, - _ => {} - } - depth == 1 && tokens[position].token_type == TokenType::Comma - }) - .ok_or_else(|| "native formatter lost a POSITION separator".to_string())?; - edits.push(( - tokens[separator].span.start, - tokens[separator].span.end, - " IN".to_string(), - )); - } - if let Some(sentinel) = cte_placeholder_sentinel(placeholder) { - let _ = sentinel; - edits.push(restore_cte_macro(&tokens, index, original)?); - continue; - } - if placeholder.starts_with("::") { - edits.extend(restore_postfix_cast(&tokens, index, original)?); - continue; - } - let start = if placeholder.starts_with('.') { - if index == 0 || tokens[index - 1].token_type != TokenType::Dot { - return Err("native formatter changed a protected path accessor".to_string()); - } - tokens[index - 1].span.start - } else { - tokens[index].span.start - }; - edits.push((start, tokens[index].span.end, original.clone())); - } - edits.sort_by_key(|edit| edit.0); - let chars: Vec = formatted.chars().collect(); - let mut result = String::new(); - let mut cursor = 0; - for (start, end, replacement) in edits { - if start < cursor { - return Err("native formatter produced overlapping syntax restorations".to_string()); - } - result.extend(&chars[cursor..start]); - result.push_str(&replacement); - cursor = end; - } - result.extend(&chars[cursor..]); - Ok(result) - } -} - -/// The sentinel of a CTE macro placeholder, with or without the comma added after it. -fn cte_placeholder_sentinel(placeholder: &str) -> Option<&str> { - let placeholder: &str = placeholder.strip_suffix(',').unwrap_or(placeholder); - placeholder.strip_suffix(" AS (SELECT 1)") -} - -pub(super) fn protect_syntax( - sql: &str, - tokens: &[Token], - cast_types: &[(usize, usize)], - expressions: bool, -) -> Result { - let mut prefix = "__sqlbuild_format_".to_string(); - let lowercase: String = sql.to_ascii_lowercase(); - while lowercase.contains(&prefix) { - prefix.push('_'); - } - let mut cast_type_ends: HashMap = HashMap::new(); - for (start, end) in cast_types { - let _ = cast_type_ends.entry(*start).or_insert(*end); - } - let mut ranges: Vec<(usize, usize, String)> = Vec::new(); - let cte_macros = cte_macro_indices(tokens); - let depths = token_depths(tokens); - let mut separators: HashSet = HashSet::new(); - let mut in_functions: HashSet = HashSet::new(); - let mut index = 0; - while index < tokens.len() { - let token = &tokens[index]; - let sentinel = || format!("{prefix}{}__", ranges.len()); - if separators.contains(&index) { - ranges.push((token.span.start, token.span.end, ",".to_string())); - index += 1; - continue; - } - let (end, replacement) = if cte_macros.contains(&index) { - let separator: &str = if cte_macro_needs_comma(tokens, index) { - "," - } else { - "" - }; - (index, format!("{} AS (SELECT 1){separator}", sentinel())) - } else if !expressions { - index += 1; - continue; - } else if let Some(end) = cast_type_ends.get(&index) { - (*end, sentinel()) - } else if token.token_type == TokenType::DColon { - ( - postfix_type_end(tokens, index + 1)?, - format!("::{}", sentinel()), - ) - } else if token.token_type == TokenType::Colon - && index > 0 - && tokens[index - 1].token_type != TokenType::String - && tokens.get(index + 1).is_some_and(|next| { - next.text - .chars() - .next() - .is_some_and(|character| character.is_alphanumeric() || character == '_') - || matches!( - next.token_type, - TokenType::QuotedIdentifier | TokenType::String - ) - }) - { - (index + 1, format!(".{}", sentinel())) - } else if is_typed_literal(tokens, index) - || (matches!(token.token_type, TokenType::Var | TokenType::Identifier) - && tokens.get(index + 1).is_some_and(|next| { - matches!(next.token_type, TokenType::Var | TokenType::Identifier) - }) - && tokens - .get(index + 2) - .is_some_and(|next| next.token_type == TokenType::Arrow)) - { - (index + 1, sentinel()) - } else if (token.text.eq_ignore_ascii_case("MOD") - || token.text.eq_ignore_ascii_case("POSITION")) - && tokens - .get(index + 1) - .is_some_and(|next| next.token_type == TokenType::LParen) - { - let sentinel: String = sentinel(); - if token.text.eq_ignore_ascii_case("POSITION") { - let separator = (index + 2..tokens.len()) - .take_while(|&position| { - !(tokens[position].token_type == TokenType::RParen - && depths[position] == depths[index] + 1) - }) - .find(|&position| { - tokens[position].token_type == TokenType::In - && depths[position] == depths[index] + 1 - }); - if let Some(separator) = separator { - separators.insert(separator); - in_functions.insert(sentinel.clone()); - } - } - (index, sentinel) - } else if matches!( - token.token_type, - TokenType::String | TokenType::DollarString - ) { - (index, format!("'{}'", sentinel())) - } else { - index += 1; - continue; - }; - ranges.push((token.span.start, tokens[end].span.end, replacement)); - index = end + 1; - } - let chars: Vec = sql.chars().collect(); - let token_indices: HashMap = tokens - .iter() - .enumerate() - .map(|(index, token)| (token.span.start, index)) - .collect(); - let mut prepared = String::new(); - let mut fragments: Vec<(String, String)> = Vec::new(); - let mut cursor = 0; - for (start, end, replacement) in ranges { - prepared.extend(&chars[cursor..start]); - prepared.push_str(&replacement); - if replacement != POSITION_ARGUMENT_SEPARATOR { - let first = token_indices[&start]; - let last = (first..tokens.len()) - .find(|&index| tokens[index].span.end == end) - .ok_or_else(|| "native formatter lost a source fragment boundary".to_string())?; - fragments.push(( - replacement, - canonical_fragment(&chars, &tokens[first..=last]), - )); - } - cursor = end; - } - prepared.extend(&chars[cursor..]); - Ok(ProtectedSql { - sql: prepared, - prefix, - fragments, - in_functions, - }) -} - -fn canonical_fragment(chars: &[char], tokens: &[Token]) -> String { - let mut result = String::new(); - for (index, token) in tokens.iter().enumerate() { - if index > 0 - && !matches!( - token.token_type, - TokenType::LParen - | TokenType::RParen - | TokenType::Comma - | TokenType::Dot - | TokenType::Colon - | TokenType::DColon - | TokenType::LBracket - | TokenType::RBracket - | TokenType::Lt - | TokenType::Gt - ) - && !matches!( - tokens[index - 1].token_type, - TokenType::LParen - | TokenType::Dot - | TokenType::Colon - | TokenType::DColon - | TokenType::LBracket - | TokenType::Lt - ) - { - result.push(' '); - } - result.extend(&chars[token.span.start..token.span.end]); - } - result -} - -pub(super) fn cte_macro_indices(tokens: &[Token]) -> HashSet { - let depths = token_depths(tokens); - let mut scopes: HashSet = HashSet::new(); - let mut indices: HashSet = HashSet::new(); - for (index, token) in tokens.iter().enumerate() { - if token.token_type == TokenType::With { - scopes.insert(depths[index]); - } else if token.token_type == TokenType::Select { - scopes.remove(&depths[index]); - } else if token.text.starts_with(LINT_SENTINEL_PREFIX) - && scopes.contains(&depths[index]) - && index > 0 - && (matches!( - tokens[index - 1].token_type, - TokenType::With | TokenType::Comma - ) || indices.contains(&(index - 1))) - && tokens.get(index + 1).is_some_and(|next| { - matches!(next.token_type, TokenType::Comma | TokenType::Select) - || starts_cte(tokens, index + 1) - }) - { - indices.insert(index); - } - } - indices -} - -/// Whether a CTE definition, `name AS (` or another macro sentinel, starts at token `index`. -fn starts_cte(tokens: &[Token], index: usize) -> bool { - let Some(token) = tokens.get(index) else { - return false; - }; - if token.text.starts_with(LINT_SENTINEL_PREFIX) { - return true; - } - let named: bool = matches!( - token.token_type, - TokenType::Var | TokenType::Identifier | TokenType::QuotedIdentifier - ) || token - .text - .chars() - .all(|character| character.is_alphanumeric() || character == '_'); - named - && tokens - .get(index + 1) - .is_some_and(|next| next.token_type == TokenType::As) - && tokens - .get(index + 2) - .is_some_and(|open| open.token_type == TokenType::LParen) -} - -/// Whether the CTE macro sentinel at token `index` needs a comma to separate it from the next CTE. -pub(super) fn cte_macro_needs_comma(tokens: &[Token], index: usize) -> bool { - starts_cte(tokens, index + 1) -} - -fn restore_cte_macro( - tokens: &[Token], - start: usize, - original: &str, -) -> Result<(usize, usize, String), String> { - let end = start + 5; - if tokens - .get(end) - .is_none_or(|token| token.token_type != TokenType::RParen) - || tokens[start + 1].token_type != TokenType::As - || tokens[start + 2].token_type != TokenType::LParen - || tokens[start + 3].token_type != TokenType::Select - || tokens[start + 4].text != CTE_MACRO_PLACEHOLDER_LITERAL - { - return Err("native formatter changed a CTE macro placeholder".to_string()); - } - Ok(( - tokens[start].span.start, - tokens[end].span.end, - original.to_string(), - )) -} - -fn restore_postfix_cast( - tokens: &[Token], - index: usize, - original: &str, -) -> Result, String> { - if index < POSTFIX_CAST_PREFIX_TOKEN_COUNT - || tokens[index - 1].token_type != TokenType::As - || tokens - .get(index + 1) - .is_none_or(|token| token.token_type != TokenType::RParen) - { - return Err("native formatter changed a postfix cast wrapper".to_string()); - } - let mut depth = 0; - let mut open = None; - for position in (0..index - 1).rev() { - match tokens[position].token_type { - TokenType::RParen => depth += 1, - TokenType::LParen if depth == 0 => { - open = Some(position); - break; - } - TokenType::LParen => depth -= 1, - _ => {} - } - } - let open = open - .filter(|&position| position > 0 && tokens[position - 1].token_type == TokenType::Cast) - .ok_or_else(|| "native formatter could not identify a postfix cast wrapper".to_string())?; - Ok(vec![ - ( - tokens[open - 1].span.start, - tokens[open + 1].span.start, - String::new(), - ), - ( - tokens[index - 2].span.end, - tokens[index + 1].span.end, - original.to_string(), - ), - ]) -} - -fn is_typed_literal(tokens: &[Token], index: usize) -> bool { - matches!( - tokens[index].text.to_ascii_uppercase().as_str(), - "DATE" - | "TIME" - | "TIMESTAMP" - | "TIMESTAMPTZ" - | "TIMESTAMP_NTZ" - | "TIMESTAMP_LTZ" - | "TIMESTAMP_TZ" - ) && tokens.get(index + 1).is_some_and(|token| { - matches!( - token.token_type, - TokenType::String | TokenType::DollarString - ) - }) -} - -fn postfix_type_end(tokens: &[Token], start: usize) -> Result { - let Some(_) = tokens.get(start) else { - return Err("native formatter could not locate a postfix cast type".to_string()); - }; - let mut end = start; - while tokens.get(end + 1).is_some_and(|token| { - matches!( - token.text.to_ascii_uppercase().as_str(), - "PRECISION" | "VARYING" | "WITH" | "WITHOUT" | "TIME" | "ZONE" - ) - }) { - end += 1; - } - if tokens - .get(end + 1) - .is_some_and(|token| token.token_type == TokenType::LParen) - { - let mut depth = 0; - for (index, token) in tokens.iter().enumerate().skip(end + 1) { - match token.token_type { - TokenType::LParen => depth += 1, - TokenType::RParen => { - depth -= 1; - if depth == 0 { - return Ok(index); - } - } - _ => {} - } - } - return Err("native formatter could not locate postfix cast parameters".to_string()); - } - Ok(end) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/inline_relations.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/inline_relations.rs deleted file mode 100644 index 7660a13eee..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/inline_relations.rs +++ /dev/null @@ -1,223 +0,0 @@ -use polyglot_sql::tokens::{Span, Token, TokenType}; - -use crate::sql_lint::_helpers::engine::is_query_from; - -struct InlineRelationContext<'a> { - tokens: &'a [Token], - depths: &'a [usize], - scopes: Vec>, - significant: &'a [usize], -} - -pub(super) fn collect_inline_query_relation_spans( - tokens: &[Token], - depths: &[usize], - significant: &[usize], -) -> Vec { - let context = InlineRelationContext { - tokens, - depths, - scopes: token_scopes(tokens), - significant, - }; - (0..significant.len()) - .filter_map(|position| inline_query_relation_span(&context, position)) - .collect() -} - -fn inline_query_relation_span( - context: &InlineRelationContext<'_>, - position: usize, -) -> Option { - let &opening = context.significant.get(position)?; - if context.tokens[opening].token_type != TokenType::LParen { - return None; - } - let marker_position = position.checked_sub(1)?; - let previous = context.significant[marker_position]; - if context.tokens[previous].token_type == TokenType::Lateral - || context.tokens[previous].text.eq_ignore_ascii_case("apply") - || !is_query_relation_marker(context, marker_position) - { - return None; - } - - let mut candidate_position = position + 1; - let mut candidate_depth = context.depths[opening] + 1; - loop { - let &candidate = context.significant.get(candidate_position)?; - if context.depths[candidate] != candidate_depth { - return None; - } - if context.tokens[candidate].token_type == TokenType::LParen { - candidate_position += 1; - candidate_depth += 1; - continue; - } - return is_query_start(&context.tokens[candidate]) - .then_some(context.tokens[candidate].span); - } -} - -fn is_query_relation_marker(context: &InlineRelationContext<'_>, marker_position: usize) -> bool { - is_query_relation_marker_base(context, marker_position) - && !query_or_ancestor_is_ignored(context, marker_position) -} - -fn is_query_relation_marker_base( - context: &InlineRelationContext<'_>, - marker_position: usize, -) -> bool { - let marker = context.significant[marker_position]; - if !(is_query_from(context.tokens, marker) - || matches!( - context.tokens[marker].token_type, - TokenType::Join | TokenType::Comma - )) - { - return false; - } - let scope = context.scopes[marker]; - let Some(select_position) = (0..marker_position).rev().find(|&position| { - let index = context.significant[position]; - context.scopes[index] == scope && context.tokens[index].token_type == TokenType::Select - }) else { - return false; - }; - let query_clause = &context.significant[select_position + 1..marker_position]; - if query_clause.iter().any(|&index| { - context.scopes[index] == scope - && matches!( - context.tokens[index].token_type, - TokenType::Semicolon | TokenType::Union | TokenType::Intersect | TokenType::Except - ) - }) { - return false; - } - if is_query_from(context.tokens, marker) { - return true; - } - let Some(from_position) = query_clause - .iter() - .position(|&index| context.scopes[index] == scope && is_query_from(context.tokens, index)) - else { - return false; - }; - !query_clause[from_position + 1..].iter().any(|&index| { - context.scopes[index] == scope && is_post_relation_clause(context.tokens[index].token_type) - }) -} - -fn query_or_ancestor_is_ignored( - context: &InlineRelationContext<'_>, - marker_position: usize, -) -> bool { - let marker = context.significant[marker_position]; - let mut scope = context.scopes[marker]; - while let Some(opening) = scope { - if scope_starts_query(context, opening) && query_scope_is_ignored(context, opening) { - return true; - } - scope = context.scopes[opening]; - } - false -} - -fn scope_starts_query(context: &InlineRelationContext<'_>, opening: usize) -> bool { - let Ok(mut position) = context.significant.binary_search(&opening) else { - return false; - }; - position += 1; - let mut expected_depth = context.depths[opening] + 1; - loop { - let Some(&candidate) = context.significant.get(position) else { - return false; - }; - if context.depths[candidate] != expected_depth { - return false; - } - if context.tokens[candidate].token_type == TokenType::LParen { - position += 1; - expected_depth += 1; - continue; - } - return is_query_start(&context.tokens[candidate]); - } -} - -fn query_scope_is_ignored(context: &InlineRelationContext<'_>, opening: usize) -> bool { - let Ok(mut opening_position) = context.significant.binary_search(&opening) else { - return false; - }; - while opening_position > 0 - && context.tokens[context.significant[opening_position - 1]].token_type == TokenType::LParen - { - opening_position -= 1; - } - let Some(marker_position) = opening_position.checked_sub(1) else { - return false; - }; - let marker = context.significant[marker_position]; - match context.tokens[marker].token_type { - TokenType::Exists | TokenType::In => true, - TokenType::From | TokenType::Join | TokenType::As => false, - TokenType::Comma => !is_query_relation_marker_base(context, marker_position), - TokenType::Union - | TokenType::Intersect - | TokenType::Except - | TokenType::All - | TokenType::Distinct - | TokenType::Semicolon - | TokenType::Lateral => false, - _ if context.tokens[marker].text.eq_ignore_ascii_case("apply") - || context.tokens[marker] - .text - .eq_ignore_ascii_case("materialized") => - { - false - } - _ => true, - } -} - -fn is_query_start(token: &Token) -> bool { - matches!(token.token_type, TokenType::Select | TokenType::Values) - || token.text.eq_ignore_ascii_case("with") -} - -fn is_post_relation_clause(token_type: TokenType) -> bool { - matches!( - token_type, - TokenType::Where - | TokenType::Group - | TokenType::Having - | TokenType::Qualify - | TokenType::Order - | TokenType::Limit - | TokenType::Offset - | TokenType::Fetch - ) -} - -fn token_scopes(tokens: &[Token]) -> Vec> { - let mut openings: Vec = Vec::new(); - tokens - .iter() - .enumerate() - .map(|(index, token)| { - let scope = openings.last().copied(); - if matches!( - token.token_type, - TokenType::LParen | TokenType::LBracket | TokenType::LBrace - ) { - openings.push(index); - } else if matches!( - token.token_type, - TokenType::RParen | TokenType::RBracket | TokenType::RBrace - ) { - let _ = openings.pop(); - } - scope - }) - .collect() -} diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/join_predicates.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/join_predicates.rs deleted file mode 100644 index bc841f66c0..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/join_predicates.rs +++ /dev/null @@ -1,694 +0,0 @@ -//! Join predicates, and whether an explicit cross join can multiply rows. - -use std::borrow::Cow; -use std::collections::{HashMap, HashSet}; - -use polyglot_sql::expressions::{Expression, JoinKind, Literal, Select, Subquery, TableRef, With}; -use polyglot_sql::parser::{Parser, ParserConfig}; -use polyglot_sql::tokens::{Span, Token, TokenType}; -use polyglot_sql::{DialectType, ExpressionWalk}; - -use crate::constants::SETTING_SNIPPET_INDENT; -use crate::sql_lint::constants::{ - CROSS_JOIN_CTE_MARKER, CROSS_JOIN_PATH_PLACEHOLDER, CROSS_JOIN_PREFIX, - CROSS_JOIN_REASON_PLACEHOLDER, CROSS_JOIN_SUFFIX, CROSS_JOIN_UNNAMED_MESSAGE, -}; -use crate::sql_lint::models::{CrossJoinRequest, LintDiagnostic, LintRuleMetadata}; - -const CROSS_JOIN_FIX_REASON: &str = "a keyed relationship requires user intent"; -const FIRST_ROW: &str = "1"; -const ROW_GENERATOR_FUNCTIONS: [&str; 16] = [ - "unnest", - "explode", - "explode_outer", - "posexplode", - "posexplode_outer", - "inline", - "inline_outer", - "flatten", - "generate_series", - "generate_subscripts", - "json_each", - "json_tree", - "json_array_elements", - "jsonb_array_elements", - "range", - "stack", -]; - -pub(super) fn join_expression_spans( - tokens: &[Token], - depths: &[usize], - dialect: DialectType, -) -> Vec { - let mut spans: Vec = Vec::new(); - for (index, token) in tokens.iter().enumerate() { - if token.token_type != TokenType::On { - continue; - } - let clause = (0..index).rev().find(|&previous| { - depths[previous] == depths[index] - && matches!( - tokens[previous].token_type, - TokenType::Join - | TokenType::Select - | TokenType::Where - | TokenType::From - | TokenType::On - ) - }); - if clause.is_none_or(|previous| tokens[previous].token_type != TokenType::Join) { - continue; - } - let end = (index + 1..tokens.len()) - .find(|&next| { - depths[next] < depths[index] - || (depths[next] == depths[index] - && matches!( - tokens[next].token_type, - TokenType::Join - | TokenType::Where - | TokenType::Group - | TokenType::Having - | TokenType::Qualify - | TokenType::Order - | TokenType::Limit - | TokenType::Union - | TokenType::Intersect - | TokenType::Except - | TokenType::Semicolon - )) - }) - .unwrap_or(tokens.len()); - let mut parser = Parser::with_config( - tokens[index + 1..end].to_vec(), - ParserConfig { - dialect: Some(dialect), - ..Default::default() - }, - ); - if !matches!(parser.parse_expressions(), Ok(Some(ref predicate)) if plain_predicate(predicate)) - { - spans.push(token.span); - } - } - spans -} - -fn plain_column(expression: &Expression) -> bool { - matches!( - expression, - Expression::Column(_) | Expression::Identifier(_) - ) || matches!(expression, Expression::Paren(value) if plain_column(&value.this)) -} - -fn literal(expression: &Expression) -> bool { - matches!( - expression, - Expression::Literal(_) | Expression::Boolean(_) | Expression::Null(_) - ) || matches!(expression, Expression::Neg(value) if matches!(&value.this, Expression::Literal(_))) -} - -fn operand(expression: &Expression) -> bool { - plain_column(expression) || literal(expression) -} - -fn comparison(left: &Expression, right: &Expression) -> bool { - operand(left) && operand(right) -} - -fn plain_predicate(expression: &Expression) -> bool { - match expression { - Expression::Boolean(_) => true, - Expression::Paren(value) => plain_predicate(&value.this), - Expression::And(value) | Expression::Or(value) => { - plain_predicate(&value.left) && plain_predicate(&value.right) - } - Expression::Eq(value) - | Expression::Neq(value) - | Expression::Lt(value) - | Expression::Lte(value) - | Expression::Gt(value) - | Expression::Gte(value) - | Expression::NullSafeEq(value) - | Expression::NullSafeNeq(value) - | Expression::Adjacent(value) - | Expression::ArrayContainsAll(value) - | Expression::ArrayContainedBy(value) - | Expression::ArrayOverlaps(value) - | Expression::ExtendsLeft(value) - | Expression::ExtendsRight(value) - | Expression::Match(value) - | Expression::TsMatch(value) => comparison(&value.left, &value.right), - Expression::Like(value) | Expression::ILike(value) => { - comparison(&value.left, &value.right) - && value.escape.as_ref().is_none_or(literal) - && value.quantifier.is_none() - } - Expression::Between(value) => { - plain_column(&value.this) && operand(&value.low) && operand(&value.high) - } - Expression::In(value) => { - plain_column(&value.this) - && value.query.is_none() - && value.unnest.is_none() - && !value.is_field - && !value.expressions.is_empty() - && value.expressions.iter().all(literal) - } - Expression::IsNull(value) => plain_column(&value.this), - _ => false, - } -} - -/// A `rule` diagnostic for every cross join that can multiply rows, naming its relation and CTE. -pub(super) fn cross_join_diagnostics( - request: &CrossJoinRequest<'_>, - rule: &LintRuleMetadata, -) -> Vec { - let one_row_relations: HashSet = request - .relation_keys - .iter() - .filter(|(_, keys)| keys.iter().any(Vec::is_empty)) - .map(|(name, _)| name.to_ascii_lowercase()) - .collect(); - let relation_names: HashMap = request - .relation_names - .iter() - .map(|(sentinel, name)| (sentinel.to_ascii_lowercase(), name.clone())) - .collect(); - let inputs = OneRowInputs { - one_row_relations: &one_row_relations, - relation_names: &relation_names, - }; - multiplying_cross_joins( - request.statements, - request.tokens, - request.cross_spans, - &inputs, - ) - .into_iter() - .map(|finding| LintDiagnostic { - code: rule.code, - message: finding - .relation - .as_deref() - .map_or(Cow::Borrowed(CROSS_JOIN_UNNAMED_MESSAGE), |relation| { - Cow::Owned(cross_join_message(relation, finding.cte.as_deref())) - }), - remediation: if finding.relation.is_some() { - Cow::Borrowed(rule.remediation) - } else { - Cow::Owned(file_wide_remediation(rule.code)) - }, - start: finding.span.start, - end: finding.span.end, - fix_unavailable_reason: Some(CROSS_JOIN_FIX_REASON), - fix: None, - }) - .collect() -} - -/// Help for a join no relation names: fix it, or record the file-wide exception it allows. -fn file_wide_remediation(code: &str) -> String { - let entry: String = [ - "[[rules.rule_exceptions]]".to_owned(), - format!("rule = \"{code}\""), - format!("path = \"{CROSS_JOIN_PATH_PLACEHOLDER}\""), - format!("reason = \"{CROSS_JOIN_REASON_PLACEHOLDER}\""), - ] - .iter() - .map(|line| format!("\n{SETTING_SNIPPET_INDENT}{line}")) - .collect(); - format!( - "Use a keyed JOIN, or make one side provably at most one row. This join reads no named \ - relation, so a relation-level exception cannot cover it; when the cartesian product is \ - intentional, record a reviewed file-wide exception in sqlbuild_project.toml:{entry}" - ) -} - -/// The finding message naming the joined relation and, inside a CTE, that CTE. -fn cross_join_message(relation: &str, cte: Option<&str>) -> String { - match cte { - Some(cte) => { - format!("{CROSS_JOIN_PREFIX}{relation}{CROSS_JOIN_CTE_MARKER}{cte}{CROSS_JOIN_SUFFIX}") - } - None => format!("{CROSS_JOIN_PREFIX}{relation}{CROSS_JOIN_SUFFIX}"), - } -} - -/// One explicit cross join found in the query tree. -struct CrossJoinTarget { - /// Character offsets where the joined relation may start in the token stream. - anchors: Vec, - /// Whether both joined sides may hold more than one row. - multiplies: bool, - /// The joined relation's name as authored, with dependency sentinels resolved. - relation: Option, - /// The top-level CTE whose query contains the join. - cte: Option, -} - -/// The cross join reported at one `CROSS` keyword. -struct CrossJoinFinding { - span: Span, - relation: Option, - cte: Option, -} - -/// What the analysis knows about relations named in the query. -struct OneRowInputs<'a> { - /// Lowercased dependency sentinels whose relation holds at most one row. - one_row_relations: &'a HashSet, - /// Lowercased dependency sentinels mapped to the relation names they stand for. - relation_names: &'a HashMap, -} - -struct Analysis<'a> { - inputs: &'a OneRowInputs<'a>, - ctes: HashMap, - targets: Vec, -} - -/// Cross joins at `cross_spans` that can multiply rows, each with its relation and CTE. -fn multiplying_cross_joins( - statements: &[Expression], - tokens: &[Token], - cross_spans: &[Span], - inputs: &OneRowInputs<'_>, -) -> Vec { - if cross_spans.is_empty() { - return Vec::new(); - } - let mut analysis = Analysis { - inputs, - ctes: HashMap::new(), - targets: Vec::new(), - }; - for statement in statements { - analysis.record_ctes(statement); - } - for statement in statements { - analysis.visit(statement, None, true); - } - let significant: Vec = (0..tokens.len()) - .filter(|&index| { - !matches!( - tokens[index].token_type, - TokenType::Space - | TokenType::Break - | TokenType::LineComment - | TokenType::BlockComment - ) - }) - .collect(); - let mut findings: Vec = Vec::new(); - for span in cross_spans { - let candidates: Vec = relation_anchor_candidates(tokens, &significant, *span); - let target: Option<&CrossJoinTarget> = analysis - .targets - .iter() - .find(|target| anchored_at(target, &candidates)); - match target { - Some(target) if !target.multiplies => {} - Some(target) => findings.push(CrossJoinFinding { - span: *span, - relation: target.relation.clone(), - cte: target.cte.clone(), - }), - None => findings.push(CrossJoinFinding { - span: *span, - relation: None, - cte: None, - }), - } - } - findings -} - -impl Analysis<'_> { - /// Record whether every CTE holds at most one row, in definition order. - fn record_ctes(&mut self, expression: &Expression) { - for node in expression.dfs() { - let Some(with) = with_clause(node) else { - continue; - }; - for cte in &with.ctes { - let one_row: bool = !with.recursive && self.at_most_one_row(&cte.this); - let name: String = cte.alias.name.to_ascii_lowercase(); - let known: bool = self.ctes.get(&name).copied().unwrap_or(one_row); - self.ctes.insert(name, known && one_row); - } - } - } - - /// Record joins under `expression`, labelled with the top-level CTE containing them. - fn visit(&mut self, expression: &Expression, cte: Option<&str>, top: bool) { - let mut bodies: Vec<&Expression> = Vec::new(); - if let Some(with) = with_clause(expression) { - for definition in &with.ctes { - bodies.push(&definition.this); - let label: Option<&str> = if top { - Some(&definition.alias.name) - } else { - cte - }; - self.visit(&definition.this, label, false); - } - } - if let Expression::Select(select) = expression { - self.record_joins(select, cte); - } - let branches_are_top: bool = top && is_set_operation(expression); - for child in expression.children() { - if bodies.iter().any(|body| std::ptr::eq(*body, child)) { - continue; - } - self.visit(child, cte, branches_are_top); - } - } - - fn record_joins(&mut self, select: &Select, cte: Option<&str>) { - let single_from: Option<&Expression> = select - .from - .as_ref() - .filter(|from| from.expressions.len() == 1) - .and_then(|from| from.expressions.first()); - for (index, join) in select.joins.iter().enumerate() { - if join.kind != JoinKind::Cross { - continue; - } - let left_one_row: bool = - index == 0 && single_from.is_some_and(|from| self.relation_one_row(from)); - let multiplies: bool = !(left_one_row || self.relation_one_row(&join.this)); - self.targets.push(CrossJoinTarget { - anchors: relation_anchors(&join.this), - multiplies, - relation: self.relation_name(&join.this), - cte: cte.map(str::to_owned), - }); - } - } - - fn relation_name(&self, relation: &Expression) -> Option { - match relation { - Expression::Table(table) => Some(self.table_name(table)), - Expression::Subquery(subquery) => { - subquery.alias.as_ref().map(|alias| alias.name.clone()) - } - Expression::Alias(alias) => Some(alias.alias.name.clone()), - _ => None, - } - } - - fn table_name(&self, table: &TableRef) -> String { - if table.schema.is_none() && table.catalog.is_none() { - let lowered: String = table.name.name.to_ascii_lowercase(); - if let Some(name) = self.inputs.relation_names.get(&lowered) { - return name.clone(); - } - } - [&table.catalog, &table.schema] - .into_iter() - .flatten() - .map(|part| part.name.as_str()) - .chain([table.name.name.as_str()]) - .collect::>() - .join(".") - } - - /// Whether a query expression returns at most one row. - fn at_most_one_row(&self, expression: &Expression) -> bool { - match expression { - Expression::Select(select) => self.select_one_row(select), - Expression::Subquery(subquery) => self.subquery_one_row(subquery), - Expression::Paren(paren) => self.at_most_one_row(&paren.this), - Expression::Values(values) => values.expressions.len() <= 1, - _ => false, - } - } - - fn subquery_one_row(&self, subquery: &Subquery) -> bool { - !subquery.lateral - && (subquery - .limit - .as_ref() - .is_some_and(|limit| !limit.percent && at_most_one(&limit.this)) - || self.at_most_one_row(&subquery.this)) - } - - fn select_one_row(&self, select: &Select) -> bool { - if select.limit_by.is_some() || select.connect.is_some() || !select.lateral_views.is_empty() - { - return false; - } - if limited_to_one_row(select) { - return true; - } - if select.expressions.iter().any(generates_rows) { - return false; - } - if select.group_by.is_none() && select.expressions.iter().any(aggregates) { - return true; - } - if select - .qualify - .as_ref() - .is_some_and(|qualify| first_unpartitioned_row(&qualify.this, select)) - { - return true; - } - if select.group_by.is_some() || !select.joins.is_empty() { - return false; - } - match &select.from { - None => true, - Some(from) => match from.expressions.as_slice() { - [relation] => self.relation_one_row(relation), - _ => false, - }, - } - } - - /// Whether a relation in FROM or JOIN holds at most one row. - fn relation_one_row(&self, relation: &Expression) -> bool { - match relation { - Expression::Table(table) => { - if table.schema.is_some() - || table.catalog.is_some() - || table.identifier_func.is_some() - || table.changes.is_some() - { - return false; - } - let name: String = table.name.name.to_ascii_lowercase(); - match self.ctes.get(&name) { - Some(one_row) => *one_row, - None => self.inputs.one_row_relations.contains(&name), - } - } - Expression::Subquery(subquery) => self.subquery_one_row(subquery), - Expression::Paren(paren) => self.relation_one_row(&paren.this), - Expression::Values(values) => values.expressions.len() <= 1, - _ => false, - } - } -} - -/// The WITH clause a query expression owns. -fn with_clause(expression: &Expression) -> Option<&With> { - match expression { - Expression::Select(select) => select.with.as_ref(), - Expression::Union(union) => union.with.as_ref(), - Expression::Intersect(intersect) => intersect.with.as_ref(), - Expression::Except(except) => except.with.as_ref(), - _ => None, - } -} - -fn is_set_operation(expression: &Expression) -> bool { - matches!( - expression, - Expression::Union(_) | Expression::Intersect(_) | Expression::Except(_) - ) -} - -fn limited_to_one_row(select: &Select) -> bool { - let limited: bool = select - .limit - .as_ref() - .is_some_and(|limit| !limit.percent && at_most_one(&limit.this)); - let fetched: bool = select.fetch.as_ref().is_some_and(|fetch| { - !fetch.percent && !fetch.with_ties && fetch.count.as_ref().is_some_and(at_most_one) - }); - let top: bool = select - .top - .as_ref() - .is_some_and(|top| !top.percent && !top.with_ties && at_most_one(&top.this)); - limited || fetched || top -} - -fn at_most_one(expression: &Expression) -> bool { - match expression { - Expression::Literal(literal) => match literal.as_ref() { - Literal::Number(text) => text.parse::().is_ok_and(|count| count <= 1), - _ => false, - }, - Expression::Paren(paren) => at_most_one(&paren.this), - _ => false, - } -} - -/// Whether a projected expression can return several rows per input row. -fn generates_rows(expression: &Expression) -> bool { - expression.dfs().any(|node| match node { - Expression::Unnest(_) - | Expression::Explode(_) - | Expression::ExplodeOuter(_) - | Expression::GenerateSeries(_) => true, - Expression::Function(function) => is_row_generator(&function.name), - _ => false, - }) -} - -fn is_row_generator(name: &str) -> bool { - let name: &str = name.rsplit('.').next().unwrap_or_default(); - ROW_GENERATOR_FUNCTIONS - .iter() - .any(|generator| generator.eq_ignore_ascii_case(name)) -} - -fn anchored_at(target: &CrossJoinTarget, candidates: &[usize]) -> bool { - target - .anchors - .iter() - .any(|anchor| candidates.contains(anchor)) -} - -/// Whether an expression aggregates outside any window function or subquery. -fn aggregates(expression: &Expression) -> bool { - if polyglot_sql::traversal::is_aggregate(expression) { - return true; - } - match expression { - Expression::WindowFunction(_) | Expression::Subquery(_) | Expression::Select(_) => false, - _ => expression.children().into_iter().any(aggregates), - } -} - -/// `QUALIFY ROW_NUMBER() OVER (ORDER BY ...) = 1` without a partition, directly or by alias. -fn first_unpartitioned_row(condition: &Expression, select: &Select) -> bool { - match condition { - Expression::And(operation) => { - first_unpartitioned_row(&operation.left, select) - || first_unpartitioned_row(&operation.right, select) - } - Expression::Paren(paren) => first_unpartitioned_row(&paren.this, select), - Expression::Eq(operation) => { - let compared: &Expression = match (&operation.left, &operation.right) { - (other, Expression::Literal(literal)) | (Expression::Literal(literal), other) - if matches!(literal.as_ref(), Literal::Number(text) if text == FIRST_ROW) => - { - other - } - _ => return false, - }; - match compared { - Expression::WindowFunction(_) => unpartitioned_row_number(compared), - Expression::Column(column) if column.table.is_none() => select - .expressions - .iter() - .any(|projection| match projection { - Expression::Alias(alias) => { - alias.alias.name.eq_ignore_ascii_case(&column.name.name) - && unpartitioned_row_number(&alias.this) - } - _ => false, - }), - _ => false, - } - } - _ => false, - } -} - -fn unpartitioned_row_number(expression: &Expression) -> bool { - matches!( - expression, - Expression::WindowFunction(window) - if matches!(window.this, Expression::RowNumber(_)) - && window.over.window_name.is_none() - && window.over.partition_by.is_empty() - ) -} - -/// Where a joined relation can start: its first name part, or a derived table's alias. -fn relation_anchors(relation: &Expression) -> Vec { - match relation { - Expression::Table(table) => { - let mut anchors: Vec = Vec::new(); - for identifier in [&table.catalog, &table.schema].into_iter().flatten() { - anchors.extend(identifier.span.map(|span| span.start)); - } - anchors.extend(table.name.span.map(|span| span.start)); - anchors.extend(table.span.map(|span| span.start)); - anchors - } - Expression::Subquery(subquery) => subquery - .alias - .as_ref() - .and_then(|alias| alias.span) - .map(|span| span.start) - .into_iter() - .collect(), - Expression::Alias(alias) => alias - .alias - .span - .map(|span| span.start) - .into_iter() - .collect(), - _ => Vec::new(), - } -} - -/// Where the relation after `CROSS JOIN` at `cross` starts, and a derived table's alias. -fn relation_anchor_candidates(tokens: &[Token], significant: &[usize], cross: Span) -> Vec { - let Some(position) = significant - .iter() - .position(|&index| tokens[index].span.start == cross.start) - else { - return Vec::new(); - }; - let Some(&first) = significant.get(position + 2) else { - return Vec::new(); - }; - let mut candidates: Vec = vec![tokens[first].span.start]; - if tokens[first].token_type == TokenType::LParen { - let mut depth: usize = 0; - let mut cursor: usize = position + 2; - while let Some(&index) = significant.get(cursor) { - match tokens[index].token_type { - TokenType::LParen => depth += 1, - TokenType::RParen => { - depth = depth.saturating_sub(1); - if depth == 0 { - break; - } - } - _ => {} - } - cursor += 1; - } - let mut alias: usize = cursor + 1; - if significant - .get(alias) - .is_some_and(|&index| tokens[index].token_type == TokenType::As) - { - alias += 1; - } - if let Some(&index) = significant.get(alias) { - candidates.push(tokens[index].span.start); - } - } - candidates -} diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/mod.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/mod.rs deleted file mode 100644 index 4b14cd8cb8..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -pub mod additional; -pub mod engine; -pub mod formatter; -pub mod formatter_syntax; -pub mod inline_relations; -pub mod join_predicates; -pub mod preparation; -pub mod printed_layout; -pub mod terminal_shape; -pub mod token_layout; diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/printed_layout.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/printed_layout.rs deleted file mode 100644 index 5aa8522702..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/printed_layout.rs +++ /dev/null @@ -1,931 +0,0 @@ -//! Printed SQL as tokens and gaps, with clause layout rules and line-width wrapping. - -use std::collections::HashMap; - -use polyglot_sql::Dialect; -use polyglot_sql::tokens::TokenType; - -use crate::sql_lint::constants::OPEN_PARENTHESIS; - -const CONTINUATION_INDENT: usize = 2; -const COMMA: &str = ","; -const SINGLE_SPACE: &str = " "; -const BETWEEN_KEYWORD: &str = "BETWEEN"; -const BETWEEN_SEPARATOR: &str = "AND"; -const VALUES_KEYWORD: &str = "VALUES"; -const DECODE_FUNCTION: &str = "DECODE"; -const CASE_KEYWORD: &str = "CASE"; -const SENTINEL_PREFIX: &str = "__SQB_LINT_"; -const SINGLE_ITEM_CLAUSES: [&str; 4] = ["SELECT", "WHERE", "QUALIFY", "HAVING"]; -const SELECT_MODIFIERS: [&str; 10] = [ - "DISTINCT", "ALL", "TOP", "PERCENT", "WITH", "TIES", "ON", "AS", "STRUCT", "VALUE", -]; -const LEADING_OPERATORS: [&str; 8] = ["+", "-", "*", "/", "%", "||", "AND", "OR"]; -const CHAIN_KEYWORDS: [&str; 2] = ["AND", "OR"]; -const BETWEEN_BOUNDARIES: [&str; 12] = [ - "AND", "OR", "WHEN", "THEN", "ELSE", "WHERE", "ON", "HAVING", "QUALIFY", "SELECT", ",", "NOT", -]; -const ARITHMETIC_OPERATORS: [&str; 5] = ["+", "-", "||", "/", "%"]; -const WINDOW_CLAUSE_KEYWORDS: [&str; 4] = ["ORDER", "ROWS", "RANGE", "GROUPS"]; -const CASE_BRANCH_KEYWORDS: [&str; 3] = ["WHEN", "THEN", "ELSE"]; -const TYPE_PARAMETER_WORDS: [&str; 1] = ["MAX"]; -const TYPE_NAMES: [&str; 30] = [ - "BIGNUMERIC", - "BINARY", - "BIT", - "CHAR", - "CHARACTER", - "DATETIME", - "DATETIME2", - "DATETIMEOFFSET", - "DEC", - "DECIMAL", - "DOUBLE", - "FLOAT", - "INTERVAL", - "NCHAR", - "NUMBER", - "NUMERIC", - "NVARCHAR", - "STRING", - "TEXT", - "TIME", - "TIMESTAMP", - "TIMESTAMPTZ", - "TIMESTAMP_LTZ", - "TIMESTAMP_NTZ", - "TIMESTAMP_TZ", - "TIMETZ", - "VARBINARY", - "VARCHAR", - "BYTES", - "BPCHAR", -]; - -/// Printed SQL whose layout passes only ever rewrite the gaps between tokens. -pub(crate) struct PrintedSql<'a> { - tokens: Vec, - kinds: Vec, - /// The text before each token, plus the text after the last token. - gaps: Vec, - depths: Vec, - partners: Vec>, - /// Gaps that line wrapping must keep: inside VALUES rows and type parameter lists. - locked: Vec, - token_widths: &'a HashMap, -} - -/// One printed line; end columns are absent when comments or multi-line tokens pin it. -struct Line { - first: usize, - last: usize, - indentation: usize, - end_columns: Option>, -} - -impl<'a> PrintedSql<'a> { - pub(crate) fn parse( - sql: &str, - dialect: &Dialect, - token_widths: &'a HashMap, - ) -> Result, String> { - let tokens = dialect - .tokenize(sql) - .map_err(|error| format!("native formatter tokenization failed: {error}"))?; - if tokens.is_empty() { - return Ok(None); - } - let characters: Vec = sql.chars().collect(); - let mut texts: Vec = Vec::with_capacity(tokens.len()); - let mut gaps: Vec = Vec::with_capacity(tokens.len() + 1); - let mut previous_end = 0_usize; - for token in &tokens { - gaps.push(characters[previous_end..token.span.start].iter().collect()); - texts.push( - characters[token.span.start..token.span.end] - .iter() - .collect(), - ); - previous_end = token.span.end; - } - gaps.push(characters[previous_end..].iter().collect()); - let (depths, partners) = bracket_structure(&texts); - let locked = vec![false; texts.len() + 1]; - Ok(Some(Self { - kinds: tokens.iter().map(|token| token.token_type).collect(), - tokens: texts, - gaps, - depths, - partners, - locked, - token_widths, - })) - } - - fn len(&self) -> usize { - self.tokens.len() - } - - fn upper(&self, index: usize) -> String { - self.tokens[index].to_ascii_uppercase() - } - - fn is(&self, index: usize, text: &str) -> bool { - self.tokens - .get(index) - .is_some_and(|token| token.eq_ignore_ascii_case(text)) - } - - fn line_ranges(&self, first: usize, last: usize) -> Vec<(usize, usize)> { - let mut ranges: Vec<(usize, usize)> = Vec::new(); - let mut start = first; - for index in first + 1..=last { - if self.gaps[index].contains('\n') { - ranges.push((start, index - 1)); - start = index; - } - } - ranges.push((start, last)); - ranges - } - - /// The first and last token of the printed line holding `index`. - fn line_of(&self, index: usize) -> (usize, usize) { - let first = (1..=index) - .rev() - .find(|&position| self.gaps[position].contains('\n')) - .unwrap_or(0); - let last = (index + 1..self.len()) - .find(|&position| self.gaps[position].contains('\n')) - .map_or(self.len() - 1, |position| position - 1); - (first, last) - } - - fn line(&self, first: usize, last: usize) -> Line { - let indentation = self.indentation(first); - let lead = self.gaps[first].rsplit('\n').next().unwrap_or_default(); - let trailing = self.gaps[last + 1].split('\n').next().unwrap_or_default(); - let commented = !lead.trim().is_empty() - || !trailing.trim().is_empty() - || (first + 1..=last).any(|index| !self.gaps[index].trim().is_empty()); - let multiline = (first..=last).any(|index| self.tokens[index].contains('\n')); - let end_columns = (!commented && !multiline).then(|| { - let mut column = indentation; - let mut ends: Vec = Vec::with_capacity(last - first + 1); - for index in first..=last { - if index > first { - column += self.gaps[index].chars().count(); - } - column += self.width(index); - ends.push(column); - } - ends - }); - Line { - first, - last, - indentation, - end_columns, - } - } - - /// Indentation of the line that starts at token `first`. - fn indentation(&self, first: usize) -> usize { - self.gaps[first] - .rsplit('\n') - .next() - .unwrap_or_default() - .chars() - .count() - } - - fn width(&self, index: usize) -> usize { - let token = &self.tokens[index]; - self.token_widths - .get(&token.to_ascii_uppercase()) - .copied() - .unwrap_or_else(|| token.chars().count()) - } - - /// Whether any gap of the tokens after `first` up to `last` holds a line break. - fn spans_lines(&self, first: usize, last: usize) -> bool { - (first + 1..=last).any(|index| self.gaps[index].contains('\n')) - } - - pub(crate) fn render(&self) -> String { - let mut output = String::new(); - for (gap, token) in self.gaps.iter().zip(&self.tokens) { - output.push_str(gap); - output.push_str(token); - } - output.push_str(&self.gaps[self.tokens.len()]); - output - } -} - -fn bracket_structure(tokens: &[String]) -> (Vec, Vec>) { - let mut depths: Vec = Vec::with_capacity(tokens.len()); - let mut partners: Vec> = vec![None; tokens.len()]; - let mut open: Vec = Vec::new(); - for (index, token) in tokens.iter().enumerate() { - match token.as_str() { - "(" | "[" | "{" => { - depths.push(open.len()); - open.push(index); - } - ")" | "]" | "}" => { - let opener = open.pop(); - depths.push(open.len()); - if let Some(opener) = opener { - partners[opener] = Some(index); - partners[index] = Some(opener); - } - } - _ => depths.push(open.len()), - } - } - (depths, partners) -} - -fn newline(indentation: usize) -> String { - format!("\n{}", " ".repeat(indentation)) -} - -impl PrintedSql<'_> { - /// Apply the layout rules that only move existing line breaks before wrapping. - pub(crate) fn arrange_clauses(&mut self, line_width: usize) { - for index in 0..self.len() { - if self.is(index, DECODE_FUNCTION) { - self.pair_decode_arguments(index); - } - } - for index in 1..self.len() { - if CHAIN_KEYWORDS.contains(&self.upper(index).as_str()) { - self.join_chain_break_in_group(index); - } - } - for index in 0..self.len() { - if self.is(index, VALUES_KEYWORD) { - self.break_values_rows_unless_fitting(index, line_width); - } - } - for index in 0..self.len() { - if self.kinds[index] == TokenType::With { - self.separate_common_table_expressions(index); - } - } - for index in 0..self.len() { - if SINGLE_ITEM_CLAUSES.contains(&self.upper(index).as_str()) { - self.join_single_line_body(index, line_width); - } - } - } - - /// Start the next line with any binary operator whose operand spans several lines. - pub(crate) fn lead_operators_of_multiline_operands(&mut self) { - for index in 1..self.len().saturating_sub(1) { - if !LEADING_OPERATORS.contains(&self.upper(index).as_str()) - || self.gaps[index] != SINGLE_SPACE - || self.gaps[index + 1].contains('\n') - || !self.ends_operand(index - 1) - || (self.is(index, "AND") && self.follows_between(index)) - { - continue; - } - let multiline = self - .left_operand_start(index - 1) - .is_some_and(|start| self.spans_lines(start, index - 1)) - || self - .right_operand_end(index + 1) - .is_some_and(|end| self.spans_lines(index + 1, end)); - if multiline { - let (first, _) = self.line_of(index); - self.gaps[index] = newline(self.indentation(first)); - } - } - } - - /// Put the first DECODE argument on its own line and each search/result pair on one line. - fn pair_decode_arguments(&mut self, name: usize) { - let opener = name + 1; - if !self.is(opener, OPEN_PARENTHESIS) || (name > 0 && self.is(name - 1, ".")) { - return; - } - let Some(closer) = self.partners[opener] else { - return; - }; - if !self.gaps[opener + 1].contains('\n') { - return; - } - let mut argument = 0_usize; - for index in opener + 2..closer { - if self.depths[index] != self.depths[opener] + 1 || self.tokens[index - 1] != COMMA { - continue; - } - argument += 1; - if argument.is_multiple_of(2) && self.gaps[index].trim().is_empty() { - self.gaps[index] = SINGLE_SPACE.to_string(); - } - } - } - - /// Join an AND or OR line break inside a bracket group whose arguments share its line. - fn join_chain_break_in_group(&mut self, index: usize) { - if !self.gaps[index].contains('\n') || !self.gaps[index].trim().is_empty() { - return; - } - let depth = self.depths[index]; - let Some(opener) = (0..index) - .rev() - .find(|&previous| self.depths[previous] < depth) - else { - return; - }; - if self.tokens[opener] == OPEN_PARENTHESIS && !self.spans_lines(opener, index - 1) { - self.gaps[index] = SINGLE_SPACE.to_string(); - } - } - - /// Start every row of a VALUES list on its own line unless the whole list fits on its line. - fn break_values_rows_unless_fitting(&mut self, values: usize, line_width: usize) { - let rows = self.values_rows(values); - let Some(&(_, last_closer)) = rows.last() else { - return; - }; - let (first, last) = self.line_of(values); - let fits = !self.spans_lines(values, last_closer) - && self - .line(first, last) - .end_columns - .is_some_and(|ends| ends.last().is_some_and(|end| *end <= line_width)); - if fits { - return; - } - let mut indentation = self.indentation(first); - if self.gaps[values].contains('\n') { - indentation = self.indentation(values); - } else if values > 0 - && self.tokens[values - 1] == OPEN_PARENTHESIS - && let Some(closer) = self.partners[values - 1] - { - self.set_break(values, &newline(indentation + CONTINUATION_INDENT)); - self.set_break(closer, &newline(indentation)); - indentation += CONTINUATION_INDENT; - } - let row_line = newline(indentation + CONTINUATION_INDENT); - for (opener, _) in rows { - self.set_break(opener, &row_line); - } - } - - /// Separate the CTEs of the WITH at `with`, and the statement after them, by one blank line. - fn separate_common_table_expressions(&mut self, with: usize) { - let (first, _) = self.line_of(with); - let indentation = self.indentation(first); - let mut item = with + 1; - if self.is(item, "RECURSIVE") { - item += 1; - } - while let Some(end) = self.common_table_expression_end(item) { - let next = end + 1; - if self.is(next, ",") { - self.blank_line_before(next + 1, indentation); - item = next + 1; - continue; - } - if next < self.len() && !self.is(next, ";") && !self.is(next, ")") { - self.blank_line_before(next, indentation); - } - break; - } - } - - /// The last token of the CTE, or CTE macro placeholder, that starts at `item`. - fn common_table_expression_end(&self, item: usize) -> Option { - let name = self.tokens.get(item)?; - if name.to_ascii_uppercase().starts_with(SENTINEL_PREFIX) - && !self.is(item + 1, "AS") - && !self.is(item + 1, OPEN_PARENTHESIS) - { - return Some(item); - } - if !matches!( - self.kinds[item], - TokenType::Var | TokenType::Identifier | TokenType::QuotedIdentifier - ) && !name.chars().all(|c| c.is_alphanumeric() || c == '_') - { - return None; - } - let mut index = item + 1; - if self.is(index, OPEN_PARENTHESIS) { - index = self.partners[index]? + 1; - } - if !self.is(index, "AS") { - return None; - } - index += 1; - if self.is(index, "NOT") { - index += 1; - } - if self.is(index, "MATERIALIZED") { - index += 1; - } - if !self.is(index, OPEN_PARENTHESIS) { - return None; - } - self.partners[index] - } - - /// Replace the line break before `index` with exactly one blank line, keeping comments. - fn blank_line_before(&mut self, index: usize, indentation: usize) { - let gap = &self.gaps[index]; - let (trailing, rest) = gap.split_once('\n').unwrap_or((gap.as_str(), "")); - let rest = rest.trim_start(); - let margin = " ".repeat(indentation); - self.gaps[index] = if rest.is_empty() { - format!("{}\n\n{margin}", trailing.trim_end()) - } else { - format!("{}\n\n{margin}{rest}", trailing.trim_end()) - }; - } - - /// Move a clause body that is one line under its clause keyword onto the keyword line. - fn join_single_line_body(&mut self, clause: usize, line_width: usize) { - let (first, last) = self.line_of(clause); - let body = last + 1; - let is_select = self.is(clause, "SELECT"); - let modifiers_only = (clause + 1..=last).all(|index| { - SELECT_MODIFIERS.contains(&self.upper(index).as_str()) - || self.kinds[index] == TokenType::Number - || matches!(self.tokens[index].as_str(), "(" | ")" | ",") - || self.depths[index] > self.depths[clause] - }); - if body >= self.len() - || clause != first - || (!is_select && clause != last) - || (is_select && !modifiers_only) - || !self.gaps[body].contains('\n') - || !self.gaps[body].trim().is_empty() - { - return; - } - let clause_indentation = self.indentation(first); - if self.indentation(body) <= clause_indentation { - return; - } - let (_, body_last) = self.line_of(body); - if body_last + 1 < self.len() && self.indentation(body_last + 1) > clause_indentation { - return; - } - let pinned = (body + 1..=body_last).any(|index| !self.gaps[index].trim().is_empty()) - || (first..=body_last).any(|index| self.tokens[index].contains('\n')); - if pinned || (!is_select && self.has_top_level_chain(body, body_last)) { - return; - } - let mut width = clause_indentation; - for index in first..=body_last { - if index > first { - width += if index == body { - 1 - } else { - self.gaps[index].chars().count() - }; - } - width += self.width(index); - } - width += self.gaps[body_last + 1] - .split('\n') - .next() - .unwrap_or_default() - .chars() - .count(); - if width <= line_width { - self.gaps[body] = SINGLE_SPACE.to_string(); - } - } - - fn has_top_level_chain(&self, first: usize, last: usize) -> bool { - let depth = (first..=last).map(|index| self.depths[index]).min(); - (first..=last).any(|index| { - Some(self.depths[index]) == depth - && CHAIN_KEYWORDS.contains(&self.upper(index).as_str()) - && !(self.is(index, "AND") && self.follows_between(index)) - }) - } - - /// Whether the AND at `index` separates the bounds of a BETWEEN. - fn follows_between(&self, index: usize) -> bool { - let depth = self.depths[index]; - for previous in (0..index).rev() { - if self.depths[previous] < depth { - return false; - } - if self.depths[previous] > depth { - continue; - } - let upper = self.upper(previous); - if upper == BETWEEN_KEYWORD { - return true; - } - if BETWEEN_BOUNDARIES.contains(&upper.as_str()) { - return false; - } - } - false - } - - fn ends_operand(&self, index: usize) -> bool { - matches!(self.tokens[index].as_str(), ")" | "]") - || self.is(index, "END") - || matches!( - self.kinds[index], - TokenType::Var - | TokenType::Identifier - | TokenType::QuotedIdentifier - | TokenType::Number - | TokenType::String - ) - } - - /// The first token of a bracket group or CASE expression that ends at `last`. - fn left_operand_start(&self, last: usize) -> Option { - if matches!(self.tokens[last].as_str(), ")" | "]") { - return self.partners[last]; - } - if !self.is(last, "END") { - return None; - } - let depth = self.depths[last]; - let mut nested = 0_usize; - for previous in (0..last).rev() { - if self.depths[previous] != depth { - continue; - } - if self.is(previous, "END") { - nested += 1; - } else if self.is(previous, "CASE") { - if nested == 0 { - return Some(previous); - } - nested -= 1; - } - } - None - } - - /// The last token of a bracket group, call, or CASE expression that starts at `first`. - fn right_operand_end(&self, first: usize) -> Option { - if self.tokens[first] == OPEN_PARENTHESIS { - return self.partners[first]; - } - let word = self.tokens[first] - .chars() - .all(|character| character.is_alphanumeric() || character == '_'); - if word && self.is(first + 1, OPEN_PARENTHESIS) { - return self.partners[first + 1]; - } - if !self.is(first, CASE_KEYWORD) { - return None; - } - self.case_end(first) - } -} - -/// Line width and the printed widths of tokens that stand in for longer authored text. -pub(crate) struct WrapOptions<'a> { - pub(crate) line_width: usize, - pub(crate) token_widths: &'a HashMap, -} - -/// Re-break every line of `printed` that is longer than the line width and can be shortened. -pub(crate) fn wrap_lines(printed: &mut PrintedSql<'_>, line_width: usize) { - printed.lock_unbreakable_groups(); - let mut pending: Vec<(usize, usize)> = printed.line_ranges(0, printed.len() - 1); - pending.reverse(); - while let Some((first, last)) = pending.pop() { - let line = printed.line(first, last); - if !overflows(&line, line_width) || !printed.break_line(&line) { - continue; - } - let mut lines = printed.line_ranges(first, last); - lines.reverse(); - pending.extend(lines); - } -} - -fn overflows(line: &Line, line_width: usize) -> bool { - line.end_columns - .as_ref() - .is_some_and(|ends| ends.last().is_some_and(|end| *end > line_width)) -} - -impl PrintedSql<'_> { - /// Lock the inside of VALUES rows and of type parameter lists such as `NUMBER(38, 2)`. - fn lock_unbreakable_groups(&mut self) { - for index in 0..self.len() { - if self.is(index, VALUES_KEYWORD) { - for (opener, closer) in self.values_rows(index) { - self.lock(opener, closer); - } - } else if let Some(closer) = self.type_parameters_closer(index) { - self.lock(index - 1, closer); - } - } - } - - fn lock(&mut self, first: usize, last: usize) { - for gap in first + 1..=last { - self.locked[gap] = true; - } - } - - /// The bracketed rows that directly follow the VALUES keyword at `values`. - fn values_rows(&self, values: usize) -> Vec<(usize, usize)> { - let mut rows: Vec<(usize, usize)> = Vec::new(); - let mut opener = values + 1; - while opener < self.len() && self.tokens[opener] == OPEN_PARENTHESIS { - let Some(closer) = self.partners[opener] else { - break; - }; - rows.push((opener, closer)); - if !(self.is(closer + 1, COMMA) && self.is(closer + 2, OPEN_PARENTHESIS)) { - break; - } - opener = closer + 2; - } - rows - } - - /// The closer of a type parameter list opened at `opener`, such as `(38, 2)` of `NUMBER`. - fn type_parameters_closer(&self, opener: usize) -> Option { - if opener < 1 || self.tokens[opener] != OPEN_PARENTHESIS { - return None; - } - let closer = self.partners[opener]?; - let name = &self.tokens[opener - 1]; - if closer == opener + 1 || !name.chars().all(|c| c.is_alphanumeric() || c == '_') { - return None; - } - let type_position = opener > 1 && (self.is(opener - 2, "::") || self.is(opener - 2, "AS")); - if !type_position && !TYPE_NAMES.contains(&name.to_ascii_uppercase().as_str()) { - return None; - } - let parameters = (opener + 1..closer).all(|index| { - let token = &self.tokens[index]; - token == COMMA - || is_digits(token) - || TYPE_PARAMETER_WORDS.contains(&token.to_ascii_uppercase().as_str()) - }); - parameters.then_some(closer) - } - - /// Apply the outermost applicable break: chains, VALUES rows, bracket groups, then commas. - fn break_line(&mut self, line: &Line) -> bool { - let values_row = self.tokens[line.first] == OPEN_PARENTHESIS && self.locked[line.first + 1]; - !values_row - && (self.break_before(line, &CHAIN_KEYWORDS, 0) - || self.break_before(line, &ARITHMETIC_OPERATORS, CONTINUATION_INDENT) - || self.break_values_rows(line) - || self.explode_group(line) - || self.break_after_commas(line)) - } - - /// Put each row of a VALUES list that starts this line on its own line. - fn break_values_rows(&mut self, line: &Line) -> bool { - let depth = self.line_depth(line); - let Some(values) = (line.first..=line.last).find(|&index| { - self.is(index, VALUES_KEYWORD) - && (self.depths[index] == depth || index == line.first) - && !self.locked[index + 1] - }) else { - return false; - }; - let rows = self.values_rows(values); - if rows.is_empty() - || rows - .iter() - .all(|(opener, _)| self.gaps[*opener].contains('\n')) - { - return false; - } - let row_line = newline(line.indentation + CONTINUATION_INDENT); - for (opener, _) in rows { - self.gaps[opener] = row_line.clone(); - } - true - } - - /// Put the arguments or CASE branches of the outermost, widest group on their own lines. - fn explode_group(&mut self, line: &Line) -> bool { - let Some(ends) = line.end_columns.as_ref() else { - return false; - }; - let start_column = |index: usize| { - if index == line.first { - line.indentation - } else { - ends[index - 1 - line.first] - } - }; - let line_end = ends[ends.len() - 1]; - let mut chosen: Option<(usize, usize, usize)> = None; - for opener in line.first..=line.last { - let Some(closer) = self.group_closer(opener, line) else { - continue; - }; - let end_column = if closer > line.last { - line_end - } else { - ends[closer - line.first] - }; - let width = end_column - start_column(opener); - let better = chosen.is_none_or(|(best, _, best_width)| { - self.depths[opener] < self.depths[best] - || (self.depths[opener] == self.depths[best] && width >= best_width) - }); - if better { - chosen = Some((opener, closer, width)); - } - } - let Some((opener, closer, _)) = chosen else { - return false; - }; - if closer > line.last { - self.indent_lines(opener + 1, closer - 1, CONTINUATION_INDENT); - } - let inner = newline(line.indentation + CONTINUATION_INDENT); - if self.is(opener, CASE_KEYWORD) { - self.explode_case(opener, closer, line.indentation); - return true; - } - self.set_break(opener + 1, &inner); - self.set_break(closer, &newline(line.indentation)); - if self.is(opener + 1, VALUES_KEYWORD) { - return true; - } - let window = opener > 0 && self.is(opener - 1, "OVER"); - let decode = opener > 0 && self.is(opener - 1, DECODE_FUNCTION); - let mut argument = 0_usize; - for index in opener + 2..closer { - if self.depths[index] != self.depths[opener] + 1 { - continue; - } - if window { - if WINDOW_CLAUSE_KEYWORDS.contains(&self.upper(index).as_str()) { - self.set_break(index, &inner); - } - } else if self.tokens[index - 1] == COMMA { - argument += 1; - if !decode || !argument.is_multiple_of(2) { - self.set_break(index, &inner); - } - } - } - true - } - - /// The closer of an explodable group opened at `opener` on `line`. - fn group_closer(&self, opener: usize, line: &Line) -> Option { - if self.is(opener, CASE_KEYWORD) { - let end = self.case_end(opener)?; - return (end <= line.last && end > opener + 2).then_some(end); - } - let closer = self.partners[opener]?; - let explodable = closer > opener + 2 - && matches!(self.tokens[opener].as_str(), "(" | "[" | "{") - && !self.locked[opener + 1] - && (closer <= line.last || opener < line.last); - explodable.then_some(closer) - } - - /// The END of the CASE expression at `case`. - fn case_end(&self, case: usize) -> Option { - let depth = self.depths[case]; - let mut nested = 0_usize; - for index in case + 1..self.len() { - if self.depths[index] < depth { - return None; - } - if self.depths[index] != depth { - continue; - } - if self.is(index, CASE_KEYWORD) { - nested += 1; - } else if self.is(index, "END") { - if nested == 0 { - return Some(index); - } - nested -= 1; - } - } - None - } - - /// Put each WHEN, THEN and ELSE of a one-line CASE on its own line and END under CASE. - fn explode_case(&mut self, case: usize, end: usize, indentation: usize) { - let inner = newline(indentation + CONTINUATION_INDENT); - let depth = self.depths[case]; - let mut nested = 0_usize; - for index in case + 1..end { - if self.depths[index] != depth { - continue; - } - if self.is(index, CASE_KEYWORD) { - nested += 1; - } else if self.is(index, "END") { - nested -= 1; - } else if nested == 0 && CASE_BRANCH_KEYWORDS.contains(&self.upper(index).as_str()) { - self.set_break(index, &inner); - } - } - self.set_break(end, &newline(indentation)); - } - - /// Replace a whitespace-only gap; comments keep their gaps. - fn set_break(&mut self, index: usize, gap: &str) { - if self.gaps[index].trim().is_empty() { - self.gaps[index] = gap.to_string(); - } - } - - /// Indent lines starting within `first..=last` by `extra`, except gaps with block comments. - fn indent_lines(&mut self, first: usize, last: usize, extra: usize) { - let margin = format!("\n{}", " ".repeat(extra)); - for index in first..=last { - if self.gaps[index].contains('\n') && !self.gaps[index].contains("/*") { - self.gaps[index] = self.gaps[index].replace('\n', &margin); - } - } - } - - /// Break before every operator of `operators` at the line's outermost depth. - fn break_before(&mut self, line: &Line, operators: &[&str], extra_indent: usize) -> bool { - let depth = self.line_depth(line); - let mut between = false; - let mut breaks: Vec = Vec::new(); - for index in line.first..=line.last { - if self.depths[index] != depth || self.locked[index] { - continue; - } - let upper = self.upper(index); - if upper == BETWEEN_KEYWORD { - between = true; - continue; - } - if upper == BETWEEN_SEPARATOR && between { - between = false; - continue; - } - if index > line.first - && operators.contains(&upper.as_str()) - && self.gaps[index] == SINGLE_SPACE - && ends_operand(&self.tokens[index - 1]) - { - breaks.push(index); - } - } - let indentation = newline(line.indentation + extra_indent); - for index in &breaks { - self.gaps[*index] = indentation.clone(); - } - !breaks.is_empty() - } - - /// Break after every comma at the outermost comma depth of the line. - fn break_after_commas(&mut self, line: &Line) -> bool { - let Some(depth) = (line.first..line.last) - .filter(|&index| self.tokens[index] == COMMA && !self.locked[index + 1]) - .map(|index| self.depths[index]) - .min() - else { - return false; - }; - let indentation = newline(line.indentation + CONTINUATION_INDENT); - let mut changed = false; - for index in line.first..line.last { - if self.depths[index] == depth && self.tokens[index] == COMMA && !self.locked[index + 1] - { - self.gaps[index + 1] = indentation.clone(); - changed = true; - } - } - changed - } - - fn line_depth(&self, line: &Line) -> usize { - (line.first..=line.last) - .map(|index| self.depths[index]) - .min() - .unwrap_or_default() - } -} - -fn is_digits(token: &str) -> bool { - token.chars().all(|character| character.is_ascii_digit()) -} - -fn ends_operand(token: &str) -> bool { - token.chars().last().is_some_and(|character| { - character.is_alphanumeric() || matches!(character, '_' | '\'' | '"' | '`' | ')' | ']') - }) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/terminal_shape.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/terminal_shape.rs deleted file mode 100644 index 73637fa9e9..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/terminal_shape.rs +++ /dev/null @@ -1,303 +0,0 @@ -use polyglot_sql::tokens::{Span, Token, TokenType}; - -const CEREMONIAL_SELECT_LITERAL: &str = "1"; - -pub(super) fn final_cte_name_spans(tokens: &[Token], depths: &[usize]) -> Vec { - let roots = root_with_indices(tokens, depths); - let mut names: Vec<(usize, bool)> = Vec::new(); - for (index, token) in tokens.iter().enumerate() { - if token.token_type != TokenType::With { - continue; - } - let depth = depths[index]; - let first = names.len(); - let mut next = index + 1; - if tokens - .get(next) - .is_some_and(|token| token.text.eq_ignore_ascii_case("recursive")) - { - next += 1; - } - loop { - let name = next; - let Some(open) = cte_body_open(tokens, depths, name) else { - break; - }; - let Some(close) = (open + 1..tokens.len()).find(|&position| { - depths[position] == depth + 1 && tokens[position].token_type == TokenType::RParen - }) else { - break; - }; - names.push((name, false)); - if tokens - .get(close + 1) - .is_none_or(|token| token.token_type != TokenType::Comma) - { - break; - } - next = close + 2; - } - if roots.contains(&index) - && names.len() > first - && let Some((_, terminal)) = names.last_mut() - { - *terminal = true; - } - } - names - .into_iter() - .filter_map(|(index, terminal)| { - let is_final = tokens[index].text.eq_ignore_ascii_case("final"); - (terminal != is_final).then_some(tokens[index].span) - }) - .collect() -} - -pub(super) fn root_with_indices(tokens: &[Token], depths: &[usize]) -> Vec { - let significant: Vec = (0..tokens.len()) - .filter(|&index| { - !matches!( - tokens[index].token_type, - TokenType::Space - | TokenType::Break - | TokenType::LineComment - | TokenType::BlockComment - ) - }) - .collect(); - significant - .split(|&index| tokens[index].token_type == TokenType::Semicolon) - .filter_map(|statement| { - root_query_tokens(tokens, depths, statement) - .first() - .copied() - }) - .filter(|&index| tokens[index].token_type == TokenType::With) - .collect() -} - -fn root_query_tokens<'a>( - tokens: &[Token], - depths: &[usize], - mut significant: &'a [usize], -) -> &'a [usize] { - while significant - .last() - .is_some_and(|&index| tokens[index].token_type == TokenType::Semicolon) - { - significant = &significant[..significant.len() - 1]; - } - while let Some((&first, rest)) = significant.split_first() { - if tokens[first].token_type != TokenType::LParen - || rest - .last() - .is_none_or(|&index| tokens[index].token_type != TokenType::RParen) - || rest.iter().any(|&index| depths[index] <= depths[first]) - { - break; - } - significant = &rest[..rest.len() - 1]; - } - significant -} - -fn cte_body_open(tokens: &[Token], depths: &[usize], name: usize) -> Option { - let mut position = name + 1; - if tokens.get(position)?.token_type == TokenType::LParen { - position = (position + 1..tokens.len()).find(|&index| { - depths[index] == depths[name] + 1 && tokens[index].token_type == TokenType::RParen - })? + 1; - } - if tokens.get(position)?.token_type != TokenType::As { - return None; - } - position += 1; - if tokens.get(position)?.token_type == TokenType::Not { - position += 1; - } - if tokens - .get(position)? - .text - .eq_ignore_ascii_case("materialized") - { - position += 1; - } - (tokens.get(position)?.token_type == TokenType::LParen).then_some(position) -} - -pub(super) fn collect_terminal_shape_facts( - tokens: &[Token], - depths: &[usize], - significant: &[usize], - allows_ceremonial_select: bool, -) -> (Vec, Vec) { - let mut cte_only_bodies: Vec = Vec::new(); - let mut terminal_selects: Vec = Vec::new(); - let significant = root_query_tokens(tokens, depths, significant); - let Some(&first) = significant.first() else { - return (cte_only_bodies, terminal_selects); - }; - let root_depth = depths[first]; - let has_top_level_with = significant.iter().any(|&index| { - depths[index] == root_depth && tokens[index].text.eq_ignore_ascii_case("with") - }); - let terminal_set_operator = significant.iter().find(|&&index| { - depths[index] == root_depth - && matches!( - tokens[index].token_type, - TokenType::Union | TokenType::Intersect | TokenType::Except - ) - }); - if let Some(&operator) = terminal_set_operator { - cte_only_bodies.push(tokens[operator].span); - if has_top_level_with { - terminal_selects.push(tokens[operator].span); - } - return (cte_only_bodies, terminal_selects); - } - let Some(root_select_position) = significant.iter().position(|&index| { - depths[index] == root_depth && tokens[index].token_type == TokenType::Select - }) else { - return (cte_only_bodies, terminal_selects); - }; - let root_select = significant[root_select_position]; - let tail: Vec = significant[root_select_position + 1..] - .iter() - .copied() - .take_while(|&index| tokens[index].token_type != TokenType::Semicolon) - .filter(|&index| depths[index] == root_depth) - .collect(); - let has_terminal_logic = tail.iter().any(|&index| { - matches!( - tokens[index].token_type, - TokenType::Join - | TokenType::Where - | TokenType::Group - | TokenType::Having - | TokenType::Qualify - | TokenType::Order - | TokenType::Limit - | TokenType::Offset - | TokenType::Union - | TokenType::Intersect - | TokenType::Except - ) - }); - if !has_top_level_with || has_terminal_logic { - cte_only_bodies.push(tokens[root_select].span); - } - if !has_top_level_with { - return (cte_only_bodies, terminal_selects); - } - - let final_cte = (0..root_select_position) - .filter(|&position| { - let index = significant[position]; - depths[index] == root_depth - && tokens[index].token_type == TokenType::As - && significant - .get(position + 1) - .is_some_and(|&next| tokens[next].token_type == TokenType::LParen) - }) - .filter_map(|position| cte_name_before_as(tokens, significant, position)) - .next_back(); - let from_position = tail - .iter() - .position(|&index| is_terminal_from(tokens, significant, index)); - let terminal_is_plain = from_position.is_some_and(|position| { - let projection = &tail[..position]; - let relation = tail.get(position + 1); - let remainder = &tail[position.saturating_add(2)..]; - !projection.is_empty() - && plain_terminal_projection(tokens, projection) - && relation.is_some_and(|&index| { - final_cte - .as_ref() - .is_some_and(|name| tokens[index].text.eq_ignore_ascii_case(name)) - }) - && remainder.is_empty() - }); - let terminal_is_ceremonial = allows_ceremonial_select - && tail.len() == 1 - && tokens[tail[0]].text == CEREMONIAL_SELECT_LITERAL; - if !terminal_is_plain && !terminal_is_ceremonial { - terminal_selects.push(tokens[root_select].span); - } - (cte_only_bodies, terminal_selects) -} - -fn cte_name_before_as( - tokens: &[Token], - significant: &[usize], - as_position: usize, -) -> Option { - let previous_position = as_position.checked_sub(1)?; - let previous = significant[previous_position]; - if is_lint_identifier(&tokens[previous]) || tokens[previous].text.eq_ignore_ascii_case("final") - { - return Some(tokens[previous].text.to_ascii_lowercase()); - } - if tokens[previous].token_type != TokenType::RParen { - return None; - } - - let mut balance = 1_usize; - for position in (0..previous_position).rev() { - let index = significant[position]; - match tokens[index].token_type { - TokenType::RParen => balance += 1, - TokenType::LParen => { - balance = balance.saturating_sub(1); - if balance == 0 { - let name = position.checked_sub(1).map(|value| significant[value])?; - return (is_lint_identifier(&tokens[name]) - || tokens[name].text.eq_ignore_ascii_case("final")) - .then(|| tokens[name].text.to_ascii_lowercase()); - } - } - _ => {} - } - } - None -} - -fn plain_terminal_projection(tokens: &[Token], projection: &[usize]) -> bool { - projection.iter().all(|&index| { - is_terminal_projection_identifier(&tokens[index]) - || matches!( - tokens[index].token_type, - TokenType::Star | TokenType::Dot | TokenType::Comma | TokenType::As - ) - }) -} - -fn is_terminal_projection_identifier(token: &Token) -> bool { - is_lint_identifier(token) - || matches!( - token.token_type, - TokenType::Comment - | TokenType::Date - | TokenType::Key - | TokenType::Sequence - | TokenType::Time - ) - || (token.token_type == TokenType::Percent && token.text.eq_ignore_ascii_case("percent")) -} - -fn is_lint_identifier(token: &Token) -> bool { - matches!( - token.token_type, - TokenType::Identifier | TokenType::QuotedIdentifier | TokenType::Var - ) -} - -fn is_terminal_from(tokens: &[Token], significant: &[usize], index: usize) -> bool { - if tokens[index].token_type != TokenType::From { - return false; - } - significant - .iter() - .position(|&candidate| candidate == index) - .and_then(|position| position.checked_sub(1)) - .is_none_or(|position| tokens[significant[position]].token_type != TokenType::Distinct) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/token_layout.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/token_layout.rs deleted file mode 100644 index 82971be7fa..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/token_layout.rs +++ /dev/null @@ -1,772 +0,0 @@ -//! Print authored SQL tokens with whitespace taken from a parse-tree layout oracle. - -use std::collections::{BTreeMap, BTreeSet}; - -use polyglot_sql::Dialect; -use polyglot_sql::tokens::{Token, TokenType}; - -use crate::sql_lint::constants::{CLOSE_PARENTHESIS, OPEN_PARENTHESIS}; -use sqlbuild_analysis::sql_tokens::main::canonical_tokens::canonical_tokens; -use sqlbuild_analysis::sql_tokens::main::case_folding::foldable_tokens; -use sqlbuild_analysis::sql_tokens::main::is_unquoted_word::is_unquoted_word; -use sqlbuild_analysis::sql_tokens::main::name_positions::name_positions; -use sqlbuild_analysis::sql_tokens::main::token_texts::token_texts; - -const TOKEN_PRESERVATION_FAILURE: &str = - "native formatter would change authored SQL tokens, not only layout"; -const COMMENT_ATTACHMENT_FAILURE: &str = - "native formatter could not preserve comment token attachments"; -const STRING_BREAK_FAILURE: &str = - "native formatter would join adjacent string literals across a line break"; -const ALIGNMENT_FAILURE: &str = "native formatter could not align its layout with authored tokens"; -const RESYNC_WINDOW: usize = 48; -const RESYNC_CONFIRMATION: usize = 1; -const NO_SPACE_BEFORE: [&str; 7] = [",", ")", "]", ".", ";", "::", ":"]; -const NO_SPACE_AFTER: [&str; 5] = ["(", "[", ".", "::", ":"]; -const STRUCTURAL_PUNCTUATION: [&str; 6] = ["(", ")", "[", "]", ",", ";"]; - -#[derive(Debug, Clone)] -pub(crate) struct Comment { - pub(crate) start: usize, - pub(crate) end: usize, - pub(crate) text: String, - pub(crate) line: bool, - pub(crate) leading: bool, -} - -/// One authored or oracle token with its raw source text and alignment key. -struct Lexeme { - start: usize, - end: usize, - raw: String, - key: LexemeKey, - /// In the reserved-keyword and built-in fold set shared with the change fingerprint. - foldable: bool, - /// An unquoted authored word outside the name positions that keep reserved words exact. - recasable: bool, - /// An unquoted word the dialect tokenizer types as a keyword. - keyword: bool, -} - -/// How lexemes compare: words ignoring ASCII case, punctuation by type, the rest by raw text. -#[derive(Clone, Copy, PartialEq, Eq)] -enum LexemeKey { - Word, - Punctuation(TokenType), - Raw, -} - -impl Lexeme { - fn same_key(&self, other: &Lexeme) -> bool { - self.key == other.key - && match self.key { - LexemeKey::Word => self.raw.eq_ignore_ascii_case(&other.raw), - LexemeKey::Punctuation(_) => true, - LexemeKey::Raw => self.raw == other.raw, - } - } -} - -/// The source text and case-fold flag of every token, read once for the token check. -pub(crate) struct TokenTexts { - texts: Vec, - foldable: Vec, -} - -impl TokenTexts { - #[cfg(test)] - pub(crate) fn read(sql: &str, tokens: &[Token], dialect: &Dialect) -> Result { - let texts = token_texts(sql, tokens)?; - let foldable = foldable_tokens(&texts, tokens, dialect); - Ok(Self { texts, foldable }) - } - - /// Whether token `index` here, upper-cased when foldable, spells `canonical`. - fn same_canonical(&self, index: usize, canonical: &str) -> bool { - let (text, fold) = (&self.texts[index], self.foldable[index]); - text.len() == canonical.len() - && text.bytes().zip(canonical.bytes()).all(|(byte, expected)| { - expected - == if fold { - byte.to_ascii_uppercase() - } else { - byte - } - }) - } -} - -/// An authored token the printer recased to the layout guide's keyword spelling. -pub(crate) struct KeywordRecase { - pub(crate) generated: String, - pub(crate) keyword: bool, -} - -/// Comments attached to the authored token they precede or follow. -#[derive(Default)] -struct Attached<'a> { - leading: Vec<&'a Comment>, - trailing: Vec<&'a Comment>, -} - -pub(crate) fn comments_in(sql: &str, tokens: &[Token]) -> Vec { - let characters: Vec = sql.chars().collect(); - let mut comments: Vec = Vec::new(); - let mut gap_start = 0_usize; - for token in tokens { - comments.extend(comments_in_gap(&characters, gap_start, token.span.start)); - gap_start = token.span.end; - } - comments.extend(comments_in_gap(&characters, gap_start, characters.len())); - comments -} - -/// Split the non-whitespace text the tokenizer skipped between two tokens into comments. -fn comments_in_gap(characters: &[char], start: usize, end: usize) -> Vec { - let mut comments: Vec = Vec::new(); - let mut index = start; - while index < end { - if characters[index].is_whitespace() { - index += 1; - continue; - } - let start = index; - let line_start = (0..start) - .rev() - .find(|&position| characters[position] == '\n') - .map_or(0, |position| position + 1); - let leading = characters[line_start..start] - .iter() - .all(|character| character.is_whitespace()); - let block = characters[index] == '/' && characters.get(index + 1) == Some(&'*'); - index = if block { - block_comment_end(characters, start, end) - } else { - (start..end) - .find(|&position| characters[position] == '\n') - .unwrap_or(end) - }; - comments.push(Comment { - start, - end: index, - text: characters[start..index].iter().collect(), - line: !block, - leading, - }); - } - comments -} - -/// End of a block comment: nested comments when they close inside the gap, else the first `*/`. -fn block_comment_end(characters: &[char], start: usize, end: usize) -> usize { - let closes_at = |position: usize| { - position + 1 < end && characters[position] == '*' && characters[position + 1] == '/' - }; - let opens_at = |position: usize| { - position + 1 < end && characters[position] == '/' && characters[position + 1] == '*' - }; - let mut depth = 0_usize; - let mut position = start; - while position < end { - if opens_at(position) { - depth += 1; - position += 2; - } else if closes_at(position) { - depth -= 1; - position += 2; - if depth == 0 { - return position; - } - } else { - position += 1; - } - } - (start + 2..end) - .find(|&position| closes_at(position)) - .map_or(end, |position| position + 2) -} - -pub(crate) fn neutralize_comments(sql: &str, comments: &[Comment]) -> String { - let mut characters: Vec = sql.chars().collect(); - for comment in comments { - for character in &mut characters[comment.start..comment.end] { - if *character != '\n' && *character != '\r' { - *character = ' '; - } - } - } - characters.into_iter().collect() -} - -/// Authored SQL text with its dialect tokens and the comments between them. -pub(crate) struct AuthoredSql<'a> { - pub(crate) sql: &'a str, - pub(crate) tokens: &'a [Token], - pub(crate) comments: &'a [Comment], -} - -/// The lexemes and characters of a layout oracle, read once for every print against it. -pub(crate) struct OracleLayout { - generated: Vec, - characters: Vec, -} - -/// Tokenize `oracle` for printing against it. -pub(crate) fn oracle_layout(oracle: &str, dialect: &Dialect) -> Result { - let oracle_tokens = dialect - .tokenize(oracle) - .map_err(|error| error.to_string())?; - Ok(OracleLayout { - generated: lexemes(oracle, &oracle_tokens, None)?, - characters: oracle.chars().collect(), - }) -} - -/// Print every authored token in order, taking only whitespace and keyword case from `oracle`. -pub(crate) fn print_authored_tokens( - source: &AuthoredSql<'_>, - oracle: &OracleLayout, - dialect: &Dialect, -) -> Result<(String, BTreeMap, TokenTexts), String> { - let AuthoredSql { - sql, - tokens, - comments, - } = *source; - let authored = lexemes(sql, tokens, Some(dialect))?; - let generated = &oracle.generated; - let alignment = align(&authored, generated)?; - let separators = separators(&authored, generated, &alignment, &oracle.characters)?; - let attached = attach_comments(tokens, comments); - let mut output = String::with_capacity(sql.len() + sql.len() / 4); - let string_breaks = string_breaks(sql, tokens); - let mut recases: BTreeMap = BTreeMap::new(); - for (index, lexeme) in authored.iter().enumerate() { - let forced_break = string_breaks.contains(&index) && !separators[index].contains('\n'); - let forced: String; - let separator: &str = if forced_break { - forced = format!("\n{}", current_indentation(&output)); - &forced - } else { - &separators[index] - }; - let leading = &attached[index].leading; - if leading.is_empty() { - output.push_str(separator); - } else if index == 0 || separator.contains('\n') { - output.push_str(separator); - let indentation = current_indentation(&output); - for comment in leading { - output.push_str(&comment.text); - output.push('\n'); - output.push_str(&indentation); - } - } else { - let indentation = current_indentation(&output); - for comment in leading { - output.push('\n'); - output.push_str(&indentation); - output.push_str(&comment.text); - } - output.push('\n'); - output.push_str(&indentation); - } - let guide = alignment[index].map(|at| &generated[at]); - if let Some(guide) = guide.filter(|guide| confirms_keyword(lexeme, guide)) { - recases.insert( - index, - KeywordRecase { - generated: guide.raw.clone(), - keyword: guide.keyword, - }, - ); - } - output.push_str(&printed_text(lexeme, guide)); - let trailing = &attached[index].trailing; - for (position, comment) in trailing.iter().enumerate() { - output.push(' '); - output.push_str(&comment.text); - if !comment.line { - continue; - } - let next_starts_line = if position + 1 < trailing.len() { - false - } else { - separators - .get(index + 1) - .is_none_or(|next| next.starts_with('\n')) - }; - if !next_starts_line { - output.push('\n'); - if position + 1 < trailing.len() { - output.push_str(¤t_indentation(&output)); - } - } - } - } - let mut texts = TokenTexts { - texts: Vec::with_capacity(authored.len()), - foldable: Vec::with_capacity(authored.len()), - }; - for lexeme in authored { - texts.texts.push(lexeme.raw); - texts.foldable.push(lexeme.foldable); - } - Ok((output, recases, texts)) -} - -/// Refuse output that changes anything but layout, fold-set case, and recorded keyword recases. -pub(crate) fn verify_printed_tokens( - (before, before_tokens, before_texts): (&str, &[Token], &TokenTexts), - (after, after_tokens): (&str, &[Token]), - dialect: &Dialect, - recases: &BTreeMap, -) -> Result<(), String> { - let after_canonical = canonical_tokens(after, after_tokens, dialect)?; - if before_texts.texts.len() != after_canonical.len() { - return Err(TOKEN_PRESERVATION_FAILURE.to_string()); - } - let mut after_texts: Option> = None; - for (index, canonical) in after_canonical.iter().enumerate() { - if before_texts.same_canonical(index, &canonical.text) { - continue; - } - let Some(recase) = recases.get(&index) else { - return Err(TOKEN_PRESERVATION_FAILURE.to_string()); - }; - let texts: &Vec = match after_texts { - Some(ref texts) => texts, - None => after_texts.insert(token_texts(after, after_tokens)?), - }; - let printed: &str = &texts[index]; - if !recase.allows(&before_texts.texts[index], printed) { - return Err(TOKEN_PRESERVATION_FAILURE.to_string()); - } - } - if comment_positions(before, before_tokens) != comment_positions(after, after_tokens) { - return Err(COMMENT_ATTACHMENT_FAILURE.to_string()); - } - if !string_breaks(before, before_tokens).is_subset(&string_breaks(after, after_tokens)) { - return Err(STRING_BREAK_FAILURE.to_string()); - } - Ok(()) -} - -impl KeywordRecase { - /// Whether `authored` became `printed` only by taking the guide's upper-case keyword spelling. - fn allows(&self, authored: &str, printed: &str) -> bool { - self.keyword - && printed == self.generated - && printed == printed.to_ascii_uppercase() - && authored.eq_ignore_ascii_case(printed) - } -} - -pub(crate) fn tokenize(sql: &str, dialect: &Dialect) -> Result, String> { - dialect.tokenize(sql).map_err(|error| error.to_string()) -} - -fn comment_positions(sql: &str, tokens: &[Token]) -> Vec<(String, usize)> { - let mut positions: Vec<(String, usize)> = Vec::new(); - for comment in comments_in(sql, tokens) { - let preceding = tokens.partition_point(|token| token.span.end <= comment.start); - positions.push((comment.text, preceding)); - } - positions -} - -/// Lexemes of `sql`; only authored lexemes carry the recase classification. -fn lexemes(sql: &str, tokens: &[Token], dialect: Option<&Dialect>) -> Result, String> { - let raws = token_texts(sql, tokens)?; - let (foldable, recasable) = match dialect { - Some(dialect) => { - let uppers: Vec = raws.iter().map(|raw| raw.to_ascii_uppercase()).collect(); - let names = name_positions(&uppers); - let foldable = foldable_tokens(&raws, tokens, dialect); - let recasable: Vec = raws - .iter() - .zip(names) - .map(|(raw, name)| !name && is_unquoted_word(raw)) - .collect(); - (foldable, recasable) - } - None => (vec![false; tokens.len()], vec![false; tokens.len()]), - }; - let mut lexemes = Vec::with_capacity(tokens.len()); - for (index, (raw, token)) in raws.into_iter().zip(tokens).enumerate() { - lexemes.push(lexeme(raw, token, foldable[index], recasable[index])); - } - Ok(lexemes) -} - -fn lexeme(raw: String, token: &Token, foldable: bool, recasable: bool) -> Lexeme { - let word = is_unquoted_word(&raw); - let key = if word { - LexemeKey::Word - } else if raw - .chars() - .all(|character| character.is_ascii_punctuation()) - { - LexemeKey::Punctuation(token.token_type) - } else { - LexemeKey::Raw - }; - let keyword = word - && !matches!( - token.token_type, - TokenType::Var | TokenType::Identifier | TokenType::QuotedIdentifier - ); - Lexeme { - start: token.span.start, - end: token.span.end, - raw, - key, - foldable, - recasable, - keyword, - } -} - -fn is_string_literal(token: &Token) -> bool { - matches!( - token.token_type, - TokenType::String - | TokenType::NationalString - | TokenType::EscapeString - | TokenType::RawString - | TokenType::ByteString - | TokenType::UnicodeString - | TokenType::HexString - | TokenType::BitString - | TokenType::DollarString - | TokenType::TripleSingleQuotedString - | TokenType::TripleDoubleQuotedString - ) -} - -/// Indices of string literals that follow another string literal across a line break. -fn string_breaks(sql: &str, tokens: &[Token]) -> BTreeSet { - let characters: Vec = sql.chars().collect(); - (1..tokens.len()) - .filter(|&index| { - is_string_literal(&tokens[index - 1]) - && is_string_literal(&tokens[index]) - && characters[tokens[index - 1].span.end..tokens[index].span.start].contains(&'\n') - }) - .collect() -} - -/// Map authored tokens to matching oracle tokens; a closer only matches its opener's partner. -fn align(authored: &[Lexeme], generated: &[Lexeme]) -> Result>, String> { - let mut state = Alignment { - authored, - generated, - authored_partners: bracket_partners(authored), - generated_partners: bracket_partners(generated), - forward: vec![None; authored.len()], - backward: vec![None; generated.len()], - }; - let (mut left, mut right) = (0_usize, 0_usize); - while left < authored.len() && right < generated.len() { - if state.matches(left, right) { - state.forward[left] = Some(right); - state.backward[right] = Some(left); - left += 1; - right += 1; - continue; - } - let (skip_left, skip_right) = state - .resync(left, right) - .ok_or_else(|| ALIGNMENT_FAILURE.to_string())?; - left += skip_left; - right += skip_right; - } - Ok(state.forward) -} - -struct Alignment<'a> { - authored: &'a [Lexeme], - generated: &'a [Lexeme], - authored_partners: Vec>, - generated_partners: Vec>, - forward: Vec>, - backward: Vec>, -} - -impl Alignment<'_> { - fn matches(&self, left: usize, right: usize) -> bool { - if !self.authored[left].same_key(&self.generated[right]) { - return false; - } - if !is_closer(&self.authored[left].raw) { - return true; - } - match (self.authored_partners[left], self.generated_partners[right]) { - (Some(opener), Some(generated_opener)) => match self.forward[opener] { - Some(aligned) => aligned == generated_opener, - None => self.backward[generated_opener].is_none(), - }, - (None, None) => true, - _ => false, - } - } - - fn resync(&self, left: usize, right: usize) -> Option<(usize, usize)> { - let remaining_left = self.authored.len() - left; - let remaining_right = self.generated.len() - right; - for cost in 1..=(2 * RESYNC_WINDOW) { - for skip_left in 0..=cost.min(RESYNC_WINDOW) { - let skip_right = cost - skip_left; - if skip_right > RESYNC_WINDOW { - continue; - } - if skip_left >= remaining_left || skip_right >= remaining_right { - if skip_left >= remaining_left && skip_right >= remaining_right { - return Some((remaining_left, remaining_right)); - } - continue; - } - if !self.matches(left + skip_left, right + skip_right) { - continue; - } - let anchor = &self.authored[left + skip_left]; - let confirmations = if matches!(anchor.key, LexemeKey::Punctuation(_)) { - RESYNC_CONFIRMATION - } else { - 0 - }; - let confirmed = (1..=confirmations).all(|offset| { - match ( - self.authored.get(left + skip_left + offset), - self.generated.get(right + skip_right + offset), - ) { - (Some(before), Some(after)) => before.same_key(after), - _ => true, - } - }); - if confirmed { - return Some((skip_left, skip_right)); - } - } - } - None - } -} - -fn is_closer(raw: &str) -> bool { - matches!(raw, ")" | "]" | "}") -} - -fn bracket_partners(lexemes: &[Lexeme]) -> Vec> { - let mut partners: Vec> = vec![None; lexemes.len()]; - let mut open: Vec = Vec::new(); - for (index, lexeme) in lexemes.iter().enumerate() { - match lexeme.raw.as_str() { - "(" | "[" | "{" => open.push(index), - ")" | "]" | "}" => { - if let Some(opener) = open.pop() { - partners[opener] = Some(index); - partners[index] = Some(opener); - } - } - _ => {} - } - } - partners -} - -fn separators( - authored: &[Lexeme], - generated: &[Lexeme], - alignment: &[Option], - oracle: &[char], -) -> Result, String> { - let mut separators: Vec = vec![String::new(); authored.len()]; - let anchors: Vec = (0..authored.len()) - .filter(|&index| alignment[index].is_some()) - .collect(); - if anchors.is_empty() { - return Err(ALIGNMENT_FAILURE.to_string()); - } - let gap = |left: usize, right: usize| -> String { - oracle[generated[left].end..generated[right].start] - .iter() - .collect() - }; - for index in 1..anchors[0] { - separators[index] = default_space(&authored[index - 1], &authored[index]).to_string(); - } - if anchors[0] > 0 { - separators[anchors[0]] = - default_space(&authored[anchors[0] - 1], &authored[anchors[0]]).to_string(); - } - for pair in anchors.windows(2) { - let (previous, next) = (pair[0], pair[1]); - let (previous_at, next_at) = ( - alignment[previous].ok_or(ALIGNMENT_FAILURE)?, - alignment[next].ok_or(ALIGNMENT_FAILURE)?, - ); - let merged = merged_gap( - &(previous_at..next_at) - .map(|at| gap(at, at + 1)) - .collect::>(), - ); - if previous + 1 == next { - separators[next] = merged; - continue; - } - for index in previous + 2..next { - separators[index] = default_space(&authored[index - 1], &authored[index]).to_string(); - } - let first = previous + 1; - let last = next - 1; - let substitution = next_at > previous_at + 1; - separators[first] = if default_space(&authored[previous], &authored[first]).is_empty() { - String::new() - } else if substitution { - gap(previous_at, previous_at + 1) - } else { - " ".to_string() - }; - separators[next] = if substitution { - let oracle_gap = (previous_at + 1..next_at) - .rev() - .find(|&at| generated[at].same_key(&authored[last])) - .map_or_else(|| gap(next_at - 1, next_at), |at| gap(at, at + 1)); - let unspaced = default_space(&authored[last], &authored[next]).is_empty(); - if unspaced && !oracle_gap.contains('\n') { - String::new() - } else { - oracle_gap - } - } else if merged.contains('\n') { - merged - } else { - default_space(&authored[last], &authored[next]).to_string() - }; - } - let last_anchor = anchors[anchors.len() - 1]; - for index in last_anchor + 1..authored.len() { - separators[index] = default_space(&authored[index - 1], &authored[index]).to_string(); - } - let last_at = alignment[last_anchor].ok_or(ALIGNMENT_FAILURE)?; - if last_anchor + 1 < authored.len() - && last_at + 1 < generated.len() - && !separators[last_anchor + 1].is_empty() - { - separators[last_anchor + 1] = gap(last_at, last_at + 1); - } - for index in 1..authored.len() { - let adjacent_in_source = authored[index - 1].end == authored[index].start; - let oracle_pair = matches!( - (alignment[index - 1], alignment[index]), - (Some(previous), Some(next)) if previous + 1 == next - ); - let structural = [&authored[index - 1], &authored[index]] - .iter() - .any(|lexeme| STRUCTURAL_PUNCTUATION.contains(&lexeme.raw.as_str())); - let call = - is_unquoted_word(&authored[index - 1].raw) && authored[index].raw == OPEN_PARENTHESIS; - let inside_parentheses = - authored[index - 1].raw == OPEN_PARENTHESIS || authored[index].raw == CLOSE_PARENTHESIS; - if (adjacent_in_source && !oracle_pair && (!structural || call)) - || (inside_parentheses && !separators[index].contains('\n')) - { - separators[index].clear(); - } else if separators[index].is_empty() - && !adjacent_in_source - && would_fuse(&authored[index - 1], &authored[index]) - { - separators[index] = " ".to_string(); - } - } - Ok(separators) -} - -fn merged_gap(gaps: &[String]) -> String { - if let [only] = gaps { - return only.clone(); - } - if let Some(newline) = gaps.iter().rev().find(|gap| gap.contains('\n')) { - return newline.clone(); - } - if gaps.iter().any(|gap| !gap.is_empty()) { - " ".to_string() - } else { - String::new() - } -} - -fn default_space(previous: &Lexeme, next: &Lexeme) -> &'static str { - if NO_SPACE_BEFORE.contains(&next.raw.as_str()) - || NO_SPACE_AFTER.contains(&previous.raw.as_str()) - { - "" - } else { - " " - } -} - -/// Return whether printing two separated tokens without a space could lex them differently. -fn would_fuse(previous: &Lexeme, next: &Lexeme) -> bool { - let (Some(left), Some(right)) = (previous.raw.chars().last(), next.raw.chars().next()) else { - return false; - }; - let word = |character: char| { - character.is_alphanumeric() || matches!(character, '_' | '$' | '\'' | '"' | '`' | '@') - }; - let operator = |character: char| "+-*/<>=!|&^%~:#?".contains(character); - (word(left) && word(right)) || (operator(left) && operator(right)) -} - -/// The authored text, recased only for keywords and built-in function names. -fn printed_text(authored: &Lexeme, generated: Option<&Lexeme>) -> String { - if let Some(generated) = generated.filter(|generated| confirms_keyword(authored, generated)) { - return generated.raw.clone(); - } - if !authored.foldable { - return authored.raw.clone(); - } - match generated { - Some(generated) if generated.raw.eq_ignore_ascii_case(&authored.raw) => { - generated.raw.clone() - } - Some(_) => authored.raw.clone(), - None => authored.raw.to_ascii_uppercase(), - } -} - -/// Whether the layout guide prints the aligned authored word as an upper-case keyword. -fn confirms_keyword(authored: &Lexeme, generated: &Lexeme) -> bool { - !authored.foldable - && authored.recasable - && generated.keyword - && generated.raw != authored.raw - && generated.raw == generated.raw.to_ascii_uppercase() - && generated.raw.eq_ignore_ascii_case(&authored.raw) -} - -fn attach_comments<'a>(tokens: &[Token], comments: &'a [Comment]) -> Vec> { - let mut attached: Vec> = (0..tokens.len()).map(|_| Attached::default()).collect(); - for comment in comments { - let next = tokens.partition_point(|token| token.span.start < comment.end); - let preceding = tokens.partition_point(|token| token.span.end <= comment.start); - let next = (next < tokens.len()).then_some(next); - let previous = preceding.checked_sub(1); - match (comment.leading, next, previous) { - (true, Some(next), _) | (false, Some(next), None) => { - attached[next].leading.push(comment); - } - (_, _, Some(previous)) => attached[previous].trailing.push(comment), - (_, None, None) => {} - } - } - attached -} - -fn current_indentation(output: &str) -> String { - let line = output.rsplit('\n').next().unwrap_or_default(); - line.chars() - .take_while(|character| *character == ' ' || *character == '\t') - .collect() -} diff --git a/crates/sqlbuild-rules/src/sql_lint/constants.rs b/crates/sqlbuild-rules/src/sql_lint/constants.rs deleted file mode 100644 index 8bec6a6bd0..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/constants.rs +++ /dev/null @@ -1,61 +0,0 @@ -pub(crate) const CAST_TYPE_SEPARATOR_KEYWORD: &str = "AS"; -pub(crate) const CLOSE_PARENTHESIS: &str = ")"; -pub(crate) const LINT_API_VERSION: u32 = 1; -pub(crate) const OPEN_PARENTHESIS: &str = "("; -pub(crate) const POSITION_ARGUMENT_SEPARATOR: &str = ","; -pub(crate) const CTE_MACRO_PLACEHOLDER_LITERAL: &str = "1"; -pub(crate) const POSTFIX_CAST_PREFIX_TOKEN_COUNT: usize = 2; -pub(crate) const RELATION_MODIFIER_KEYWORDS: [&str; 43] = [ - "ANTI", - "APPLY", - "ASOF", - "AT", - "BEFORE", - "CHANGES", - "CONNECT", - "CROSS", - "EXCEPT", - "FETCH", - "FINAL", - "FOR", - "FULL", - "GROUP", - "HAVING", - "INNER", - "INTERSECT", - "JOIN", - "LATERAL", - "LEFT", - "LIMIT", - "MATCH_RECOGNIZE", - "MINUS", - "NATURAL", - "OFFSET", - "ON", - "ORDER", - "OUTER", - "PASTE", - "PIVOT", - "POSITIONAL", - "QUALIFY", - "RETURNING", - "RIGHT", - "SAMPLE", - "SEMI", - "START", - "TABLESAMPLE", - "UNION", - "UNPIVOT", - "USING", - "WHERE", - "WINDOW", -]; -/// The parts of a `SQBRSQL038` message that names its joined relation and CTE. -pub(crate) const CROSS_JOIN_PREFIX: &str = "CROSS JOIN with `"; -pub(crate) const CROSS_JOIN_CTE_MARKER: &str = "` in CTE `"; -pub(crate) const CROSS_JOIN_SUFFIX: &str = "` creates an explicit cartesian product"; -/// The `SQBRSQL038` message for a join whose relation cannot be named. -pub(crate) const CROSS_JOIN_UNNAMED_MESSAGE: &str = "CROSS JOIN creates an explicit cartesian product with an unnamed relation; only a file-wide exception (rule and path) can cover it"; -/// Stands for the finding's project-relative path until the caller that knows it fills it in. -pub(crate) const CROSS_JOIN_PATH_PLACEHOLDER: &str = ""; -pub(crate) const CROSS_JOIN_REASON_PLACEHOLDER: &str = ""; diff --git a/crates/sqlbuild-rules/src/sql_lint/main/batch_engine.rs b/crates/sqlbuild-rules/src/sql_lint/main/batch_engine.rs deleted file mode 100644 index 2d7de066ba..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/main/batch_engine.rs +++ /dev/null @@ -1,19 +0,0 @@ -use crate::sql_lint::_helpers::engine::lint; -use crate::sql_lint::models::{LintRequest, LintResponse}; -use rayon::iter::{IntoParallelIterator, ParallelIterator}; - -pub fn lint_batch_json(request_json: &str) -> Result { - let requests: Vec = - serde_json::from_str(request_json).map_err(|error| error.to_string())?; - if requests.is_empty() { - return Ok("[]".to_owned()); - } - let pool = rayon::ThreadPoolBuilder::new() - .num_threads(requests.len().min(4)) - .stack_size(16 * 1024 * 1024) - .build() - .map_err(|error| error.to_string())?; - let results: Vec> = - pool.install(|| requests.into_par_iter().map(lint).collect()); - serde_json::to_string(&results).map_err(|error| error.to_string()) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/main/catalogue.rs b/crates/sqlbuild-rules/src/sql_lint/main/catalogue.rs deleted file mode 100644 index c8789f765a..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/main/catalogue.rs +++ /dev/null @@ -1,6 +0,0 @@ -use crate::sql_lint::_helpers::engine::rule_metadata; -use crate::sql_lint::models::LintRuleMetadata; - -pub(crate) fn catalogue() -> Vec { - rule_metadata() -} diff --git a/crates/sqlbuild-rules/src/sql_lint/main/cross_join_subject.rs b/crates/sqlbuild-rules/src/sql_lint/main/cross_join_subject.rs deleted file mode 100644 index c9a03220dd..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/main/cross_join_subject.rs +++ /dev/null @@ -1,15 +0,0 @@ -//! The relation and CTE a `SQBRSQL038` finding names, for relation-level Rule exceptions. - -use crate::sql_lint::constants::{CROSS_JOIN_CTE_MARKER, CROSS_JOIN_PREFIX, CROSS_JOIN_SUFFIX}; - -/// The `(relation, cte)` a cross join finding message names, `None` for the generic message. -#[must_use] -pub fn cross_join_subject(message: &str) -> Option<(String, Option)> { - let named: &str = message - .strip_prefix(CROSS_JOIN_PREFIX)? - .strip_suffix(CROSS_JOIN_SUFFIX)?; - match named.split_once(CROSS_JOIN_CTE_MARKER) { - Some((relation, cte)) => Some((relation.to_owned(), Some(cte.to_owned()))), - None => Some((named.to_owned(), None)), - } -} diff --git a/crates/sqlbuild-rules/src/sql_lint/main/engine.rs b/crates/sqlbuild-rules/src/sql_lint/main/engine.rs deleted file mode 100644 index 2533f4f949..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/main/engine.rs +++ /dev/null @@ -1,5 +0,0 @@ -use crate::sql_lint::_helpers::engine::lint_json_impl; - -pub fn lint_json(request_json: &str) -> Result { - lint_json_impl(request_json) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/main/fill_finding_path.rs b/crates/sqlbuild-rules/src/sql_lint/main/fill_finding_path.rs deleted file mode 100644 index d5a65801ce..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/main/fill_finding_path.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! `fill_finding_path`: a SQL finding's help with its file's path written into the entry it prints. - -use crate::sql_lint::constants::CROSS_JOIN_PATH_PLACEHOLDER; - -/// `remediation` with the path placeholder of a printed exception entry replaced by `path`. -#[must_use] -pub fn fill_finding_path(remediation: &str, path: &str) -> String { - remediation.replace(CROSS_JOIN_PATH_PLACEHOLDER, path) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/main/format_requests.rs b/crates/sqlbuild-rules/src/sql_lint/main/format_requests.rs deleted file mode 100644 index 6794204ce3..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/main/format_requests.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! Typed, in-process batch formatting of neutralized SQL bodies. - -use crate::sql_lint::models::{FormatRequest, FormatResponse}; - -/// Format every request in parallel and return the responses in request order. -pub(crate) fn format_requests(requests: Vec) -> Vec { - crate::sql_lint::_helpers::formatter::format_requests(requests) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/main/formatter.rs b/crates/sqlbuild-rules/src/sql_lint/main/formatter.rs deleted file mode 100644 index 42da9c9c03..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/main/formatter.rs +++ /dev/null @@ -1,5 +0,0 @@ -use crate::sql_lint::_helpers::formatter::format_json_impl; - -pub fn format_json(request_json: &str) -> Result { - format_json_impl(request_json) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/main/mod.rs b/crates/sqlbuild-rules/src/sql_lint/main/mod.rs deleted file mode 100644 index 7fbbc4fae5..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/main/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -pub mod backtick_identifiers; -pub mod batch_engine; -pub mod catalogue; -pub mod cross_join_subject; -pub mod defined_cte_names; -pub mod engine; -pub mod fill_finding_path; -pub mod format_requests; -pub mod formatter; -pub mod preparation; diff --git a/crates/sqlbuild-rules/src/sql_lint/mod.rs b/crates/sqlbuild-rules/src/sql_lint/mod.rs deleted file mode 100644 index 1b51a640b0..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! SQLBuild-owned generic SQL linting over the external Polyglot parser. - -pub mod _helpers; -mod constants; -pub mod main; -pub mod models; -pub mod types; - -#[cfg(test)] -mod tests; diff --git a/crates/sqlbuild-rules/src/sql_lint/models.rs b/crates/sqlbuild-rules/src/sql_lint/models.rs deleted file mode 100644 index 59ea49ceee..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/models.rs +++ /dev/null @@ -1,177 +0,0 @@ -use std::borrow::Cow; -use std::collections::{HashMap, HashSet}; - -use polyglot_sql::Expression; -use polyglot_sql::tokens::{Span, Token}; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Default)] -pub(crate) struct QueryFacts { - pub null_comparisons: Vec, - pub implicit_cartesian_joins: Vec, - pub joins_without_condition: Vec, - pub unordered_limits: Vec, - pub redundant_distincts: Vec, - pub positional_set_stars: Vec, - pub unused_cte_names: Vec, - pub cross_joins: Vec, - pub cte_only_bodies: Vec, - pub terminal_selects: Vec, - pub nested_ctes: Vec, - pub recursive_ctes: Vec, - pub invalid_comment_attachments: Vec, - pub inline_query_relations: Vec, - pub additional: AdditionalQueryFacts, -} - -#[derive(Debug, Default)] -pub(crate) struct AdditionalQueryFacts { - pub bare_unions: Vec, - pub duplicate_table_aliases: Vec, - pub duplicate_output_aliases: Vec, - pub mixed_group_order_references: Vec, - pub redundant_else_nulls: Vec, - pub parenthesized_distincts: Vec, - pub constant_predicates: Vec, - pub consecutive_semicolons: Vec, - pub redundant_self_aliases: Vec, - pub count_one_literals: Vec, - pub unstable_row_numbers: Vec, - pub null_not_in_predicates: Vec, - pub set_arity_mismatches: Vec, - pub projected_stars: Vec, - pub unaliased_calculations: Vec, - pub unused_table_aliases: Vec, - pub null_rejected_left_joins: Vec, - pub implicit_inner_joins: Vec, - pub ambiguous_order_directions: Vec, - pub unqualified_multi_source_columns: Vec, - pub inconsistent_single_source_qualification: Vec, - pub unknown_relation_qualifiers: Vec, - pub simple_boolean_cases: Vec, - pub nested_else_cases: Vec, - pub unused_joined_relations: Vec, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct LintRequest { - pub version: u32, - pub sql: String, - pub dialect: String, - #[serde(default)] - pub enabled_rules: Option>, - #[serde(default)] - pub ignored_rules: Vec, - #[serde(default)] - pub external_identifiers: Vec, - #[serde(default)] - pub dependency_identifiers: Vec, - #[serde(default)] - pub externally_referenced_ctes: Vec, - #[serde(default)] - pub allows_ceremonial_select: bool, - #[serde(default)] - pub allows_dynamic_output_star: bool, - #[serde(default)] - pub allows_empty_fixture_star: bool, - #[serde(default)] - pub max_literal_length: Option, - #[serde(default)] - pub max_ranking_order_by: Option, - #[serde(default)] - pub relation_keys: HashMap>>, - #[serde(default)] - pub relation_names: HashMap, - /// The SQL each `@sql.` call in the query generated, so column qualification is judged as the macro wrote it. - #[serde(default)] - pub sql_macro_outputs: Vec, -} - -/// The character range one `@sql.` call's output fills, and the names its arguments pass, which a plain slot read such as `@rel.amount` expands to qualify with. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)] -pub(crate) struct SqlMacroOutput { - pub start: usize, - pub end: usize, - #[serde(default)] - pub relations: Vec, -} - -#[derive(Debug)] -pub(crate) struct AdditionalFactOptions<'a> { - pub external_identifiers: &'a HashSet, - pub dependency_identifiers: &'a HashSet, - pub sql_macro_outputs: &'a [SqlMacroOutput], - pub allows_ceremonial_select: bool, - pub allows_dynamic_output_star: bool, - pub allows_empty_fixture_star: bool, -} - -#[derive(Debug, Serialize)] -pub(crate) struct LintResponse { - pub version: u32, - pub diagnostics: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct LintDiagnostic { - pub code: &'static str, - pub message: Cow<'static, str>, - pub remediation: Cow<'static, str>, - pub start: usize, - pub end: usize, - #[serde(skip_serializing_if = "Option::is_none")] - pub fix: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub fix_unavailable_reason: Option<&'static str>, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct LintEdit { - pub start: usize, - pub end: usize, - pub replacement: String, -} - -#[derive(Debug, Clone, Copy)] -pub(crate) struct LintRuleMetadata { - pub code: &'static str, - pub message: &'static str, - pub remediation: &'static str, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct FormatRequest { - pub version: u32, - pub sql: String, - pub dialect: String, - #[serde(default)] - pub max_function_call_depth: Option, - #[serde(default)] - pub line_width: Option, - #[serde(default)] - pub token_widths: HashMap, -} - -#[derive(Debug, Serialize)] -pub(crate) struct FormatResponse { - pub version: u32, - pub sql: String, - pub changed: bool, - pub formatted: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub reason: Option, -} - -/// One lint request's explicit cross joins, with what is known about the relations they read. -#[derive(Debug)] -pub(crate) struct CrossJoinRequest<'a> { - pub(crate) statements: &'a [Expression], - pub(crate) tokens: &'a [Token], - pub(crate) cross_spans: &'a [Span], - /// Declared keys by dependency sentinel; the empty key proves at most one row. - pub(crate) relation_keys: &'a HashMap>>, - /// The relation name each dependency sentinel stands for. - pub(crate) relation_names: &'a HashMap, -} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests.rs b/crates/sqlbuild-rules/src/sql_lint/tests.rs deleted file mode 100644 index dde04c7550..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/tests.rs +++ /dev/null @@ -1,34 +0,0 @@ -#[path = "tests/test_authored_tokens.rs"] -mod authored_tokens; -#[path = "tests/test_batch_lint.rs"] -mod batch_lint; -#[path = "tests/test_comment_formatter.rs"] -mod comment_formatter; -#[path = "tests/test_cross_joins.rs"] -mod cross_joins; -#[path = "tests/test_empty_fixture_lint.rs"] -mod empty_fixture_lint; -#[path = "tests/test_format_corpus.rs"] -mod format_corpus; -#[path = "tests/test_function_call_spacing.rs"] -mod function_call_spacing; -#[path = "tests/helpers.rs"] -mod helpers; -#[path = "tests/test_idempotency_reprint.rs"] -mod idempotency_reprint; -#[path = "tests/test_layout_rules.rs"] -mod layout_rules; -#[path = "tests/test_line_wrap.rs"] -mod line_wrap; -#[path = "tests/test_lint_exemptions.rs"] -mod lint_exemptions; -#[path = "tests/test_set_operation_formatter.rs"] -mod set_operation_formatter; -#[path = "tests/test_sql_lint.rs"] -mod sql_lint; -#[path = "tests/test_types.rs"] -mod test_types; -#[path = "tests/test_unused_table_alias.rs"] -mod unused_table_alias; -#[path = "tests/test_values_formatter.rs"] -mod values_formatter; diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/fixtures/FORMAT_CORPUS_NOTICE.md b/crates/sqlbuild-rules/src/sql_lint/tests/fixtures/FORMAT_CORPUS_NOTICE.md deleted file mode 100644 index 39e0868f78..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/tests/fixtures/FORMAT_CORPUS_NOTICE.md +++ /dev/null @@ -1,32 +0,0 @@ -# Formatter corpus notice - -`format_corpus.json` contains SELECT and WITH queries taken from the SQLGlot dialect identity test -fixtures for BigQuery, Databricks, DuckDB, PostgreSQL, Snowflake and T-SQL, as extracted by the -Polyglot project. One literal was reworded. The expected `formatted` and `previously_formatted` -values are SQLBuild formatter output. - -The queries are used under the following license: - -```text -MIT License - -Copyright (c) 2026 Toby Mao - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -``` diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/fixtures/format_corpus.json b/crates/sqlbuild-rules/src/sql_lint/tests/fixtures/format_corpus.json deleted file mode 100644 index b45dd71a3f..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/tests/fixtures/format_corpus.json +++ /dev/null @@ -1,5354 +0,0 @@ -[ - { - "dialect": "bigquery", - "sql": "SELECT 'foo' 'bar'", - "formatted": "SELECT 'foo' 'bar'" - }, - { - "dialect": "bigquery", - "sql": "SELECT 'foo'/* c */'bar'", - "formatted": "SELECT\n 'foo' /* c */ 'bar'" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM `domain.com:project-id.mydataset.mytable`", - "formatted": "SELECT *\nFROM `domain.com:project-id.mydataset.mytable`" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM `domain.com:project-id.region-us.INFORMATION_SCHEMA`.JOBS", - "formatted": "SELECT *\nFROM `domain.com:project-id.region-us.INFORMATION_SCHEMA`.JOBS" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM `domain.com:project-id.region-us.INFORMATION_SCHEMA.JOBS`", - "formatted": "SELECT *\nFROM `domain.com:project-id.region-us.INFORMATION_SCHEMA.JOBS`" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM `a.b.com:project-id.mydataset.mytable`", - "formatted": "SELECT *\nFROM `a.b.com:project-id.mydataset.mytable`" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM `a.b.com:project-id.region-us.INFORMATION_SCHEMA.JOBS`", - "formatted": "SELECT *\nFROM `a.b.com:project-id.region-us.INFORMATION_SCHEMA.JOBS`" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM x-0.y", - "formatted": "SELECT *\nFROM x-0.y" - }, - { - "dialect": "bigquery", - "sql": "SELECT `p.d.UdF`(data) FROM `p.d.t`", - "formatted": "SELECT `p.d.UdF`(data)\nFROM `p.d.t`" - }, - { - "dialect": "bigquery", - "sql": "SELECT EXP(1)", - "formatted": "SELECT EXP(1)" - }, - { - "dialect": "bigquery", - "sql": "SELECT ARRAY_CONCAT([1])", - "formatted": "SELECT ARRAY_CONCAT([1])" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM READ_CSV('bla.csv')", - "formatted": "SELECT *\nFROM READ_CSV('bla.csv')" - }, - { - "dialect": "bigquery", - "sql": "SELECT jsondoc['some_key']", - "formatted": "SELECT jsondoc['some_key']" - }, - { - "dialect": "bigquery", - "sql": "SELECT `p.d.UdF`(data).* FROM `p.d.t`", - "formatted": "SELECT `p.d.UdF`(data).*\nFROM `p.d.t`" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM `my-project.my-dataset.my-table`", - "formatted": "SELECT *\nFROM `my-project.my-dataset.my-table`" - }, - { - "dialect": "bigquery", - "sql": "SELECT x, 1 AS y GROUP BY 1 ORDER BY 1", - "formatted": "SELECT\n x,\n 1 AS y\nGROUP BY\n 1\nORDER BY\n 1" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM x.*", - "formatted": "SELECT *\nFROM x.*" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM x.y*", - "formatted": "SELECT *\nFROM x.y*" - }, - { - "dialect": "bigquery", - "sql": "SELECT SEARCH(data_to_search, 'search_query')", - "formatted": "SELECT SEARCH(data_to_search, 'search_query')" - }, - { - "dialect": "bigquery", - "sql": "SELECT SEARCH(data_to_search, 'search_query', json_scope => 'JSON_KEYS_AND_VALUES')", - "formatted": "SELECT SEARCH(data_to_search, 'search_query', json_scope => 'JSON_KEYS_AND_VALUES')" - }, - { - "dialect": "bigquery", - "sql": "SELECT SEARCH(data_to_search, 'search_query', analyzer => 'PATTERN_ANALYZER')", - "formatted": "SELECT SEARCH(data_to_search, 'search_query', analyzer => 'PATTERN_ANALYZER')" - }, - { - "dialect": "bigquery", - "sql": "SELECT SEARCH(data_to_search, 'search_query', analyzer_options => 'analyzer_options_values')", - "formatted": "SELECT SEARCH(data_to_search, 'search_query', analyzer_options => 'analyzer_options_values')" - }, - { - "dialect": "bigquery", - "sql": "SELECT SEARCH(data_to_search, 'search_query', json_scope => 'JSON_VALUES', analyzer => 'LOG_ANALYZER')", - "formatted": "SELECT\n SEARCH(\n data_to_search,\n 'search_query',\n json_scope => 'JSON_VALUES',\n analyzer => 'LOG_ANALYZER'\n )" - }, - { - "dialect": "bigquery", - "sql": "SELECT SEARCH(data_to_search, 'search_query', analyzer => 'PATTERN_ANALYZER', analyzer_options => 'options')", - "formatted": "SELECT\n SEARCH(\n data_to_search,\n 'search_query',\n analyzer => 'PATTERN_ANALYZER',\n analyzer_options => 'options'\n )" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM dataset.my_table TABLESAMPLE SYSTEM (10 PERCENT)", - "formatted": "SELECT *\nFROM dataset.my_table TABLESAMPLE SYSTEM (10 PERCENT)" - }, - { - "dialect": "bigquery", - "sql": "SELECT '\\n\\r\\a\\v\\f\\t'", - "formatted": "SELECT '\\n\\r\\a\\v\\f\\t'" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM tbl FOR SYSTEM_TIME AS OF z", - "formatted": "SELECT *\nFROM tbl FOR SYSTEM_TIME AS OF z" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM tbl FOR SYSTEM TIME AS OF z", - "formatted": "SELECT *\nFROM tbl FOR SYSTEM TIME AS OF z" - }, - { - "dialect": "bigquery", - "sql": "SELECT PARSE_TIMESTAMP('%c', 'Thu Dec 25 07:30:00 2008', 'UTC')", - "formatted": "SELECT PARSE_TIMESTAMP('%c', 'Thu Dec 25 07:30:00 2008', 'UTC')" - }, - { - "dialect": "bigquery", - "sql": "SELECT ANY_VALUE(fruit HAVING MAX sold) FROM fruits", - "formatted": "SELECT ANY_VALUE(fruit HAVING MAX sold)\nFROM fruits" - }, - { - "dialect": "bigquery", - "sql": "SELECT ANY_VALUE(fruit HAVING MIN sold) FROM fruits", - "formatted": "SELECT ANY_VALUE(fruit HAVING MIN sold)\nFROM fruits" - }, - { - "dialect": "bigquery", - "sql": "SELECT `project-id`.udfs.func(call.dir)", - "formatted": "SELECT `project-id`.udfs.func(call.dir)" - }, - { - "dialect": "bigquery", - "sql": "SELECT CAST(CURRENT_DATE AS STRING FORMAT 'DAY') AS current_day", - "formatted": "SELECT CAST(CURRENT_DATE AS STRING FORMAT 'DAY') AS current_day" - }, - { - "dialect": "bigquery", - "sql": "SELECT foo IN UNNEST(bar) AS bla", - "formatted": "SELECT foo IN UNNEST(bar) AS bla" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM x-0.a", - "formatted": "SELECT *\nFROM x-0.a" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM pivot CROSS JOIN foo", - "formatted": "SELECT *\nFROM pivot\nCROSS JOIN foo" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM a-b-c.mydataset.mytable", - "formatted": "SELECT *\nFROM a-b-c.mydataset.mytable" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM abc-def-ghi", - "formatted": "SELECT *\nFROM abc-def-ghi" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM a-b-c", - "formatted": "SELECT *\nFROM a-b-c" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM my-table", - "formatted": "SELECT *\nFROM my-table" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM my-project.mydataset.mytable", - "formatted": "SELECT *\nFROM my-project.mydataset.mytable" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM pro-ject_id.c.d CROSS JOIN foo-bar", - "formatted": "SELECT *\nFROM pro-ject_id.c.d\nCROSS JOIN foo-bar" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM foo.bar.25", - "formatted": "SELECT *\nFROM foo.bar.25" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM foo.bar.25_", - "formatted": "SELECT *\nFROM foo.bar.25_" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM foo.bar.25x a", - "formatted": "SELECT *\nFROM foo.bar.25x a" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM foo.bar.25ab c", - "formatted": "SELECT *\nFROM foo.bar.25ab c" - }, - { - "dialect": "bigquery", - "sql": "SELECT b'abc'", - "formatted": "SELECT b'abc'" - }, - { - "dialect": "bigquery", - "sql": "SELECT AS STRUCT 1 AS a, 2 AS b", - "formatted": "SELECT AS STRUCT\n 1 AS a,\n 2 AS b" - }, - { - "dialect": "bigquery", - "sql": "SELECT DISTINCT AS STRUCT 1 AS a, 2 AS b", - "formatted": "SELECT DISTINCT AS STRUCT\n 1 AS a,\n 2 AS b" - }, - { - "dialect": "bigquery", - "sql": "SELECT AS VALUE STRUCT(1 AS a, 2 AS b)", - "formatted": "SELECT AS VALUE STRUCT(1 AS a, 2 AS b)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM q UNPIVOT(values FOR quarter IN (b, c))", - "formatted": "SELECT *\nFROM q UNPIVOT(values FOR quarter IN (b, c))" - }, - { - "dialect": "bigquery", - "sql": "SELECT MAKE_INTERVAL(100, 11, 1, 12, 30, 10)", - "formatted": "SELECT MAKE_INTERVAL(100, 11, 1, 12, 30, 10)" - }, - { - "dialect": "bigquery", - "sql": "SELECT y + 1 FROM x GROUP BY y + 1 ORDER BY 1", - "formatted": "SELECT y + 1\nFROM x\nGROUP BY\n y + 1\nORDER BY\n 1" - }, - { - "dialect": "bigquery", - "sql": "SELECT TIMESTAMP_SECONDS(2) AS t", - "formatted": "SELECT TIMESTAMP_SECONDS(2) AS t" - }, - { - "dialect": "bigquery", - "sql": "SELECT TIMESTAMP_MILLIS(2) AS t", - "formatted": "SELECT TIMESTAMP_MILLIS(2) AS t" - }, - { - "dialect": "bigquery", - "sql": "SELECT COUNT(x RESPECT NULLS)", - "formatted": "SELECT COUNT(x RESPECT NULLS)" - }, - { - "dialect": "bigquery", - "sql": "SELECT LAST_VALUE(x IGNORE NULLS) OVER y AS x", - "formatted": "SELECT LAST_VALUE(x IGNORE NULLS) OVER y AS x" - }, - { - "dialect": "bigquery", - "sql": "SELECT ARRAY((SELECT AS STRUCT 1 AS a, 2 AS b))", - "formatted": "SELECT\n ARRAY((\n SELECT AS STRUCT\n 1 AS a,\n 2 AS b\n ))" - }, - { - "dialect": "bigquery", - "sql": "SELECT ARRAY((SELECT AS STRUCT 1 AS a, 2 AS b) LIMIT 10)", - "formatted": "SELECT\n ARRAY((\n SELECT AS STRUCT\n 1 AS a,\n 2 AS b\n ) LIMIT 10)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM x WHERE x.y >= (SELECT MAX(a) FROM b-c) - 20", - "formatted": "SELECT *\nFROM x\nWHERE\n x.y >= (\n SELECT MAX(a)\n FROM b-c\n )\n - 20" - }, - { - "dialect": "bigquery", - "sql": "SELECT FORMAT_TIMESTAMP('%F %T', CURRENT_TIMESTAMP(), 'Europe/Berlin') AS ts", - "formatted": "SELECT FORMAT_TIMESTAMP('%F %T', CURRENT_TIMESTAMP(), 'Europe/Berlin') AS ts" - }, - { - "dialect": "bigquery", - "sql": "SELECT cars, apples FROM some_table PIVOT(SUM(total_counts) FOR products IN ('general.cars' AS cars, 'food.apples' AS apples))", - "formatted": "SELECT\n cars,\n apples\nFROM some_table PIVOT(SUM(total_counts) FOR products IN ('general.cars' AS cars, 'food.apples' AS apples))" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM test QUALIFY a IS DISTINCT FROM b WINDOW c AS (PARTITION BY d)", - "formatted": "SELECT *\nFROM test\nQUALIFY a IS DISTINCT FROM b\nWINDOW\n c AS (PARTITION BY d)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM (SELECT * FROM `t`) AS a UNPIVOT((c) FOR c_name IN (v1, v2))", - "formatted": "SELECT *\nFROM (\n SELECT *\n FROM `t`\n) AS a UNPIVOT((c) FOR c_name IN (v1, v2))" - }, - { - "dialect": "bigquery", - "sql": "SELECT ROW() OVER (y ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM x WINDOW y AS (PARTITION BY CATEGORY)", - "formatted": "SELECT ROW() OVER (y ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING)\nFROM x\nWINDOW\n y AS (PARTITION BY CATEGORY)" - }, - { - "dialect": "bigquery", - "sql": "SELECT item, purchases, LAST_VALUE(item) OVER (item_window ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING) AS most_popular FROM Produce WINDOW item_window AS (ORDER BY purchases)", - "formatted": "SELECT\n item,\n purchases,\n LAST_VALUE(item) OVER (item_window ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING) AS most_popular\nFROM Produce\nWINDOW\n item_window AS (ORDER BY purchases)" - }, - { - "dialect": "bigquery", - "sql": "SELECT LAST_VALUE(a IGNORE NULLS) OVER y FROM x WINDOW y AS (PARTITION BY CATEGORY)", - "formatted": "SELECT LAST_VALUE(a IGNORE NULLS) OVER y\nFROM x\nWINDOW\n y AS (PARTITION BY CATEGORY)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM foo AS t0 FOR SYSTEM_TIME AS OF '2026-02-12T23:22:21.743416+00:00'", - "formatted": "SELECT *\nFROM foo AS t0 FOR SYSTEM_TIME AS OF '2026-02-12T23:22:21.743416+00:00'" - }, - { - "dialect": "bigquery", - "sql": "SELECT b\"\\x0a$'x'00\"", - "formatted": "SELECT b\"\\x0a$'x'00\"" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM t1, t2", - "formatted": "SELECT *\nFROM t1, t2" - }, - { - "dialect": "bigquery", - "sql": "SELECT r\"\\t\"", - "formatted": "SELECT r\"\\t\"" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM `proj.dataset.INFORMATION_SCHEMA.SOME_VIEW`", - "formatted": "SELECT *\nFROM `proj.dataset.INFORMATION_SCHEMA.SOME_VIEW`" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM region_or_dataset.INFORMATION_SCHEMA.TABLES", - "formatted": "SELECT *\nFROM region_or_dataset.INFORMATION_SCHEMA.TABLES" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM region_or_dataset.INFORMATION_SCHEMA.TABLES AS some_name", - "formatted": "SELECT *\nFROM region_or_dataset.INFORMATION_SCHEMA.TABLES AS some_name" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM proj.region_or_dataset.INFORMATION_SCHEMA.TABLES", - "formatted": "SELECT *\nFROM proj.region_or_dataset.INFORMATION_SCHEMA.TABLES" - }, - { - "dialect": "bigquery", - "sql": "SELECT ARRAY(SELECT AS STRUCT 1 a, 2 b)", - "formatted": "SELECT\n ARRAY(SELECT AS STRUCT\n 1 a,\n 2 b)", - "previously_formatted": "SELECT\n ARRAY(SELECT AS STRUCT\n 1 AS a,\n 2 AS b)" - }, - { - "dialect": "bigquery", - "sql": "select array_contains([1, 2, 3], 1)", - "formatted": "SELECT array_contains([1, 2, 3], 1)" - }, - { - "dialect": "bigquery", - "sql": "SELECT SPLIT(foo)", - "formatted": "SELECT SPLIT(foo)" - }, - { - "dialect": "bigquery", - "sql": "SELECT 1 AS hash", - "formatted": "SELECT 1 AS hash" - }, - { - "dialect": "bigquery", - "sql": "SELECT 1 AS at", - "formatted": "SELECT 1 AS at" - }, - { - "dialect": "bigquery", - "sql": "SELECT \"\"\"ends with \\\"word\\\"\"\"\"", - "formatted": "SELECT \"\"\"ends with \\\"word\\\"\"\"\"" - }, - { - "dialect": "bigquery", - "sql": "SELECT '''ends with \\'word\\''''", - "formatted": "SELECT '''ends with \\'word\\''''" - }, - { - "dialect": "bigquery", - "sql": "SELECT \"\"\"a\\\"b\"\"\"", - "formatted": "SELECT \"\"\"a\\\"b\"\"\"" - }, - { - "dialect": "bigquery", - "sql": "SELECT r\"\"\"ends with \\\"\"\"\"", - "formatted": "SELECT r\"\"\"ends with \\\"\"\"\"" - }, - { - "dialect": "bigquery", - "sql": "SELECT r\"\"\"a\\\"b\"\"\"", - "formatted": "SELECT r\"\"\"a\\\"b\"\"\"" - }, - { - "dialect": "bigquery", - "sql": "SELECT a overlaps", - "formatted": "SELECT a overlaps", - "previously_formatted": "SELECT\n a AS overlaps" - }, - { - "dialect": "bigquery", - "sql": "SELECT y + 1 z FROM x GROUP BY y + 1 ORDER BY z", - "formatted": "SELECT y + 1 z\nFROM x\nGROUP BY\n y + 1\nORDER BY\n z", - "previously_formatted": "SELECT\n y + 1 AS z\nFROM x\nGROUP BY\n y + 1\nORDER BY\n z" - }, - { - "dialect": "bigquery", - "sql": "SELECT y + 1 z FROM x GROUP BY y + 1", - "formatted": "SELECT y + 1 z\nFROM x\nGROUP BY\n y + 1", - "previously_formatted": "SELECT\n y + 1 AS z\nFROM x\nGROUP BY\n y + 1" - }, - { - "dialect": "bigquery", - "sql": "SELECT JSON '\"foo\"' AS json_data", - "formatted": "SELECT JSON '\"foo\"' AS json_data" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM (SELECT a, b, c FROM test) PIVOT(SUM(b) d, COUNT(*) e FOR c IN ('x', 'y'))", - "formatted": "SELECT *\nFROM (\n SELECT\n a,\n b,\n c\n FROM test\n) PIVOT(SUM(b) d, COUNT(*) e FOR c IN ('x', 'y'))", - "previously_formatted": "SELECT\n *\nFROM (\n SELECT\n a,\n b,\n c\n FROM test\n) PIVOT(SUM(b) AS d, COUNT(*) AS e FOR c IN ('x', 'y'))" - }, - { - "dialect": "bigquery", - "sql": "SELECT CAST(1 AS BYTEINT)", - "formatted": "SELECT CAST(1 AS BYTEINT)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM a-b c", - "formatted": "SELECT *\nFROM a-b c", - "previously_formatted": "SELECT\n *\nFROM a-b AS c" - }, - { - "dialect": "bigquery", - "sql": "SELECT PARSE_DATETIME('%a %b %e %I:%M:%S %Y', 'Thu Dec 25 07:30:00 2008')", - "formatted": "SELECT PARSE_DATETIME('%a %b %e %I:%M:%S %Y', 'Thu Dec 25 07:30:00 2008')" - }, - { - "dialect": "bigquery", - "sql": "SELECT TRANSLATE(MODEL, 'in', 't') FROM (SELECT 'input' AS MODEL)", - "formatted": "SELECT TRANSLATE(MODEL, 'in', 't')\nFROM (\n SELECT 'input' AS MODEL\n)" - }, - { - "dialect": "bigquery", - "sql": "SELECT GRANT FROM (SELECT 'input' AS GRANT)", - "formatted": "SELECT GRANT\nFROM (\n SELECT 'input' AS GRANT\n)" - }, - { - "dialect": "bigquery", - "sql": "SELECT CAST(1 AS NUMERIC(10, 2))", - "formatted": "SELECT CAST(1 AS NUMERIC(10, 2))" - }, - { - "dialect": "bigquery", - "sql": "SELECT CAST('1' AS STRING(10)) UNION ALL SELECT CAST('2' AS STRING(10))", - "formatted": "SELECT CAST('1' AS STRING(10))\nUNION ALL\nSELECT CAST('2' AS STRING(10))" - }, - { - "dialect": "bigquery", - "sql": "SELECT cola FROM (SELECT CAST('1' AS STRING(10)) AS cola UNION ALL SELECT CAST('2' AS STRING(10)) AS cola)", - "formatted": "SELECT cola\nFROM (\n SELECT CAST('1' AS STRING(10)) AS cola\n UNION ALL\n SELECT CAST('2' AS STRING(10)) AS cola\n)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM GAP_FILL(TABLE device_data, ts_column => 'time', bucket_width => INTERVAL '1' MINUTE, value_columns => [('signal', 'locf')]) ORDER BY time", - "formatted": "SELECT *\nFROM GAP_FILL(\n TABLE device_data,\n ts_column => 'time',\n bucket_width => INTERVAL '1' MINUTE,\n value_columns => [('signal', 'locf')]\n)\nORDER BY\n time" - }, - { - "dialect": "bigquery", - "sql": "SELECT a, b, c, d, e FROM GAP_FILL(TABLE foo, ts_column => 'b', partitioning_columns => ['a'], value_columns => [('c', 'bar'), ('d', 'baz'), ('e', 'bla')], bucket_width => INTERVAL '1' DAY)", - "formatted": "SELECT\n a,\n b,\n c,\n d,\n e\nFROM GAP_FILL(\n TABLE foo,\n ts_column => 'b',\n partitioning_columns => ['a'],\n value_columns => [\n ('c', 'bar'),\n ('d', 'baz'),\n ('e', 'bla')\n ],\n bucket_width => INTERVAL '1' DAY\n)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM GAP_FILL(TABLE device_data, ts_column => 'time', bucket_width => INTERVAL '1' MINUTE, value_columns => [('signal', 'linear')], ignore_null_values => FALSE) ORDER BY time", - "formatted": "SELECT *\nFROM GAP_FILL(\n TABLE device_data,\n ts_column => 'time',\n bucket_width => INTERVAL '1' MINUTE,\n value_columns => [('signal', 'linear')],\n ignore_null_values => FALSE\n)\nORDER BY\n time" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM GAP_FILL(TABLE device_data, ts_column => 'time', bucket_width => INTERVAL '1' MINUTE) ORDER BY time", - "formatted": "SELECT *\nFROM GAP_FILL(\n TABLE device_data,\n ts_column => 'time',\n bucket_width => INTERVAL '1' MINUTE\n)\nORDER BY\n time" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM GAP_FILL(TABLE device_data, ts_column => 'time', bucket_width => INTERVAL '1' MINUTE, value_columns => [('signal', 'null')], origin => CAST('2023-11-01 09:30:01' AS DATETIME)) ORDER BY time", - "formatted": "SELECT *\nFROM GAP_FILL(\n TABLE device_data,\n ts_column => 'time',\n bucket_width => INTERVAL '1' MINUTE,\n value_columns => [('signal', 'null')],\n origin => CAST('2023-11-01 09:30:01' AS DATETIME)\n)\nORDER BY\n time" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM GAP_FILL(TABLE device_data, ts_column => 'time', bucket_width => INTERVAL '1' MINUTE, value_columns => [('signal', 'locf')]) ORDER BY time", - "formatted": "SELECT *\nFROM GAP_FILL(\n TABLE device_data,\n ts_column => 'time',\n bucket_width => INTERVAL '1' MINUTE,\n value_columns => [('signal', 'locf')]\n)\nORDER BY\n time" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM ML.PREDICT(MODEL mydataset.mymodel, (SELECT label, column1, column2 FROM mydataset.mytable))", - "formatted": "SELECT *\nFROM ML.PREDICT(MODEL mydataset.mymodel, (\n SELECT\n label,\n column1,\n column2\n FROM mydataset.mytable\n))" - }, - { - "dialect": "bigquery", - "sql": "SELECT label, predicted_label1, predicted_label AS predicted_label2 FROM ML.PREDICT(MODEL mydataset.mymodel2, (SELECT * EXCEPT (predicted_label), predicted_label AS predicted_label1 FROM ML.PREDICT(MODEL mydataset.mymodel1, TABLE mydataset.mytable)))", - "formatted": "SELECT\n label,\n predicted_label1,\n predicted_label AS predicted_label2\nFROM ML.PREDICT(\n MODEL mydataset.mymodel2,\n (\n SELECT\n * EXCEPT (predicted_label),\n predicted_label AS predicted_label1\n FROM ML.PREDICT(MODEL mydataset.mymodel1, TABLE mydataset.mytable)\n )\n)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM ML.PREDICT(MODEL mydataset.mymodel, (SELECT custom_label, column1, column2 FROM mydataset.mytable), STRUCT(0.55 AS threshold))", - "formatted": "SELECT *\nFROM ML.PREDICT(\n MODEL mydataset.mymodel,\n (\n SELECT\n custom_label,\n column1,\n column2\n FROM mydataset.mytable\n ),\n STRUCT(0.55 AS threshold)\n)" - }, - { - "dialect": "bigquery", - "sql": "SELECT COSH(1.5)", - "formatted": "SELECT COSH(1.5)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM ML.PREDICT(MODEL `my_project`.my_dataset.my_model, (SELECT * FROM input_data))", - "formatted": "SELECT *\nFROM ML.PREDICT(MODEL `my_project`.my_dataset.my_model, (\n SELECT *\n FROM input_data\n))" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM ML.PREDICT(MODEL my_dataset.vision_model, (SELECT uri, ML.RESIZE_IMAGE(ML.DECODE_IMAGE(data), 480, 480, FALSE) AS input FROM my_dataset.object_table))", - "formatted": "SELECT *\nFROM ML.PREDICT(\n MODEL my_dataset.vision_model,\n (\n SELECT\n uri,\n ML.RESIZE_IMAGE(ML.DECODE_IMAGE(data), 480, 480, FALSE) AS input\n FROM my_dataset.object_table\n )\n)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM ML.PREDICT(MODEL my_dataset.vision_model, (SELECT uri, ML.CONVERT_COLOR_SPACE(ML.RESIZE_IMAGE(ML.DECODE_IMAGE(data), 224, 280, TRUE), 'YIQ') AS input FROM my_dataset.object_table WHERE content_type = 'image/jpeg'))", - "formatted": "SELECT *\nFROM ML.PREDICT(\n MODEL my_dataset.vision_model,\n (\n SELECT\n uri,\n ML.CONVERT_COLOR_SPACE(ML.RESIZE_IMAGE(ML.DECODE_IMAGE(data), 224, 280, TRUE), 'YIQ') AS input\n FROM my_dataset.object_table\n WHERE content_type = 'image/jpeg'\n )\n)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM ML.FEATURES_AT_TIME((SELECT 1), num_rows => 1)", - "formatted": "SELECT *\nFROM ML.FEATURES_AT_TIME((\n SELECT 1\n), num_rows => 1)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM ML.FEATURES_AT_TIME(TABLE mydataset.feature_table, time => '2022-06-11 10:00:00+00', num_rows => 1, ignore_feature_nulls => TRUE)", - "formatted": "SELECT *\nFROM ML.FEATURES_AT_TIME(\n TABLE mydataset.feature_table,\n time => '2022-06-11 10:00:00+00',\n num_rows => 1,\n ignore_feature_nulls => TRUE\n)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM VECTOR_SEARCH(TABLE mydataset.base_table, 'column_to_search', TABLE mydataset.query_table, 'query_column_to_search', top_k => 2, distance_type => 'cosine', options => '{\"fraction_lists_to_search\":0.15}')", - "formatted": "SELECT *\nFROM VECTOR_SEARCH(\n TABLE mydataset.base_table,\n 'column_to_search',\n TABLE mydataset.query_table,\n 'query_column_to_search',\n top_k => 2,\n distance_type => 'cosine',\n options => '{\"fraction_lists_to_search\":0.15}'\n)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM VECTOR_SEARCH(TABLE mydataset.base_table, 'column_to_search', TABLE mydataset.query_table, query_column_to_search => 'query_column_to_search', top_k => 2, distance_type => 'cosine', options => '{\"fraction_lists_to_search\":0.15}')", - "formatted": "SELECT *\nFROM VECTOR_SEARCH(\n TABLE mydataset.base_table,\n 'column_to_search',\n TABLE mydataset.query_table,\n query_column_to_search => 'query_column_to_search',\n top_k => 2,\n distance_type => 'cosine',\n options => '{\"fraction_lists_to_search\":0.15}'\n)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM VECTOR_SEARCH((SELECT * FROM mydataset.base_table), 'column_to_search', (SELECT * FROM mydataset.query_table), 'query_column_to_search')", - "formatted": "SELECT *\nFROM VECTOR_SEARCH(\n (\n SELECT *\n FROM mydataset.base_table\n ),\n 'column_to_search',\n (\n SELECT *\n FROM mydataset.query_table\n ),\n 'query_column_to_search'\n)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM VECTOR_SEARCH(TABLE mydataset.base_table, 'column_to_search', TABLE mydataset.query_table)", - "formatted": "SELECT *\nFROM VECTOR_SEARCH(TABLE mydataset.base_table, 'column_to_search', TABLE mydataset.query_table)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM ML.TRANSLATE(MODEL `mydataset.mytranslatemodel`, TABLE `mydataset.mybqtable`, STRUCT('translate_text' AS translate_mode, 'zh-CN' AS target_language_code))", - "formatted": "SELECT *\nFROM ML.TRANSLATE(\n MODEL `mydataset.mytranslatemodel`,\n TABLE `mydataset.mybqtable`,\n STRUCT(\n 'translate_text' AS translate_mode,\n 'zh-CN' AS target_language_code\n )\n)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM ML.FORECAST(MODEL `mydataset.mymodel`, STRUCT(2 AS horizon))", - "formatted": "SELECT *\nFROM ML.FORECAST(MODEL `mydataset.mymodel`, STRUCT(2 AS horizon))" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM ML.FORECAST(MODEL `mydataset.mymodel`, TABLE `mydataset.mybqtable`, STRUCT(2 AS horizon, 4 AS confidence_level))", - "formatted": "SELECT *\nFROM ML.FORECAST(\n MODEL `mydataset.mymodel`,\n TABLE `mydataset.mybqtable`,\n STRUCT(2 AS horizon, 4 AS confidence_level)\n)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM ML.FORECAST(MODEL `mydataset.mymodel`, (SELECT * FROM mydataset.query_table), STRUCT())", - "formatted": "SELECT *\nFROM ML.FORECAST(MODEL `mydataset.mymodel`, (\n SELECT *\n FROM mydataset.query_table\n), STRUCT())" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM AI.FORECAST(TABLE citibike_trips, data_col => 'num_trips', timestamp_col => 'date', horizon => 30)", - "formatted": "SELECT *\nFROM AI.FORECAST(\n TABLE citibike_trips,\n data_col => 'num_trips',\n timestamp_col => 'date',\n horizon => 30\n)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM AI.FORECAST((SELECT * FROM citibike_trips), data_col => 'num_trips', timestamp_col => 'date', horizon => 30)", - "formatted": "SELECT *\nFROM AI.FORECAST(\n (\n SELECT *\n FROM citibike_trips\n ),\n data_col => 'num_trips',\n timestamp_col => 'date',\n horizon => 30\n)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM ML.GENERATE_TEXT(MODEL `mydataset.gemini_model`, TABLE `mydataset.prompt_table`, STRUCT(0.15 AS temperature))", - "formatted": "SELECT *\nFROM ML.GENERATE_TEXT(\n MODEL `mydataset.gemini_model`,\n TABLE `mydataset.prompt_table`,\n STRUCT(0.15 AS temperature)\n)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM AI.GENERATE_TEXT(MODEL `mydataset.gemini_model`, TABLE `mydataset.prompt_table`, STRUCT(0.15 AS temperature))", - "formatted": "SELECT *\nFROM AI.GENERATE_TEXT(\n MODEL `mydataset.gemini_model`,\n TABLE `mydataset.prompt_table`,\n STRUCT(0.15 AS temperature)\n)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM AI.GENERATE_TABLE(MODEL `mydataset.gemini_model`, (SELECT 'Q' AS prompt), STRUCT('name STRING' AS output_schema))", - "formatted": "SELECT *\nFROM AI.GENERATE_TABLE(\n MODEL `mydataset.gemini_model`,\n (\n SELECT 'Q' AS prompt\n ),\n STRUCT('name STRING' AS output_schema)\n)" - }, - { - "dialect": "bigquery", - "sql": "SELECT AI.GENERATE_BOOL(MODEL `mydataset.gemini_model`, 'Is sky blue?')", - "formatted": "SELECT AI.GENERATE_BOOL(MODEL `mydataset.gemini_model`, 'Is sky blue?')" - }, - { - "dialect": "bigquery", - "sql": "SELECT AI.EMBED('hello')", - "formatted": "SELECT AI.EMBED('hello')" - }, - { - "dialect": "bigquery", - "sql": "SELECT AI.SIMILARITY('a', 'b')", - "formatted": "SELECT AI.SIMILARITY('a', 'b')" - }, - { - "dialect": "bigquery", - "sql": "SELECT AI.GENERATE('Write a haiku')", - "formatted": "SELECT AI.GENERATE('Write a haiku')" - }, - { - "dialect": "bigquery", - "sql": "SELECT JSON_OBJECT() AS json_data", - "formatted": "SELECT JSON_OBJECT() AS json_data" - }, - { - "dialect": "bigquery", - "sql": "SELECT JSON_OBJECT('foo', 10, 'bar', TRUE) AS json_data", - "formatted": "SELECT JSON_OBJECT('foo', 10, 'bar', TRUE) AS json_data" - }, - { - "dialect": "bigquery", - "sql": "SELECT JSON_OBJECT('foo', 10, 'bar', ['a', 'b']) AS json_data", - "formatted": "SELECT JSON_OBJECT('foo', 10, 'bar', ['a', 'b']) AS json_data" - }, - { - "dialect": "bigquery", - "sql": "SELECT JSON_OBJECT('a', 10, 'a', 'foo') AS json_data", - "formatted": "SELECT JSON_OBJECT('a', 10, 'a', 'foo') AS json_data" - }, - { - "dialect": "bigquery", - "sql": "SELECT JSON_OBJECT(['a', 'b'], [10, NULL]) AS json_data", - "formatted": "SELECT JSON_OBJECT(['a', 'b'], [10, NULL]) AS json_data" - }, - { - "dialect": "bigquery", - "sql": "SELECT JSON_OBJECT(['a', 'b'], [JSON '10', JSON '\"foo\"']) AS json_data", - "formatted": "SELECT JSON_OBJECT(['a', 'b'], [JSON '10', JSON '\"foo\"']) AS json_data" - }, - { - "dialect": "bigquery", - "sql": "SELECT JSON_OBJECT(['a', 'b'], [STRUCT(10 AS id, 'Red' AS color), STRUCT(20 AS id, 'Blue' AS color)]) AS json_data", - "formatted": "SELECT\n JSON_OBJECT(['a', 'b'], [\n STRUCT(10 AS id, 'Red' AS color),\n STRUCT(20 AS id, 'Blue' AS color)\n ]) AS json_data" - }, - { - "dialect": "bigquery", - "sql": "SELECT JSON_OBJECT(['a', 'b'], [TO_JSON(10), TO_JSON(['foo', 'bar'])]) AS json_data", - "formatted": "SELECT JSON_OBJECT(['a', 'b'], [TO_JSON(10), TO_JSON(['foo', 'bar'])]) AS json_data" - }, - { - "dialect": "bigquery", - "sql": "SELECT MOD((SELECT 1), 2)", - "formatted": "SELECT\n MOD((\n SELECT 1\n ), 2)" - }, - { - "dialect": "bigquery", - "sql": "SELECT STRUCT>([\"2023-01-17\"])", - "formatted": "SELECT STRUCT> ([\"2023-01-17\"])" - }, - { - "dialect": "bigquery", - "sql": "SELECT STRUCT((SELECT 'foo')).*", - "formatted": "SELECT\n STRUCT ((\n SELECT 'foo'\n )) .*" - }, - { - "dialect": "bigquery", - "sql": "SELECT RANGE(CAST('2022-12-01' AS DATE), CAST('2022-12-31' AS DATE))", - "formatted": "SELECT\n RANGE(\n CAST('2022-12-01' AS DATE),\n CAST('2022-12-31' AS DATE)\n )" - }, - { - "dialect": "bigquery", - "sql": "SELECT RANGE(NULL, CAST('2022-12-31' AS DATE))", - "formatted": "SELECT RANGE(NULL, CAST('2022-12-31' AS DATE))" - }, - { - "dialect": "bigquery", - "sql": "SELECT RANGE(CAST('2022-10-01 14:53:27' AS DATETIME), CAST('2022-10-01 16:00:00' AS DATETIME))", - "formatted": "SELECT\n RANGE(\n CAST('2022-10-01 14:53:27' AS DATETIME),\n CAST('2022-10-01 16:00:00' AS DATETIME)\n )" - }, - { - "dialect": "bigquery", - "sql": "SELECT RANGE(CAST('2022-10-01 14:53:27 America/Los_Angeles' AS TIMESTAMP), CAST('2022-10-01 16:00:00 America/Los_Angeles' AS TIMESTAMP))", - "formatted": "SELECT\n RANGE(\n CAST('2022-10-01 14:53:27 America/Los_Angeles' AS TIMESTAMP),\n CAST('2022-10-01 16:00:00 America/Los_Angeles' AS TIMESTAMP)\n )" - }, - { - "dialect": "bigquery", - "sql": "SELECT CAST('2026-03-24' AS STRING FORMAT ('YYYY'))", - "formatted": "SELECT CAST('2026-03-24' AS STRING FORMAT('YYYY'))" - }, - { - "dialect": "bigquery", - "sql": "SELECT CAST(date AS STRING FORMAT ('YYYY')) FROM (SELECT DATE('2026-03-24') AS date)", - "formatted": "SELECT CAST(date AS STRING FORMAT('YYYY'))\nFROM (\n SELECT DATE('2026-03-24') AS date\n)" - }, - { - "dialect": "bigquery", - "sql": "SELECT CAST(date AS STRING FORMAT ('YYYY-MM-DD'))", - "formatted": "SELECT CAST(date AS STRING FORMAT('YYYY-MM-DD'))" - }, - { - "dialect": "bigquery", - "sql": "SELECT CAST(timestamp AS STRING FORMAT ('YYYY-MM-DD') AT TIME ZONE 'UTC')", - "formatted": "SELECT CAST(timestamp AS STRING FORMAT('YYYY-MM-DD') AT TIME ZONE 'UTC')" - }, - { - "dialect": "bigquery", - "sql": "SELECT CAST(date AS TIMESTAMP FORMAT ('YYYY-MM-DD HH24:MI:SS'))", - "formatted": "SELECT CAST(date AS TIMESTAMP FORMAT('YYYY-MM-DD HH24:MI:SS'))" - }, - { - "dialect": "bigquery", - "sql": "SELECT a, GROUP_CONCAT(b) FROM table GROUP BY a", - "formatted": "SELECT\n a,\n GROUP_CONCAT(b)\nFROM table\nGROUP BY\n a" - }, - { - "dialect": "bigquery", - "sql": "SELECT 1 AS foo INNER UNION ALL SELECT 3 AS foo, 4 AS bar", - "formatted": "SELECT 1 AS foo\nINNER UNION ALL\nSELECT\n 3 AS foo,\n 4 AS bar" - }, - { - "dialect": "bigquery", - "sql": "SELECT 1 AS x UNION ALL CORRESPONDING SELECT 2 AS x", - "formatted": "SELECT 1 AS x\nUNION ALL CORRESPONDING\nSELECT 2 AS x" - }, - { - "dialect": "bigquery", - "sql": "SELECT 1 AS x UNION ALL CORRESPONDING BY (foo, bar) SELECT 2 AS x", - "formatted": "SELECT 1 AS x\nUNION ALL CORRESPONDING BY (foo, bar)\nSELECT 2 AS x" - }, - { - "dialect": "bigquery", - "sql": "SELECT 1 AS x LEFT UNION ALL CORRESPONDING SELECT 2 AS x", - "formatted": "SELECT 1 AS x\nLEFT UNION ALL CORRESPONDING\nSELECT 2 AS x" - }, - { - "dialect": "bigquery", - "sql": "SELECT 1 AS x UNION ALL STRICT CORRESPONDING SELECT 2 AS x", - "formatted": "SELECT 1 AS x\nUNION ALL STRICT CORRESPONDING\nSELECT 2 AS x" - }, - { - "dialect": "bigquery", - "sql": "SELECT 1 AS x UNION ALL STRICT CORRESPONDING BY (foo, bar) SELECT 2 AS x", - "formatted": "SELECT 1 AS x\nUNION ALL STRICT CORRESPONDING BY (foo, bar)\nSELECT 2 AS x" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM UNNEST(x) WITH OFFSET EXCEPT DISTINCT SELECT * FROM UNNEST(y) WITH OFFSET", - "formatted": "SELECT *\nFROM UNNEST(x) WITH OFFSET\nEXCEPT DISTINCT\nSELECT *\nFROM UNNEST(y) WITH OFFSET" - }, - { - "dialect": "bigquery", - "sql": "SELECT col FROM t WHERE _PARTITIONTIME BETWEEN a AND b", - "formatted": "SELECT col\nFROM t\nWHERE _PARTITIONTIME BETWEEN a AND b" - }, - { - "dialect": "bigquery", - "sql": "SELECT _DBT_MAX_PARTITION FROM t", - "formatted": "SELECT _DBT_MAX_PARTITION\nFROM t" - }, - { - "dialect": "bigquery", - "sql": "WITH cte(c) AS (SELECT * FROM t) SELECT * FROM cte", - "formatted": "WITH cte(c) AS (\n SELECT *\n FROM t\n)\n\nSELECT *\nFROM cte" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM t AS t(c1, c2)", - "formatted": "SELECT *\nFROM t AS t(c1, c2)" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM t AS t(c1, c2)", - "formatted": "SELECT *\nFROM t AS t(c1, c2)" - }, - { - "dialect": "bigquery", - "sql": "WITH foo AS (SELECT [1, 2, 3] AS array_col) SELECT array_col[offset] FROM foo CROSS JOIN UNNEST(array_col) WITH OFFSET AS offset", - "formatted": "WITH foo AS (\n SELECT [1, 2, 3] AS array_col\n)\n\nSELECT array_col[offset]\nFROM foo\nCROSS JOIN UNNEST(array_col) WITH OFFSET AS offset" - }, - { - "dialect": "bigquery", - "sql": "SELECT `db.t`.`c` FROM `db.t`", - "formatted": "SELECT `db.t`.`c`\nFROM `db.t`" - }, - { - "dialect": "bigquery", - "sql": "SELECT `p.d.t`.`c`.`f` FROM `p.d.t`", - "formatted": "SELECT `p.d.t`.`c`.`f`\nFROM `p.d.t`" - }, - { - "dialect": "bigquery", - "sql": "SELECT * FROM ML.TRANSLATE(MODEL `mydataset.mymodel`, (SELECT comment AS text_content FROM mydataset.mytable), STRUCT('translate_text' AS translate_mode, 'en' AS target_language_code))", - "formatted": "SELECT *\nFROM ML.TRANSLATE(\n MODEL `mydataset.mymodel`,\n (\n SELECT comment AS text_content\n FROM mydataset.mytable\n ),\n STRUCT(\n 'translate_text' AS translate_mode,\n 'en' AS target_language_code\n )\n)" - }, - { - "dialect": "bigquery", - "sql": "WITH t AS (SELECT '{\"x-y\": \"z\"}' AS c) SELECT JSON_EXTRACT(c, '$.x-y') FROM t", - "formatted": "WITH t AS (\n SELECT '{\"x-y\": \"z\"}' AS c\n)\n\nSELECT JSON_EXTRACT(c, '$.x-y')\nFROM t" - }, - { - "dialect": "databricks", - "sql": "SELECT CAST('a' AS STRING COLLATE UTF8_BINARY)", - "formatted": "SELECT CAST('a' AS STRING COLLATE UTF8_BINARY)" - }, - { - "dialect": "databricks", - "sql": "SELECT COSH(1.5)", - "formatted": "SELECT COSH(1.5)" - }, - { - "dialect": "databricks", - "sql": "SELECT BITMAP_BIT_POSITION(10)", - "formatted": "SELECT BITMAP_BIT_POSITION(10)" - }, - { - "dialect": "databricks", - "sql": "SELECT BITMAP_BUCKET_NUMBER(32769)", - "formatted": "SELECT BITMAP_BUCKET_NUMBER(32769)" - }, - { - "dialect": "databricks", - "sql": "SELECT BITMAP_CONSTRUCT_AGG(value)", - "formatted": "SELECT BITMAP_CONSTRUCT_AGG(value)" - }, - { - "dialect": "databricks", - "sql": "SELECT EXP(1)", - "formatted": "SELECT EXP(1)" - }, - { - "dialect": "databricks", - "sql": "SELECT MODE(category)", - "formatted": "SELECT MODE(category)" - }, - { - "dialect": "databricks", - "sql": "SELECT MODE(price, TRUE) AS deterministic_mode FROM products", - "formatted": "SELECT MODE(price, TRUE) AS deterministic_mode\nFROM products" - }, - { - "dialect": "databricks", - "sql": "SELECT CAST(NULL AS VOID)", - "formatted": "SELECT CAST(NULL AS VOID)" - }, - { - "dialect": "databricks", - "sql": "SELECT void FROM t", - "formatted": "SELECT void\nFROM t" - }, - { - "dialect": "databricks", - "sql": "SELECT * FROM stream", - "formatted": "SELECT *\nFROM stream" - }, - { - "dialect": "databricks", - "sql": "SELECT * FROM STREAM t", - "formatted": "SELECT *\nFROM STREAM t" - }, - { - "dialect": "databricks", - "sql": "SELECT t.current_time FROM t", - "formatted": "SELECT t.current_time\nFROM t" - }, - { - "dialect": "databricks", - "sql": "SELECT col1 FROM t CLUSTER BY col1", - "formatted": "SELECT col1\nFROM t\nCLUSTER BY\n col1" - }, - { - "dialect": "databricks", - "sql": "SELECT col1, col2 FROM t CLUSTER BY col1, col2", - "formatted": "SELECT\n col1,\n col2\nFROM t\nCLUSTER BY\n col1,\n col2" - }, - { - "dialect": "databricks", - "sql": "SELECT CAST('11 23:4:0' AS INTERVAL DAY TO HOUR)", - "formatted": "SELECT CAST('11 23:4:0' AS INTERVAL DAY TO HOUR)" - }, - { - "dialect": "databricks", - "sql": "SELECT CAST('11 23:4:0' AS INTERVAL DAY TO MINUTE)", - "formatted": "SELECT CAST('11 23:4:0' AS INTERVAL DAY TO MINUTE)" - }, - { - "dialect": "databricks", - "sql": "SELECT CAST('11 23:4:0' AS INTERVAL DAY TO SECOND)", - "formatted": "SELECT CAST('11 23:4:0' AS INTERVAL DAY TO SECOND)" - }, - { - "dialect": "databricks", - "sql": "SELECT CAST('23:00:00' AS INTERVAL HOUR TO MINUTE)", - "formatted": "SELECT CAST('23:00:00' AS INTERVAL HOUR TO MINUTE)" - }, - { - "dialect": "databricks", - "sql": "SELECT CAST('23:00:00' AS INTERVAL HOUR TO SECOND)", - "formatted": "SELECT CAST('23:00:00' AS INTERVAL HOUR TO SECOND)" - }, - { - "dialect": "databricks", - "sql": "SELECT CAST('23:00:00' AS INTERVAL MINUTE TO SECOND)", - "formatted": "SELECT CAST('23:00:00' AS INTERVAL MINUTE TO SECOND)" - }, - { - "dialect": "databricks", - "sql": "SELECT ${x} FROM ${y} WHERE ${z} > 1", - "formatted": "SELECT ${x}\nFROM ${y}\nWHERE ${z} > 1" - }, - { - "dialect": "databricks", - "sql": "SELECT PARSE_JSON('{}')", - "formatted": "SELECT PARSE_JSON('{}')" - }, - { - "dialect": "databricks", - "sql": "SELECT RANDSTR(123)", - "formatted": "SELECT RANDSTR(123)" - }, - { - "dialect": "databricks", - "sql": "SELECT RANDSTR(123, 456)", - "formatted": "SELECT RANDSTR(123, 456)" - }, - { - "dialect": "databricks", - "sql": "SELECT * FROM sales UNPIVOT INCLUDE NULLS (sales FOR quarter IN (q1 AS `Jan-Mar`))", - "formatted": "SELECT *\nFROM sales UNPIVOT INCLUDE NULLS (sales FOR quarter IN (q1 AS `Jan-Mar`))" - }, - { - "dialect": "databricks", - "sql": "SELECT * FROM sales UNPIVOT EXCLUDE NULLS (sales FOR quarter IN (q1 AS `Jan-Mar`))", - "formatted": "SELECT *\nFROM sales UNPIVOT EXCLUDE NULLS (sales FOR quarter IN (q1 AS `Jan-Mar`))" - }, - { - "dialect": "databricks", - "sql": "SELECT * FROM t1, t2", - "formatted": "SELECT *\nFROM t1, t2" - }, - { - "dialect": "databricks", - "sql": "SELECT TIMESTAMP '2025-04-29 18.47.18'::DATE", - "formatted": "SELECT TIMESTAMP '2025-04-29 18.47.18'::DATE" - }, - { - "dialect": "databricks", - "sql": "SELECT DATE_FORMAT(CAST(FROM_UTC_TIMESTAMP(foo, 'America/Los_Angeles') AS TIMESTAMP), 'yyyy-MM-dd HH:mm:ss') AS foo FROM t", - "formatted": "SELECT\n DATE_FORMAT(\n CAST(FROM_UTC_TIMESTAMP(foo, 'America/Los_Angeles') AS TIMESTAMP),\n 'yyyy-MM-dd HH:mm:ss'\n ) AS foo\nFROM t" - }, - { - "dialect": "databricks", - "sql": "SELECT r\"\\\\foo.bar\\\"", - "formatted": "SELECT r\"\\\\foo.bar\\\"" - }, - { - "dialect": "databricks", - "sql": "SELECT SUBSTRING_INDEX(str, delim, count)", - "formatted": "SELECT SUBSTRING_INDEX(str, delim, count)" - }, - { - "dialect": "databricks", - "sql": "SELECT SUBSTR('Spark' FROM 5 FOR 1)", - "formatted": "SELECT SUBSTR('Spark' FROM 5 FOR 1)" - }, - { - "dialect": "databricks", - "sql": "SELECT SUBSTR('Spark SQL', 5)", - "formatted": "SELECT SUBSTR('Spark SQL', 5)" - }, - { - "dialect": "databricks", - "sql": "SELECT SUBSTR(ENCODE('Spark SQL', 'utf-8'), 5)", - "formatted": "SELECT SUBSTR(ENCODE('Spark SQL', 'utf-8'), 5)" - }, - { - "dialect": "databricks", - "sql": "SELECT test, LISTAGG(email, '') AS Email FROM organizations GROUP BY test", - "formatted": "SELECT\n test,\n LISTAGG(email, '') AS Email\nFROM organizations\nGROUP BY\n test" - }, - { - "dialect": "databricks", - "sql": "WITH t AS (VALUES ('foo_val') AS t(foo1)) SELECT foo1 FROM t", - "formatted": "WITH t AS (\n VALUES ('foo_val') AS t(foo1)\n)\n\nSELECT foo1\nFROM t" - }, - { - "dialect": "databricks", - "sql": "SELECT ELT(2, 'foo', 'bar', 'baz') AS Result", - "formatted": "SELECT ELT(2, 'foo', 'bar', 'baz') AS Result" - }, - { - "dialect": "databricks", - "sql": "SELECT MAKE_INTERVAL(100, 11, 12, 13, 14, 14, 15)", - "formatted": "SELECT MAKE_INTERVAL(100, 11, 12, 13, 14, 14, 15)" - }, - { - "dialect": "databricks", - "sql": "SELECT name, GROUPING_ID() FROM customer GROUP BY ROLLUP (name)", - "formatted": "SELECT\n name,\n GROUPING_ID()\nFROM customer\nGROUP BY\n ROLLUP (\n name\n )" - }, - { - "dialect": "databricks", - "sql": "SELECT CURDATE()", - "formatted": "SELECT CURDATE()" - }, - { - "dialect": "databricks", - "sql": "SELECT c1:price, c1:price.foo, c1:price.bar[1]", - "formatted": "SELECT\n c1:price,\n c1:price.foo,\n c1:price.bar[1]" - }, - { - "dialect": "databricks", - "sql": "SELECT TRY_CAST(c1:price AS ARRAY)", - "formatted": "SELECT TRY_CAST(c1:price AS ARRAY)" - }, - { - "dialect": "databricks", - "sql": "SELECT TRY_CAST(c1:[\"foo bar\"][\"baz qux\"] AS ARRAY)", - "formatted": "SELECT TRY_CAST(c1:[\"foo bar\"][\"baz qux\"] AS ARRAY)" - }, - { - "dialect": "databricks", - "sql": "SELECT c1:item[1].price FROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)", - "formatted": "SELECT c1:item[1].price\nFROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)" - }, - { - "dialect": "databricks", - "sql": "SELECT c1:item[*].price FROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)", - "formatted": "SELECT c1:item[*].price\nFROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)" - }, - { - "dialect": "databricks", - "sql": "SELECT FROM_JSON(c1:item[*].price, 'ARRAY')[0] FROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)", - "formatted": "SELECT FROM_JSON(c1:item[*].price, 'ARRAY')[0]\nFROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)" - }, - { - "dialect": "databricks", - "sql": "SELECT INLINE(FROM_JSON(c1:item[*], 'ARRAY>')) FROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)", - "formatted": "SELECT INLINE(FROM_JSON(c1:item[*], 'ARRAY>'))\nFROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)" - }, - { - "dialect": "databricks", - "sql": "SELECT c1:['price'] FROM VALUES ('{ \"price\": 5 }') AS T(c1)", - "formatted": "SELECT c1:['price']\nFROM VALUES ('{ \"price\": 5 }') AS T(c1)" - }, - { - "dialect": "databricks", - "sql": "SELECT GET_JSON_OBJECT(c1, '$.price') FROM VALUES ('{ \"price\": 5 }') AS T(c1)", - "formatted": "SELECT GET_JSON_OBJECT(c1, '$.price')\nFROM VALUES ('{ \"price\": 5 }') AS T(c1)" - }, - { - "dialect": "databricks", - "sql": "SELECT GET_JSON_OBJECT(col, path_col)", - "formatted": "SELECT GET_JSON_OBJECT(col, path_col)" - }, - { - "dialect": "databricks", - "sql": "SELECT GET_JSON_OBJECT(col, CONCAT('$.', field_name))", - "formatted": "SELECT GET_JSON_OBJECT(col, CONCAT('$.', field_name))" - }, - { - "dialect": "databricks", - "sql": "SELECT GET_JSON_OBJECT(GET_JSON_OBJECT(col, '$[0]'), '$.a')", - "formatted": "SELECT GET_JSON_OBJECT(GET_JSON_OBJECT(col, '$[0]'), '$.a')" - }, - { - "dialect": "databricks", - "sql": "SELECT raw:`zip code`, raw:`fb:testid`, raw:store['bicycle'], raw:store[\"zip code\"]", - "formatted": "SELECT\n raw:`zip code`,\n raw:`fb:testid`,\n raw:store['bicycle'],\n raw:store[\"zip code\"]" - }, - { - "dialect": "databricks", - "sql": "SELECT DATE_ADD(MONTH, 1, '2020-01-01')", - "formatted": "SELECT DATE_ADD(MONTH, 1, '2020-01-01')" - }, - { - "dialect": "databricks", - "sql": "SELECT DATEADD(e, 24) FROM t", - "formatted": "SELECT DATEADD(e, 24)\nFROM t" - }, - { - "dialect": "databricks", - "sql": "SELECT '20'?::INTEGER", - "formatted": "SELECT '20'?::INTEGER" - }, - { - "dialect": "databricks", - "sql": "SELECT OVERLAY('Spark SQL', 'ANSI ', 7, 0)", - "formatted": "SELECT OVERLAY('Spark SQL', 'ANSI ', 7, 0)" - }, - { - "dialect": "databricks", - "sql": "SELECT OVERLAY('Spark SQL' PLACING 'CORE' FROM 7)", - "formatted": "SELECT OVERLAY('Spark SQL' PLACING 'CORE' FROM 7)" - }, - { - "dialect": "databricks", - "sql": "SELECT OVERLAY(ENCODE('Spark SQL', 'utf-8') PLACING ENCODE('_', 'utf-8') FROM 6)", - "formatted": "SELECT OVERLAY(ENCODE('Spark SQL', 'utf-8') PLACING ENCODE('_', 'utf-8') FROM 6)" - }, - { - "dialect": "databricks", - "sql": "SELECT OVERLAY('Spark SQL' PLACING 'ANSI ' FROM 7 FOR 0)", - "formatted": "SELECT OVERLAY('Spark SQL' PLACING 'ANSI ' FROM 7 FOR 0)" - }, - { - "dialect": "databricks", - "sql": "WITH t AS (SELECT '{\"x-y\": \"z\"}' AS c) SELECT get_json_object(c, '$.x-y') FROM t", - "formatted": "WITH t AS (\n SELECT '{\"x-y\": \"z\"}' AS c\n)\n\nSELECT get_json_object(c, '$.x-y')\nFROM t" - }, - { - "dialect": "duckdb", - "sql": "SELECT ([1,2,3])[:-:-1]", - "formatted": "SELECT\n (\n [1, 2, 3]\n )[:-:-1]" - }, - { - "dialect": "duckdb", - "sql": "SELECT INTERVAL '1 hour'::VARCHAR", - "formatted": "SELECT INTERVAL '1 hour'::VARCHAR" - }, - { - "dialect": "duckdb", - "sql": "SELECT str[0:1]", - "formatted": null - }, - { - "dialect": "duckdb", - "sql": "SELECT COSH(1.5)", - "formatted": "SELECT COSH(1.5)" - }, - { - "dialect": "duckdb", - "sql": "SELECT MODE(category)", - "formatted": "SELECT MODE(category)" - }, - { - "dialect": "duckdb", - "sql": "SELECT e'\\n'", - "formatted": "SELECT e'\\n'" - }, - { - "dialect": "duckdb", - "sql": "SELECT e'\\t'", - "formatted": "SELECT e'\\t'" - }, - { - "dialect": "duckdb", - "sql": "SELECT e'update table_name set a = \\'foo\\' where 1 = 0' AS x FROM tab", - "formatted": "SELECT e'update table_name set a = \\'foo\\' where 1 = 0' AS x\nFROM tab" - }, - { - "dialect": "duckdb", - "sql": "SELECT GET_BIT(CAST('0110010' AS BIT), 2)", - "formatted": "SELECT GET_BIT(CAST('0110010' AS BIT), 2)" - }, - { - "dialect": "duckdb", - "sql": "SELECT 1 WHERE x > $1", - "formatted": "SELECT 1\nWHERE x > $1" - }, - { - "dialect": "duckdb", - "sql": "SELECT 1 WHERE x > $name", - "formatted": "SELECT 1\nWHERE x > $name" - }, - { - "dialect": "duckdb", - "sql": "SELECT 1 AS a$b", - "formatted": "SELECT 1 AS a$b" - }, - { - "dialect": "duckdb", - "sql": "SELECT '{\"x\": 1}' -> c FROM t", - "formatted": "SELECT '{\"x\": 1}' -> c\nFROM t" - }, - { - "dialect": "duckdb", - "sql": "SELECT EXP(1)", - "formatted": "SELECT EXP(1)" - }, - { - "dialect": "duckdb", - "sql": "SELECT '{\"duck\": [1, 2, 3]}' -> '$.duck[#-1]'", - "formatted": "SELECT '{\"duck\": [1, 2, 3]}' -> '$.duck[#-1]'" - }, - { - "dialect": "duckdb", - "sql": "SELECT tbl.x*1e4+tbl.y FROM tbl", - "formatted": "SELECT tbl.x * 1e4 + tbl.y\nFROM tbl" - }, - { - "dialect": "duckdb", - "sql": "SELECT LIST_TRANSFORM([5, NULL, 6], (x, y) -> COALESCE(x, y, 0) + 1)", - "formatted": "SELECT LIST_TRANSFORM([5, NULL, 6], (x, y) -> COALESCE(x, y, 0) + 1)" - }, - { - "dialect": "duckdb", - "sql": "SELECT LIST_TRANSFORM([5, NULL, 6], LAMBDA x, y : COALESCE(x, y, 0) + 1)", - "formatted": null - }, - { - "dialect": "duckdb", - "sql": "SELECT LIST_TRANSFORM(LIST_FILTER([0, 1, 2, 3, 4, 5], LAMBDA x : x % 2 = 0), LAMBDA y : y * y)", - "formatted": null - }, - { - "dialect": "duckdb", - "sql": "SELECT LIST_TRANSFORM([5, NULL, 6], LAMBDA x : COALESCE(x, 0) + 1)", - "formatted": null - }, - { - "dialect": "duckdb", - "sql": "SELECT LIST_TRANSFORM(nbr, LAMBDA x : x + 1) FROM article AS a", - "formatted": null - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM my_ducklake.demo AT (VERSION => 2)", - "formatted": "SELECT *\nFROM my_ducklake.demo AT (VERSION => 2)" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM t1 AS a AT (VERSION => 3) JOIN t2 AS b AT (TIMESTAMP => NOW() - INTERVAL '1' WEEK) ON a.id = b.id", - "formatted": "SELECT *\nFROM t1 AS a AT (VERSION => 3)\nJOIN t2 AS b AT (TIMESTAMP => NOW() - INTERVAL '1' WEEK)\n ON a.id = b.id" - }, - { - "dialect": "duckdb", - "sql": "SELECT TO_BINARY('test')", - "formatted": "SELECT TO_BINARY('test')" - }, - { - "dialect": "duckdb", - "sql": "SELECT UUIDV7()", - "formatted": "SELECT UUIDV7()" - }, - { - "dialect": "duckdb", - "sql": "SELECT TRY(LOG(0))", - "formatted": "SELECT TRY(LOG(0))" - }, - { - "dialect": "duckdb", - "sql": "SELECT 1 AS x UNION ALL BY NAME SELECT 2 AS x", - "formatted": "SELECT 1 AS x\nUNION ALL BY NAME\nSELECT 2 AS x" - }, - { - "dialect": "duckdb", - "sql": "SELECT SUM(x) FILTER (x = 1)", - "formatted": "SELECT SUM(x) FILTER(x = 1)" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM GLOB(x)", - "formatted": "SELECT *\nFROM GLOB(x)" - }, - { - "dialect": "duckdb", - "sql": "SELECT MAP(['key1', 'key2', 'key3'], [10, 20, 30])", - "formatted": "SELECT\n MAP(\n ['key1', 'key2', 'key3'],\n [10, 20, 30]\n )" - }, - { - "dialect": "duckdb", - "sql": "SELECT MAP {'x': 1}", - "formatted": "SELECT MAP {'x': 1}" - }, - { - "dialect": "duckdb", - "sql": "SELECT (MAP {'x': 1})['x']", - "formatted": "SELECT\n (\n MAP {'x': 1}\n )['x']" - }, - { - "dialect": "duckdb", - "sql": "SELECT df1.*, df2.* FROM df1 POSITIONAL JOIN df2", - "formatted": "SELECT\n df1.*,\n df2.*\nFROM df1\nPOSITIONAL JOIN df2" - }, - { - "dialect": "duckdb", - "sql": "SELECT EPOCH_MS(10) AS t", - "formatted": "SELECT EPOCH_MS(10) AS t" - }, - { - "dialect": "duckdb", - "sql": "SELECT MAKE_TIMESTAMP(10) AS t", - "formatted": "SELECT MAKE_TIMESTAMP(10) AS t" - }, - { - "dialect": "duckdb", - "sql": "SELECT TO_TIMESTAMP(10) AS t", - "formatted": "SELECT TO_TIMESTAMP(10) AS t" - }, - { - "dialect": "duckdb", - "sql": "SELECT UNNEST(col, recursive := TRUE) FROM t", - "formatted": "SELECT UNNEST(col, recursive := TRUE)\nFROM t" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM foo ASOF LEFT JOIN bar ON a = b", - "formatted": "SELECT *\nFROM foo\nASOF LEFT JOIN bar\n ON a = b" - }, - { - "dialect": "duckdb", - "sql": "SELECT {'a': 1} AS x", - "formatted": "SELECT {'a': 1} AS x" - }, - { - "dialect": "duckdb", - "sql": "SELECT {'a': {'b': {'c': 1}}, 'd': {'e': 2}} AS x", - "formatted": "SELECT {'a': {'b': {'c': 1}}, 'd': {'e': 2}} AS x" - }, - { - "dialect": "duckdb", - "sql": "SELECT {'x': 1, 'y': 2, 'z': 3}", - "formatted": "SELECT {'x': 1, 'y': 2, 'z': 3}" - }, - { - "dialect": "duckdb", - "sql": "SELECT {'key1': 'string', 'key2': 1, 'key3': 12.345}", - "formatted": "SELECT {'key1': 'string', 'key2': 1, 'key3': 12.345}" - }, - { - "dialect": "duckdb", - "sql": "SELECT ROW(x, x + 1, y) FROM (SELECT 1 AS x, 'a' AS y)", - "formatted": "SELECT ROW(x, x + 1, y)\nFROM (\n SELECT\n 1 AS x,\n 'a' AS y\n)" - }, - { - "dialect": "duckdb", - "sql": "SELECT (x, x + 1, y) FROM (SELECT 1 AS x, 'a' AS y)", - "formatted": "SELECT (x, x + 1, y)\nFROM (\n SELECT\n 1 AS x,\n 'a' AS y\n)" - }, - { - "dialect": "duckdb", - "sql": "SELECT a.x FROM (SELECT {'x': 1, 'y': 2, 'z': 3} AS a)", - "formatted": "SELECT a.x\nFROM (\n SELECT {'x': 1, 'y': 2, 'z': 3} AS a\n)" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM t1 WHERE NOT EXISTS(FROM t2 WHERE t2.id = t1.id)", - "formatted": "SELECT *\nFROM t1\nWHERE\n NOT EXISTS(\n FROM t2\n WHERE t2.id = t1.id\n )" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM foo WHERE bar > $baz AND bla = $bob", - "formatted": "SELECT *\nFROM foo\nWHERE\n bar > $baz AND bla = $bob" - }, - { - "dialect": "duckdb", - "sql": "SELECT STAR(tbl, exclude := [foo])", - "formatted": "SELECT STAR(tbl, exclude := [foo])" - }, - { - "dialect": "duckdb", - "sql": "SELECT species, island, COUNT(*) FROM t GROUP BY GROUPING SETS (species), GROUPING SETS (island)", - "formatted": "SELECT\n species,\n island,\n COUNT(*)\nFROM t\nGROUP BY\n GROUPING SETS (\n species\n ),\n GROUPING SETS (\n island\n )" - }, - { - "dialect": "duckdb", - "sql": "SELECT species, island, COUNT(*) FROM t GROUP BY CUBE (species), CUBE (island)", - "formatted": "SELECT\n species,\n island,\n COUNT(*)\nFROM t\nGROUP BY\n CUBE (\n species\n ),\n CUBE (\n island\n )" - }, - { - "dialect": "duckdb", - "sql": "SELECT species, island, COUNT(*) FROM t GROUP BY ROLLUP (species), ROLLUP (island)", - "formatted": "SELECT\n species,\n island,\n COUNT(*)\nFROM t\nGROUP BY\n ROLLUP (\n species\n ),\n ROLLUP (\n island\n )" - }, - { - "dialect": "duckdb", - "sql": "SELECT '{ \"family\": \"anatidae\", \"species\": [ \"duck\", \"goose\", \"swan\", null ] }' ->> ['$.family', '$.species']", - "formatted": "SELECT '{ \"family\": \"anatidae\", \"species\": [ \"duck\", \"goose\", \"swan\", null ] }' ->> ['$.family', '$.species']" - }, - { - "dialect": "duckdb", - "sql": "SELECT $🦆$foo$🦆$", - "formatted": "SELECT $🦆$foo$🦆$" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM t PIVOT(FIRST(t) AS t, FOR quarter IN ('Q1', 'Q2'))", - "formatted": "SELECT *\nFROM t PIVOT(FIRST(t) AS t, FOR quarter IN ('Q1', 'Q2'))" - }, - { - "dialect": "duckdb", - "sql": "SELECT JSON_EXTRACT_STRING('{ \"family\": \"anatidae\", \"species\": [ \"duck\", \"goose\", \"swan\", null ] }', ['$.family', '$.species'])", - "formatted": "SELECT JSON_EXTRACT_STRING('{ \"family\": \"anatidae\", \"species\": [ \"duck\", \"goose\", \"swan\", null ] }', ['$.family', '$.species'])" - }, - { - "dialect": "duckdb", - "sql": "SELECT col FROM t WHERE JSON_EXTRACT_STRING(col, '$.id') NOT IN ('b')", - "formatted": "SELECT col\nFROM t\nWHERE JSON_EXTRACT_STRING(col, '$.id') NOT IN ('b')" - }, - { - "dialect": "duckdb", - "sql": "SELECT a, LOGICAL_OR(b) FROM foo GROUP BY a", - "formatted": "SELECT\n a,\n LOGICAL_OR(b)\nFROM foo\nGROUP BY\n a" - }, - { - "dialect": "duckdb", - "sql": "SELECT JSON_EXTRACT_STRING(c, '$.k1') = 'v1'", - "formatted": "SELECT JSON_EXTRACT_STRING(c, '$.k1') = 'v1'" - }, - { - "dialect": "duckdb", - "sql": "SELECT JSON_EXTRACT(c, '$.k1') = 'v1'", - "formatted": "SELECT JSON_EXTRACT(c, '$.k1') = 'v1'" - }, - { - "dialect": "duckdb", - "sql": "SELECT JSON_EXTRACT(c, '$[*].id')[0:2]", - "formatted": null - }, - { - "dialect": "duckdb", - "sql": "SELECT JSON_EXTRACT_STRING(c, '$[*].id')[0:2]", - "formatted": null - }, - { - "dialect": "duckdb", - "sql": "SELECT '{\"foo\": [1, 2, 3]}' -> 'foo' -> 0", - "formatted": "SELECT '{\"foo\": [1, 2, 3]}' -> 'foo' -> 0" - }, - { - "dialect": "duckdb", - "sql": "SELECT ($$hello)'world$$)", - "formatted": "SELECT\n (\n $$hello)'world$$\n )" - }, - { - "dialect": "duckdb", - "sql": "SELECT $$foo$$", - "formatted": "SELECT $$foo$$" - }, - { - "dialect": "duckdb", - "sql": "SELECT $tag$foo$tag$", - "formatted": "SELECT $tag$foo$tag$" - }, - { - "dialect": "duckdb", - "sql": "SELECT {'yes': 'duck', 'maybe': 'goose', 'huh': NULL, 'no': 'heron'}", - "formatted": "SELECT {'yes': 'duck', 'maybe': 'goose', 'huh': NULL, 'no': 'heron'}" - }, - { - "dialect": "duckdb", - "sql": "SELECT a['x space'] FROM (SELECT {'x space': 1, 'y': 2, 'z': 3} AS a)", - "formatted": "SELECT a['x space']\nFROM (\n SELECT {'x space': 1, 'y': 2, 'z': 3} AS a\n)" - }, - { - "dialect": "duckdb", - "sql": "WITH pivot_alias AS (PIVOT Cities ON Year USING SUM(Population) GROUP BY Country) SELECT * FROM pivot_alias", - "formatted": "WITH pivot_alias AS (\n PIVOT Cities ON Year USING SUM(Population) GROUP BY Country\n)\n\nSELECT *\nFROM pivot_alias" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM (PIVOT Cities ON Year USING SUM(Population) GROUP BY Country) AS pivot_alias", - "formatted": "SELECT *\nFROM (PIVOT Cities ON Year USING SUM(Population) GROUP BY Country) AS pivot_alias" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM cities PIVOT(SUM(population) FOR year IN (2000, 2010, 2020) GROUP BY country)", - "formatted": "SELECT *\nFROM cities PIVOT(SUM(population) FOR year IN (2000, 2010, 2020) GROUP BY country)" - }, - { - "dialect": "duckdb", - "sql": "SELECT schema_name, function_name, ROW_NUMBER() OVER my_window AS function_rank FROM DUCKDB_FUNCTIONS() WINDOW my_window AS (PARTITION BY schema_name ORDER BY function_name) QUALIFY ROW_NUMBER() OVER my_window < 3", - "formatted": "SELECT\n schema_name,\n function_name,\n ROW_NUMBER() OVER my_window AS function_rank\nFROM DUCKDB_FUNCTIONS()\nWINDOW\n my_window AS (PARTITION BY schema_name ORDER BY function_name)\nQUALIFY ROW_NUMBER() OVER my_window < 3" - }, - { - "dialect": "duckdb", - "sql": "SELECT SHA256('abc')", - "formatted": "SELECT SHA256('abc')" - }, - { - "dialect": "duckdb", - "sql": "SELECT SPLIT_PART('11.22.33', '.', 1)", - "formatted": "SELECT SPLIT_PART('11.22.33', '.', 1)" - }, - { - "dialect": "duckdb", - "sql": "SELECT NTH_VALUE(is_deleted, 2) OVER (PARTITION BY id) AS nth_is_deleted FROM my_table", - "formatted": "SELECT NTH_VALUE(is_deleted, 2) OVER (PARTITION BY id) AS nth_is_deleted\nFROM my_table" - }, - { - "dialect": "duckdb", - "sql": "SELECT NTH_VALUE(is_deleted, 2 IGNORE NULLS) OVER (PARTITION BY id) AS nth_is_deleted FROM my_table", - "formatted": "SELECT NTH_VALUE(is_deleted, 2 IGNORE NULLS) OVER (PARTITION BY id) AS nth_is_deleted\nFROM my_table" - }, - { - "dialect": "duckdb", - "sql": "SELECT LIST_MAX(values) FROM table1", - "formatted": "SELECT LIST_MAX(values)\nFROM table1" - }, - { - "dialect": "duckdb", - "sql": "SELECT LIST_MIN(values) FROM table1", - "formatted": "SELECT LIST_MIN(values)\nFROM table1" - }, - { - "dialect": "duckdb", - "sql": "SELECT LIST_SORT(x, 'ASC')", - "formatted": "SELECT LIST_SORT(x, 'ASC')" - }, - { - "dialect": "duckdb", - "sql": "SELECT LIST_SORT(x, 'DESC')", - "formatted": "SELECT LIST_SORT(x, 'DESC')" - }, - { - "dialect": "duckdb", - "sql": "SELECT LIST_SORT(x, 'ASC', 'NULLS FIRST')", - "formatted": "SELECT LIST_SORT(x, 'ASC', 'NULLS FIRST')" - }, - { - "dialect": "duckdb", - "sql": "SELECT LIST_SORT(x, 'ASC', 'NULLS LAST')", - "formatted": "SELECT LIST_SORT(x, 'ASC', 'NULLS LAST')" - }, - { - "dialect": "duckdb", - "sql": "SELECT LIST_SORT(x, 'DESC', 'NULLS FIRST')", - "formatted": "SELECT LIST_SORT(x, 'DESC', 'NULLS FIRST')" - }, - { - "dialect": "duckdb", - "sql": "SELECT LIST_SORT(x, 'DESC', 'NULLS LAST')", - "formatted": "SELECT LIST_SORT(x, 'DESC', 'NULLS LAST')" - }, - { - "dialect": "duckdb", - "sql": "SELECT LIST_SORT(x, 'DE' || 'SC')", - "formatted": "SELECT LIST_SORT(x, 'DE' || 'SC')" - }, - { - "dialect": "duckdb", - "sql": "SELECT LIST_SORT(x, 'DESC', 'NULLS' || ' FIRST')", - "formatted": "SELECT LIST_SORT(x, 'DESC', 'NULLS' || ' FIRST')" - }, - { - "dialect": "duckdb", - "sql": "SELECT LIST_SORT(x, 'DE' || 'SC', 'NULLS' || ' FIRST')", - "formatted": "SELECT\n LIST_SORT(\n x,\n 'DE' || 'SC',\n 'NULLS' || ' FIRST'\n )" - }, - { - "dialect": "duckdb", - "sql": "SELECT PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY y DESC) FROM t", - "formatted": "SELECT PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY y DESC)\nFROM t" - }, - { - "dialect": "duckdb", - "sql": "SELECT PERCENTILE_DISC(0.25) WITHIN GROUP (ORDER BY y DESC) FROM t", - "formatted": "SELECT PERCENTILE_DISC(0.25) WITHIN GROUP (ORDER BY y DESC)\nFROM t" - }, - { - "dialect": "duckdb", - "sql": "SELECT REGEXP_EXTRACT(a, 'pattern', 0)", - "formatted": "SELECT REGEXP_EXTRACT(a, 'pattern', 0)" - }, - { - "dialect": "duckdb", - "sql": "SELECT REGEXP_EXTRACT(a, 'pattern', 0, 'i')", - "formatted": "SELECT REGEXP_EXTRACT(a, 'pattern', 0, 'i')" - }, - { - "dialect": "duckdb", - "sql": "SELECT REGEXP_EXTRACT(a, 'pattern', 1, 'i')", - "formatted": "SELECT REGEXP_EXTRACT(a, 'pattern', 1, 'i')" - }, - { - "dialect": "duckdb", - "sql": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 0)", - "formatted": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 0)" - }, - { - "dialect": "duckdb", - "sql": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 1)", - "formatted": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 1)" - }, - { - "dialect": "duckdb", - "sql": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 0, 'i')", - "formatted": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 0, 'i')" - }, - { - "dialect": "duckdb", - "sql": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 1, 'im')", - "formatted": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 1, 'im')" - }, - { - "dialect": "duckdb", - "sql": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 0)[2:]", - "formatted": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 0)[2:]" - }, - { - "dialect": "duckdb", - "sql": "SELECT ISNAN(x)", - "formatted": "SELECT ISNAN(x)" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM RANGE(1, 5, 10)", - "formatted": "SELECT *\nFROM RANGE(1, 5, 10)" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM GENERATE_SERIES(2, 13, 4)", - "formatted": "SELECT *\nFROM GENERATE_SERIES(2, 13, 4)" - }, - { - "dialect": "duckdb", - "sql": "SELECT i FROM RANGE(5) AS _(i) ORDER BY i ASC", - "formatted": "SELECT i\nFROM RANGE(5) AS _(i)\nORDER BY\n i ASC" - }, - { - "dialect": "duckdb", - "sql": "SELECT i FROM GENERATE_SERIES(12) AS _(i) ORDER BY i ASC", - "formatted": "SELECT i\nFROM GENERATE_SERIES(12) AS _(i)\nORDER BY\n i ASC" - }, - { - "dialect": "duckdb", - "sql": "SELECT LENGTH(foo)", - "formatted": "SELECT LENGTH(foo)" - }, - { - "dialect": "duckdb", - "sql": "SELECT ARRAY[1, 2, 3]", - "formatted": "SELECT ARRAY[1, 2, 3]" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM (DESCRIBE t)", - "formatted": "SELECT *\nFROM (DESCRIBE t)" - }, - { - "dialect": "duckdb", - "sql": "SELECT UNNEST([*COLUMNS('alias_.*')]) AS column_name", - "formatted": "SELECT UNNEST([*COLUMNS('alias_.*')]) AS column_name" - }, - { - "dialect": "duckdb", - "sql": "SELECT COALESCE(*COLUMNS(*)) FROM (SELECT NULL, 2, 3) AS t(a, b, c)", - "formatted": "SELECT COALESCE(*COLUMNS(*))\nFROM (\n SELECT\n NULL,\n 2,\n 3\n) AS t(a, b, c)" - }, - { - "dialect": "duckdb", - "sql": "SELECT id, STRUCT_PACK(*COLUMNS('m\\d')) AS measurements FROM many_measurements", - "formatted": "SELECT\n id,\n STRUCT_PACK(*COLUMNS('m\\d')) AS measurements\nFROM many_measurements" - }, - { - "dialect": "duckdb", - "sql": "SELECT COLUMNS(c -> c LIKE '%num%') FROM numbers", - "formatted": "SELECT COLUMNS(c -> c LIKE '%num%')\nFROM numbers" - }, - { - "dialect": "duckdb", - "sql": "SELECT MIN(COLUMNS(* REPLACE (number + id AS number))), COUNT(COLUMNS(* EXCLUDE (number))) FROM numbers", - "formatted": "SELECT\n MIN(COLUMNS(* REPLACE (number + id AS number))),\n COUNT(COLUMNS(* EXCLUDE (number)))\nFROM numbers" - }, - { - "dialect": "duckdb", - "sql": "SELECT COLUMNS(*) + COLUMNS(*) FROM numbers", - "formatted": "SELECT COLUMNS(*) + COLUMNS(*)\nFROM numbers" - }, - { - "dialect": "duckdb", - "sql": "SELECT COLUMNS('(id|numbers?)') FROM numbers", - "formatted": "SELECT COLUMNS('(id|numbers?)')\nFROM numbers" - }, - { - "dialect": "duckdb", - "sql": "SELECT COALESCE(COLUMNS(['a', 'b', 'c'])) AS result FROM (SELECT NULL AS a, 42 AS b, TRUE AS c)", - "formatted": "SELECT COALESCE(COLUMNS(['a', 'b', 'c'])) AS result\nFROM (\n SELECT\n NULL AS a,\n 42 AS b,\n TRUE AS c\n)" - }, - { - "dialect": "duckdb", - "sql": "SELECT COALESCE(*COLUMNS(['a', 'b', 'c'])) AS result FROM (SELECT NULL AS a, 42 AS b, TRUE AS c)", - "formatted": "SELECT COALESCE(*COLUMNS(['a', 'b', 'c'])) AS result\nFROM (\n SELECT\n NULL AS a,\n 42 AS b,\n TRUE AS c\n)" - }, - { - "dialect": "duckdb", - "sql": "SELECT DATE_ADD(CAST('2020-01-01' AS DATE), INTERVAL 1 DAY)", - "formatted": "SELECT DATE_ADD(CAST('2020-01-01' AS DATE), INTERVAL 1 DAY)" - }, - { - "dialect": "duckdb", - "sql": "SELECT #2, #1 FROM (VALUES (1, 'foo'))", - "formatted": "SELECT\n #2,\n #1\nFROM (VALUES (1, 'foo'))" - }, - { - "dialect": "duckdb", - "sql": "SELECT #2 AS a, #1 AS b FROM (VALUES (1, 'foo'))", - "formatted": "SELECT\n #2 AS a,\n #1 AS b\nFROM (VALUES (1, 'foo'))" - }, - { - "dialect": "duckdb", - "sql": "SELECT CUME_DIST( ORDER BY foo) OVER (ORDER BY 1) FROM (SELECT 1 AS foo)", - "formatted": "SELECT CUME_DIST(ORDER BY foo) OVER (ORDER BY 1)\nFROM (\n SELECT 1 AS foo\n)" - }, - { - "dialect": "duckdb", - "sql": "SELECT NTILE(1 ORDER BY foo) OVER (ORDER BY 1) FROM (SELECT 1 AS foo)", - "formatted": "SELECT NTILE(1 ORDER BY foo) OVER (ORDER BY 1)\nFROM (\n SELECT 1 AS foo\n)" - }, - { - "dialect": "duckdb", - "sql": "SELECT RANK( ORDER BY foo) OVER (ORDER BY 1) FROM (SELECT 1 AS foo)", - "formatted": "SELECT RANK(ORDER BY foo) OVER (ORDER BY 1)\nFROM (\n SELECT 1 AS foo\n)" - }, - { - "dialect": "duckdb", - "sql": "SELECT PERCENT_RANK( ORDER BY foo) OVER (ORDER BY 1) FROM (SELECT 1 AS foo)", - "formatted": "SELECT PERCENT_RANK(ORDER BY foo) OVER (ORDER BY 1)\nFROM (\n SELECT 1 AS foo\n)" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM t LIMIT 10 PERCENT", - "formatted": "SELECT *\nFROM t\nLIMIT 10 PERCENT" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM t LIMIT 10%", - "formatted": "SELECT *\nFROM t\nLIMIT 10%", - "previously_formatted": "SELECT\n *\nFROM t\nLIMIT 10 %" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM t LIMIT 10 PERCENT OFFSET 1", - "formatted": "SELECT *\nFROM t\nLIMIT 10 PERCENT\nOFFSET 1" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM t LIMIT 10% OFFSET 1", - "formatted": "SELECT *\nFROM t\nLIMIT 10%\nOFFSET 1", - "previously_formatted": "SELECT\n *\nFROM t\nLIMIT 10 %\nOFFSET 1" - }, - { - "dialect": "duckdb", - "sql": "SELECT CAST(ROW(1, 2) AS ROW(a INTEGER, b INTEGER))", - "formatted": "SELECT CAST(ROW(1, 2) AS ROW(a INTEGER, b INTEGER))" - }, - { - "dialect": "duckdb", - "sql": "SELECT row", - "formatted": "SELECT row" - }, - { - "dialect": "duckdb", - "sql": "SELECT TRY_STRPTIME('2013-04-28T20:57:01.123456789+07:00', '%Y-%m-%dT%H:%M:%S.%n%z')", - "formatted": "SELECT TRY_STRPTIME('2013-04-28T20:57:01.123456789+07:00', '%Y-%m-%dT%H:%M:%S.%n%z')" - }, - { - "dialect": "duckdb", - "sql": "SELECT REPLACE('apple pie', 'pie', 'cobbler') AS result", - "formatted": "SELECT REPLACE('apple pie', 'pie', 'cobbler') AS result" - }, - { - "dialect": "duckdb", - "sql": "SELECT REPLACE(CAST(CAST('apple pie' AS BLOB) AS TEXT), CAST(CAST('pie' AS BLOB) AS TEXT), CAST(CAST('cobbler' AS BLOB) AS TEXT)) AS result", - "formatted": "SELECT\n REPLACE(\n CAST(CAST('apple pie' AS BLOB) AS TEXT),\n CAST(CAST('pie' AS BLOB) AS TEXT),\n CAST(CAST('cobbler' AS BLOB) AS TEXT)\n ) AS result" - }, - { - "dialect": "duckdb", - "sql": "SELECT TRIM('***apple***', '*') AS result", - "formatted": "SELECT TRIM('***apple***', '*') AS result" - }, - { - "dialect": "duckdb", - "sql": "SELECT CAST(TRIM(CAST(CAST('***apple***' AS BLOB) AS TEXT), CAST(CAST('*' AS BLOB) AS TEXT)) AS BLOB) AS result", - "formatted": "SELECT CAST(TRIM(CAST(CAST('***apple***' AS BLOB) AS TEXT), CAST(CAST('*' AS BLOB) AS TEXT)) AS BLOB) AS result" - }, - { - "dialect": "duckdb", - "sql": "SELECT GREATEST(1.0, 2.5, NULL, 3.7)", - "formatted": "SELECT GREATEST(1.0, 2.5, NULL, 3.7)" - }, - { - "dialect": "duckdb", - "sql": "SELECT MAKE_DATE(DATE_PART(['year', 'month', 'day'], CURRENT_DATE))", - "formatted": "SELECT\n MAKE_DATE(\n DATE_PART(\n ['year', 'month', 'day'],\n CURRENT_DATE\n )\n )" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM t PIVOT(SUM(y) FOR foo IN y_enum)", - "formatted": "SELECT *\nFROM t PIVOT(SUM(y) FOR foo IN y_enum)" - }, - { - "dialect": "duckdb", - "sql": "SELECT 20_000 AS literal", - "formatted": "SELECT 20_000 AS literal" - }, - { - "dialect": "duckdb", - "sql": "SELECT 1_2E+1_0::FLOAT", - "formatted": "SELECT 1_2E+1_0::FLOAT" - }, - { - "dialect": "duckdb", - "sql": "SELECT [1, 2, 3][1 + 1:LENGTH([1, 2, 3]) + -1]", - "formatted": null - }, - { - "dialect": "duckdb", - "sql": "SELECT TODAY()", - "formatted": "SELECT TODAY()" - }, - { - "dialect": "duckdb", - "sql": "SELECT GET_CURRENT_TIME()", - "formatted": "SELECT GET_CURRENT_TIME()" - }, - { - "dialect": "duckdb", - "sql": "SELECT SUM(x) OVER (ORDER BY x GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) FROM t", - "formatted": "SELECT SUM(x) OVER (ORDER BY x GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW)\nFROM t" - }, - { - "dialect": "duckdb", - "sql": "SELECT LAST_VALUE(x ORDER BY x IGNORE NULLS) OVER (ORDER BY x) FROM t", - "formatted": "SELECT LAST_VALUE(x ORDER BY x IGNORE NULLS) OVER (ORDER BY x)\nFROM t" - }, - { - "dialect": "duckdb", - "sql": "SELECT LAST_VALUE(x ORDER BY x RESPECT NULLS) OVER (ORDER BY x) FROM t", - "formatted": "SELECT LAST_VALUE(x ORDER BY x RESPECT NULLS) OVER (ORDER BY x)\nFROM t" - }, - { - "dialect": "duckdb", - "sql": "SELECT CURRENT_DATE", - "formatted": "SELECT CURRENT_DATE" - }, - { - "dialect": "duckdb", - "sql": "SELECT CURRENT_TIMESTAMP", - "formatted": "SELECT CURRENT_TIMESTAMP" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM tbl USING SAMPLE 5", - "formatted": "SELECT *\nFROM tbl\nUSING SAMPLE 5" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM tbl USING SAMPLE 10%", - "formatted": "SELECT *\nFROM tbl\nUSING SAMPLE 10%" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM tbl USING SAMPLE 10 PERCENT (bernoulli)", - "formatted": "SELECT *\nFROM tbl\nUSING SAMPLE 10 PERCENT (bernoulli)" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM tbl USING SAMPLE reservoir(50 ROWS) REPEATABLE (100)", - "formatted": "SELECT *\nFROM tbl\nUSING SAMPLE reservoir (50 ROWS) REPEATABLE (100)" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM tbl USING SAMPLE 10% (system, 377)", - "formatted": "SELECT *\nFROM tbl\nUSING SAMPLE 10% (system, 377)" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM tbl TABLESAMPLE RESERVOIR(20%), tbl2 WHERE tbl.i=tbl2.i", - "formatted": "SELECT *\nFROM tbl TABLESAMPLE RESERVOIR (20%), tbl2\nWHERE tbl.i = tbl2.i", - "previously_formatted": "SELECT\n *\nFROM tbl TABLESAMPLE RESERVOIR (20 %), tbl2\nWHERE\n tbl.i = tbl2.i" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM tbl, tbl2 WHERE tbl.i=tbl2.i USING SAMPLE RESERVOIR(20%)", - "formatted": "SELECT *\nFROM tbl, tbl2\nWHERE tbl.i = tbl2.i\nUSING SAMPLE RESERVOIR (20%)", - "previously_formatted": "SELECT\n *\nFROM tbl, tbl2\nWHERE\n tbl.i = tbl2.i\nUSING SAMPLE RESERVOIR (20 %)" - }, - { - "dialect": "duckdb", - "sql": "SELECT x::INT[3][3]", - "formatted": null - }, - { - "dialect": "duckdb", - "sql": "SELECT ARRAY[[[1]]]::INT[1][1][1]", - "formatted": null - }, - { - "dialect": "duckdb", - "sql": "WITH unpivot_alias AS (UNPIVOT monthly_sales ON COLUMNS(* EXCLUDE (empid, dept)) INTO NAME month VALUE sales) SELECT * FROM unpivot_alias", - "formatted": "WITH unpivot_alias AS (\n UNPIVOT monthly_sales ON COLUMNS(* EXCLUDE (empid, dept)) INTO NAME month VALUE sales\n)\n\nSELECT *\nFROM unpivot_alias" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM (UNPIVOT monthly_sales ON COLUMNS(* EXCLUDE (empid, dept)) INTO NAME month VALUE sales) AS unpivot_alias", - "formatted": "SELECT *\nFROM (UNPIVOT monthly_sales ON COLUMNS(* EXCLUDE (empid, dept)) INTO NAME month VALUE sales) AS unpivot_alias" - }, - { - "dialect": "duckdb", - "sql": "WITH cities(country, name, year, population) AS (SELECT 'NL', 'Amsterdam', 2000, 1005 UNION ALL SELECT 'US', 'Seattle', 2020, 738) PIVOT cities ON year USING SUM(population)", - "formatted": "WITH cities(country, name, year, population) AS (\n SELECT\n 'NL',\n 'Amsterdam',\n 2000,\n 1005\n UNION ALL\n SELECT\n 'US',\n 'Seattle',\n 2020,\n 738\n)\n\nPIVOT cities ON year USING SUM(population)" - }, - { - "dialect": "duckdb", - "sql": "WITH t1 AS (FROM (FROM t2 SELECT foo1, foo2)) FROM t1", - "formatted": "WITH t1 AS (\n FROM (FROM t2\n SELECT\n foo1,\n foo2\n )\n)\n\nFROM t1" - }, - { - "dialect": "duckdb", - "sql": "SELECT foo: 1", - "formatted": "SELECT foo:1" - }, - { - "dialect": "duckdb", - "sql": "SELECT foo: bar", - "formatted": "SELECT foo:bar" - }, - { - "dialect": "duckdb", - "sql": "SELECT foo: t.col FROM t", - "formatted": "SELECT foo:t.col\nFROM t" - }, - { - "dialect": "duckdb", - "sql": "SELECT \"foo\" /* bla */: 1", - "formatted": "SELECT\n \"foo\" /* bla */:1" - }, - { - "dialect": "duckdb", - "sql": "SELECT \"foo\": 1 /* bla */", - "formatted": "SELECT \"foo\":1 /* bla */" - }, - { - "dialect": "duckdb", - "sql": "SELECT \"foo\": /* bla */ 1", - "formatted": "SELECT\n \"foo\": /* bla */1" - }, - { - "dialect": "duckdb", - "sql": "SELECT \"foo\": /* bla */ 1 /* foo */", - "formatted": "SELECT\n \"foo\": /* bla */1 /* foo */" - }, - { - "dialect": "duckdb", - "sql": "SELECT \"foo\": 1", - "formatted": "SELECT \"foo\":1" - }, - { - "dialect": "duckdb", - "sql": "SELECT foo: 1, bar: 2, baz: 3", - "formatted": "SELECT\n foo:1,\n bar:2,\n baz:3" - }, - { - "dialect": "duckdb", - "sql": "SELECT e: 1 + 2, f: len('asdf'), s: (SELECT 42)", - "formatted": "SELECT\n e:1 + 2,\n f:len('asdf'), s:\n (\n SELECT 42\n )" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM foo: bar", - "formatted": "SELECT *\nFROM foo:bar" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM foo: c.db.tbl", - "formatted": "SELECT *\nFROM foo:c.db.tbl" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM foo /* bla */: bar", - "formatted": "SELECT *\nFROM foo /* bla */:bar" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM foo /* bla */: bar /* baz */", - "formatted": "SELECT *\nFROM foo /* bla */:bar /* baz */" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM foo /* bla */: /* baz */ bar /* boo */", - "formatted": "SELECT *\nFROM foo /* bla */: /* baz */bar /* boo */" - }, - { - "dialect": "duckdb", - "sql": "SELECT * FROM r: range(10), v: (VALUES (42)), s: (FROM range(10))", - "formatted": "SELECT *\nFROM r:range(10), v:(VALUES (42)), s:(\n FROM RANGE(10)\n)" - }, - { - "dialect": "duckdb", - "sql": "\n SELECT\n l_returnflag,\n l_linestatus,\n sum_qty: sum(l_quantity),\n sum_base_price: sum(l_extendedprice),\n sum_disc_price: sum(l_extendedprice * (1-l_discount)),\n sum_charge: sum(l_extendedprice * (1-l_discount) * (1+l_tax)),\n avg_qty: avg(l_quantity),\n avg_price: avg(l_extendedprice),\n avg_disc: avg(l_discount),\n count_order: count(*)\n ", - "formatted": "SELECT\n l_returnflag,\n l_linestatus,\n sum_qty:sum(l_quantity),\n sum_base_price:sum(l_extendedprice),\n sum_disc_price:sum(l_extendedprice\n * (\n 1 - l_discount\n )),\n sum_charge:sum(l_extendedprice\n * (\n 1 - l_discount\n )\n * (\n 1 + l_tax\n )),\n avg_qty:avg(l_quantity),\n avg_price:avg(l_extendedprice),\n avg_disc:avg(l_discount),\n count_order:count(*)" - }, - { - "dialect": "duckdb", - "sql": "SELECT @col FROM t", - "formatted": "SELECT @col\nFROM t" - }, - { - "dialect": "duckdb", - "sql": "SELECT @col + 1 FROM t", - "formatted": "SELECT @col + 1\nFROM t" - }, - { - "dialect": "duckdb", - "sql": "SELECT (@col) + 1 FROM t", - "formatted": "SELECT\n (@col\n )\n + 1\nFROM t" - }, - { - "dialect": "duckdb", - "sql": "SELECT @(-1)", - "formatted": "SELECT\n @(\n -1\n )" - }, - { - "dialect": "duckdb", - "sql": "SELECT @(-1) + 1", - "formatted": "SELECT\n @(\n -1\n )\n + 1" - }, - { - "dialect": "duckdb", - "sql": "SELECT (@-1) + 1", - "formatted": "SELECT\n (@-1\n )\n + 1" - }, - { - "dialect": "duckdb", - "sql": "WITH RECURSIVE tbl(a, b) USING KEY (a) AS (SELECT a, b FROM (VALUES (1, 3), (2, 4)) AS t(a, b) UNION SELECT a + 1, b FROM tbl WHERE a < 3) SELECT * FROM tbl", - "formatted": "WITH RECURSIVE tbl(a, b) USING KEY (a) AS (\n SELECT\n a,\n b\n FROM (VALUES (1, 3), (2, 4)) AS t(a, b)\n UNION\n SELECT\n a + 1,\n b\n FROM tbl\n WHERE a < 3\n)\nSELECT *\nFROM tbl" - }, - { - "dialect": "duckdb", - "sql": "WITH RECURSIVE tbl(a, b) USING KEY (a, b) AS (SELECT a, b FROM (VALUES (1, 3), (2, 4)) AS t(a, b) UNION SELECT a + 1, b FROM tbl WHERE a < 3) SELECT * FROM tbl", - "formatted": "WITH RECURSIVE tbl(a, b) USING KEY (a, b) AS (\n SELECT\n a,\n b\n FROM (VALUES (1, 3), (2, 4)) AS t(a, b)\n UNION\n SELECT\n a + 1,\n b\n FROM tbl\n WHERE a < 3\n)\nSELECT *\nFROM tbl" - }, - { - "dialect": "duckdb", - "sql": "WITH _data AS (SELECT [{'a': 1, 'b': 2}, {'a': 2, 'b': 3}] AS col) SELECT t.col['b'] FROM _data, UNNEST(_data.col) AS t(col) WHERE t.col['a'] = 1", - "formatted": "WITH _data AS (\n SELECT\n [\n {'a': 1, 'b': 2},\n {'a': 2, 'b': 3}\n ] AS col\n)\n\nSELECT t.col['b']\nFROM _data, UNNEST(_data.col) AS t(col)\nWHERE t.col['a'] = 1" - }, - { - "dialect": "duckdb", - "sql": "SELECT [4, 5, 6] AS l, [x FOR x, i IN l IF i = 2] AS filtered", - "formatted": "SELECT\n [4, 5, 6] AS l,\n [x FOR x, i IN l IF i = 2] AS filtered" - }, - { - "dialect": "duckdb", - "sql": "SELECT LIST_VALUE(1)[i]", - "formatted": "SELECT LIST_VALUE(1)[i]" - }, - { - "dialect": "duckdb", - "sql": "SELECT LIST_APPLY(RANGE(1, 4), i -> {'f1': LIST_VALUE(1, 2, 3)[i], 'f2': LIST_VALUE(1, 2, 3)[i]})", - "formatted": "SELECT\n LIST_APPLY(\n RANGE(1, 4),\n i -> {'f1': LIST_VALUE(1, 2, 3)[i], 'f2': LIST_VALUE(1, 2, 3)[i]}\n )" - }, - { - "dialect": "duckdb", - "sql": "SELECT UNNEST([1, 2])", - "formatted": "SELECT UNNEST([1, 2])" - }, - { - "dialect": "duckdb", - "sql": "SELECT file[:256] FROM GLOB('*')", - "formatted": null - }, - { - "dialect": "duckdb", - "sql": "SELECT file[256] FROM GLOB('*')", - "formatted": "SELECT file[256]\nFROM GLOB('*')" - }, - { - "dialect": "postgres", - "sql": "WITH t AS (SELECT 1 AS \"null\") SELECT t.null FROM t", - "formatted": "WITH t AS (\n SELECT 1 AS \"null\"\n)\n\nSELECT t.null\nFROM t" - }, - { - "dialect": "postgres", - "sql": "WITH t AS (SELECT 1 AS \"true\") SELECT t.true FROM t", - "formatted": "WITH t AS (\n SELECT 1 AS \"true\"\n)\n\nSELECT t.true\nFROM t" - }, - { - "dialect": "postgres", - "sql": "SELECT '%' SIMILAR TO '^%' ESCAPE '^'", - "formatted": "SELECT '%' SIMILAR TO '^%' ESCAPE '^'" - }, - { - "dialect": "postgres", - "sql": "SELECT GET_BIT(CAST(44 AS BIT(10)), 6)", - "formatted": "SELECT GET_BIT(CAST(44 AS BIT(10)), 6)" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM t GROUP BY ROLLUP (a || '^' || b)", - "formatted": "SELECT *\nFROM t\nGROUP BY\n ROLLUP (\n a || '^' || b\n )" - }, - { - "dialect": "postgres", - "sql": "SELECT COSH(1.5)", - "formatted": "SELECT COSH(1.5)" - }, - { - "dialect": "postgres", - "sql": "SELECT EXP(1)", - "formatted": "SELECT EXP(1)" - }, - { - "dialect": "postgres", - "sql": "SELECT MODE() WITHIN GROUP (ORDER BY status DESC) AS most_common FROM orders", - "formatted": "SELECT MODE() WITHIN GROUP (ORDER BY status DESC) AS most_common\nFROM orders" - }, - { - "dialect": "postgres", - "sql": "SELECT ST_DISTANCE(gg1, gg2, FALSE) AS sphere_dist", - "formatted": "SELECT ST_DISTANCE(gg1, gg2, FALSE) AS sphere_dist" - }, - { - "dialect": "postgres", - "sql": "SELECT EXTRACT(QUARTER FROM CAST('2025-04-26' AS DATE))", - "formatted": "SELECT EXTRACT(QUARTER FROM CAST('2025-04-26' AS DATE))" - }, - { - "dialect": "postgres", - "sql": "SELECT DATE_TRUNC('QUARTER', CAST('2025-04-26' AS DATE))", - "formatted": "SELECT DATE_TRUNC('QUARTER', CAST('2025-04-26' AS DATE))" - }, - { - "dialect": "postgres", - "sql": "SELECT x FROM t WHERE CAST($1 AS TEXT) = 'ok'", - "formatted": "SELECT x\nFROM t\nWHERE CAST($1 AS TEXT) = 'ok'" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM t TABLESAMPLE SYSTEM (50) REPEATABLE (55)", - "formatted": "SELECT *\nFROM t TABLESAMPLE SYSTEM (50) REPEATABLE (55)" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM events WHERE doc @? '$.a[*] ? (@ > 2)'", - "formatted": "SELECT *\nFROM events\nWHERE doc @? '$.a[*] ? (@ > 2)'" - }, - { - "dialect": "postgres", - "sql": "SELECT ARRAY[1, 2, 3]", - "formatted": "SELECT ARRAY[1, 2, 3]" - }, - { - "dialect": "postgres", - "sql": "SELECT ARRAY(SELECT 1)", - "formatted": "SELECT\n ARRAY(SELECT\n 1)" - }, - { - "dialect": "postgres", - "sql": "SELECT CASE WHEN SUBSTRING('abcdefg') IN ('ab') THEN 1 ELSE 0 END", - "formatted": "SELECT\n CASE\n WHEN SUBSTRING('abcdefg') IN ('ab')\n THEN 1\n ELSE 0\n END" - }, - { - "dialect": "postgres", - "sql": "SELECT e'\\xDEADBEEF'", - "formatted": "SELECT e'\\xDEADBEEF'" - }, - { - "dialect": "postgres", - "sql": "SELECT CAST(e'\\176' AS BYTEA)", - "formatted": "SELECT CAST(e'\\176' AS BYTEA)" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM x WHERE SUBSTRING('Thomas' FROM '...$') IN ('mas')", - "formatted": "SELECT *\nFROM x\nWHERE SUBSTRING('Thomas' FROM '...$') IN ('mas')" - }, - { - "dialect": "postgres", - "sql": "SELECT TRIM(' X' FROM ' XXX ')", - "formatted": "SELECT TRIM(' X' FROM ' XXX ')" - }, - { - "dialect": "postgres", - "sql": "SELECT TRIM(LEADING 'bla' FROM ' XXX ' COLLATE utf8_bin)", - "formatted": "SELECT TRIM(LEADING 'bla' FROM ' XXX ' COLLATE utf8_bin)" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM JSON_TO_RECORDSET(z) AS y(\"rank\" INT)", - "formatted": "SELECT *\nFROM JSON_TO_RECORDSET(z) AS y(\"rank\" INT)" - }, - { - "dialect": "postgres", - "sql": "SELECT ~x", - "formatted": "SELECT ~x" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM r CROSS JOIN LATERAL UNNEST(ARRAY[1]) AS s(location)", - "formatted": "SELECT *\nFROM r\nCROSS JOIN LATERAL UNNEST(ARRAY[1]) AS s(location)" - }, - { - "dialect": "postgres", - "sql": "SELECT CURRENT_SCHEMA", - "formatted": "SELECT CURRENT_SCHEMA" - }, - { - "dialect": "postgres", - "sql": "SELECT CURRENT_USER", - "formatted": "SELECT CURRENT_USER" - }, - { - "dialect": "postgres", - "sql": "SELECT CURRENT_ROLE", - "formatted": "SELECT CURRENT_ROLE" - }, - { - "dialect": "postgres", - "sql": "SELECT VERSION()", - "formatted": "SELECT VERSION()" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM ONLY t1", - "formatted": "SELECT *\nFROM ONLY t1" - }, - { - "dialect": "postgres", - "sql": "SELECT INTERVAL '-1 MONTH'", - "formatted": "SELECT INTERVAL '-1 MONTH'" - }, - { - "dialect": "postgres", - "sql": "SELECT INTERVAL '4.1 DAY'", - "formatted": "SELECT INTERVAL '4.1 DAY'" - }, - { - "dialect": "postgres", - "sql": "SELECT INTERVAL '3.14159 HOUR'", - "formatted": "SELECT INTERVAL '3.14159 HOUR'" - }, - { - "dialect": "postgres", - "sql": "SELECT INTERVAL '2.5 MONTH'", - "formatted": "SELECT INTERVAL '2.5 MONTH'" - }, - { - "dialect": "postgres", - "sql": "SELECT INTERVAL '-10.75 MINUTE'", - "formatted": "SELECT INTERVAL '-10.75 MINUTE'" - }, - { - "dialect": "postgres", - "sql": "SELECT INTERVAL '0.123456789 SECOND'", - "formatted": "SELECT INTERVAL '0.123456789 SECOND'" - }, - { - "dialect": "postgres", - "sql": "SELECT date_col - INTERVAL '30' FROM t", - "formatted": "SELECT date_col - INTERVAL '30'\nFROM t" - }, - { - "dialect": "postgres", - "sql": "SELECT date_col - INTERVAL '1' AS one_second_later", - "formatted": "SELECT date_col - INTERVAL '1' AS one_second_later" - }, - { - "dialect": "postgres", - "sql": "SELECT date_col - INTERVAL '30' DAY FROM t", - "formatted": null - }, - { - "dialect": "postgres", - "sql": "SELECT date_col - INTERVAL '1' HOUR AS one_hour_later", - "formatted": null - }, - { - "dialect": "postgres", - "sql": "SELECT SUM(x) OVER (PARTITION BY y ORDER BY interval ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) - SUM(x) OVER (PARTITION BY y ORDER BY interval ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS total", - "formatted": "SELECT SUM(x) OVER (PARTITION BY y ORDER BY interval ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) - SUM(x) OVER (PARTITION BY y ORDER BY interval ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS total" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM test_data, LATERAL JSONB_ARRAY_ELEMENTS(data) WITH ORDINALITY AS elem(value, ordinality)", - "formatted": "SELECT *\nFROM test_data, LATERAL JSONB_ARRAY_ELEMENTS(data) WITH ORDINALITY AS elem(value, ordinality)" - }, - { - "dialect": "postgres", - "sql": "SELECT id, name FROM xml_data AS t, XMLTABLE('/root/user' PASSING t.xml COLUMNS id INT PATH '@id', name TEXT PATH 'name/text()') AS x", - "formatted": "SELECT\n id,\n name\nFROM xml_data AS t, XMLTABLE(\n '/root/user'\n PASSING\n t.xml\n COLUMNS\n id INT PATH '@id',\n name TEXT PATH 'name/text()'\n) AS x" - }, - { - "dialect": "postgres", - "sql": "SELECT id, value FROM xml_content AS t, XMLTABLE(XMLNAMESPACES('http://example.com/ns1' AS ns1, 'http://example.com/ns2' AS ns2), '/root/data' PASSING t.xml COLUMNS id INT PATH '@ns1:id', value TEXT PATH 'ns2:value/text()') AS x", - "formatted": null - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM t WHERE some_column >= CURRENT_DATE + INTERVAL '1 day 1 hour' AND some_another_column IS TRUE", - "formatted": "SELECT *\nFROM t\nWHERE\n some_column >= CURRENT_DATE + INTERVAL '1 day 1 hour' AND some_another_column IS TRUE" - }, - { - "dialect": "postgres", - "sql": "WITH t1 AS MATERIALIZED (SELECT 1), t2 AS NOT MATERIALIZED (SELECT 2) SELECT * FROM t1, t2", - "formatted": "WITH t1 AS MATERIALIZED (\n SELECT 1\n),\n\nt2 AS NOT MATERIALIZED (\n SELECT 2\n)\n\nSELECT *\nFROM t1, t2" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM JSON_ARRAY_ELEMENTS('[1,true, [2,false]]') WITH ORDINALITY", - "formatted": "SELECT *\nFROM JSON_ARRAY_ELEMENTS('[1,true, [2,false]]') WITH ORDINALITY" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM JSON_ARRAY_ELEMENTS('[1,true, [2,false]]') WITH ORDINALITY AS kv_json", - "formatted": "SELECT *\nFROM JSON_ARRAY_ELEMENTS('[1,true, [2,false]]') WITH ORDINALITY AS kv_json" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM JSON_ARRAY_ELEMENTS('[1,true, [2,false]]') WITH ORDINALITY AS kv_json(a, b)", - "formatted": "SELECT *\nFROM JSON_ARRAY_ELEMENTS('[1,true, [2,false]]') WITH ORDINALITY AS kv_json(a, b)" - }, - { - "dialect": "postgres", - "sql": "SELECT SUM(x) OVER a, SUM(y) OVER b FROM c WINDOW a AS (PARTITION BY d), b AS (PARTITION BY e)", - "formatted": "SELECT\n SUM(x) OVER a,\n SUM(y) OVER b\nFROM c\nWINDOW\n a AS (PARTITION BY d),\n b AS (PARTITION BY e)" - }, - { - "dialect": "postgres", - "sql": "SELECT CASE WHEN SUBSTRING('abcdefg' FROM 1) IN ('ab') THEN 1 ELSE 0 END", - "formatted": "SELECT\n CASE\n WHEN SUBSTRING('abcdefg' FROM 1) IN ('ab')\n THEN 1\n ELSE 0\n END" - }, - { - "dialect": "postgres", - "sql": "SELECT CASE WHEN SUBSTRING('abcdefg' FROM 1 FOR 2) IN ('ab') THEN 1 ELSE 0 END", - "formatted": "SELECT\n CASE\n WHEN SUBSTRING('abcdefg' FROM 1 FOR 2) IN ('ab')\n THEN 1\n ELSE 0\n END" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM \"x\" WHERE SUBSTRING(\"x\".\"foo\" FROM 1 FOR 2) IN ('mas')", - "formatted": "SELECT *\nFROM \"x\"\nWHERE SUBSTRING(\"x\".\"foo\" FROM 1 FOR 2) IN ('mas')" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM x WHERE SUBSTRING('Thomas' FROM '%#\"o_a#\"_' FOR '#') IN ('mas')", - "formatted": "SELECT *\nFROM x\nWHERE SUBSTRING('Thomas' FROM '%#\"o_a#\"_' FOR '#') IN ('mas')" - }, - { - "dialect": "postgres", - "sql": "SELECT SUBSTRING('bla' + 'foo' || 'bar' FROM 3 - 1 + 5 FOR 4 + SOME_FUNC(arg1, arg2))", - "formatted": "SELECT SUBSTRING('bla' + 'foo' || 'bar' FROM 3 - 1 + 5 FOR 4 + SOME_FUNC(arg1, arg2))" - }, - { - "dialect": "postgres", - "sql": "SELECT TO_TIMESTAMP(1284352323.5), TO_TIMESTAMP('05 Dec 2000', 'DD Mon YYYY')", - "formatted": "SELECT\n TO_TIMESTAMP(1284352323.5),\n TO_TIMESTAMP('05 Dec 2000', 'DD Mon YYYY')" - }, - { - "dialect": "postgres", - "sql": "SELECT TO_TIMESTAMP('05 Dec 2000 10:00 AM', 'DD Mon YYYY HH:MI AM')", - "formatted": "SELECT TO_TIMESTAMP('05 Dec 2000 10:00 AM', 'DD Mon YYYY HH:MI AM')" - }, - { - "dialect": "postgres", - "sql": "SELECT TO_TIMESTAMP('05 Dec 2000 10:00 PM', 'DD Mon YYYY HH:MI PM')", - "formatted": "SELECT TO_TIMESTAMP('05 Dec 2000 10:00 PM', 'DD Mon YYYY HH:MI PM')" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM foo, LATERAL (SELECT * FROM bar WHERE bar.id = foo.bar_id) AS ss", - "formatted": "SELECT *\nFROM foo, LATERAL (\n SELECT *\n FROM bar\n WHERE bar.id = foo.bar_id\n) AS ss" - }, - { - "dialect": "postgres", - "sql": "SELECT c.oid, n.nspname, c.relname FROM pg_catalog.pg_class AS c LEFT JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace WHERE c.relname OPERATOR(pg_catalog.~) '^(courses)$' COLLATE pg_catalog.default AND pg_catalog.PG_TABLE_IS_VISIBLE(c.oid) ORDER BY 2, 3", - "formatted": "SELECT\n c.oid,\n n.nspname,\n c.relname\nFROM pg_catalog.pg_class AS c\nLEFT JOIN pg_catalog.pg_namespace AS n\n ON n.oid = c.relnamespace\nWHERE\n c.relname OPERATOR(pg_catalog.~) '^(courses)$' COLLATE pg_catalog.default AND pg_catalog.PG_TABLE_IS_VISIBLE(c.oid)\nORDER BY\n 2,\n 3" - }, - { - "dialect": "postgres", - "sql": "SELECT date_add(current_date, interval '7' day)", - "formatted": null - }, - { - "dialect": "postgres", - "sql": "SELECT e'foo \\' bar'", - "formatted": "SELECT e'foo \\' bar'" - }, - { - "dialect": "postgres", - "sql": "SELECT e'\\n'", - "formatted": "SELECT e'\\n'" - }, - { - "dialect": "postgres", - "sql": "SELECT e'\\t'", - "formatted": "SELECT e'\\t'" - }, - { - "dialect": "postgres", - "sql": "SELECT e'update table_name set a = \\'foo\\' where 1 = 0' AS x FROM tab", - "formatted": "SELECT e'update table_name set a = \\'foo\\' where 1 = 0' AS x\nFROM tab" - }, - { - "dialect": "postgres", - "sql": "select count() OVER(partition by a order by a range offset preceding exclude current row)", - "formatted": "SELECT COUNT() OVER (PARTITION BY a ORDER BY a range offset preceding exclude CURRENT ROW)" - }, - { - "dialect": "postgres", - "sql": "SELECT SUBSTRING('Thomas' FOR 3 FROM 2)", - "formatted": "SELECT SUBSTRING('Thomas' FOR 3 FROM 2)" - }, - { - "dialect": "postgres", - "sql": "SELECT DATE_PART('isodow'::varchar(6), current_date)", - "formatted": "SELECT DATE_PART('isodow'::varchar(6), CURRENT_DATE)" - }, - { - "dialect": "postgres", - "sql": "SELECT $$a$$", - "formatted": "SELECT $$a$$" - }, - { - "dialect": "postgres", - "sql": "SELECT $$Dianne's order$$", - "formatted": "SELECT $$Dianne's order$$" - }, - { - "dialect": "postgres", - "sql": "SELECT $$The price is $9.95$$ AS msg", - "formatted": "SELECT $$The price is $9.95$$ AS msg" - }, - { - "dialect": "postgres", - "sql": "SELECT 'testa 1' NOT LIKE ALL (ARRAY['testa%', 'testb%'])", - "formatted": "SELECT\n 'testa 1' NOT LIKE ALL (\n ARRAY['testa%', 'testb%']\n )" - }, - { - "dialect": "postgres", - "sql": "SELECT 'testa 1' NOT LIKE ANY(ARRAY['testa%', 'testb%'])", - "formatted": "SELECT\n 'testa 1' NOT LIKE ANY(\n ARRAY['testa%', 'testb%']\n )" - }, - { - "dialect": "postgres", - "sql": "SELECT 'testa 1' NOT ILIKE ALL (ARRAY['testa%', 'testb%'])", - "formatted": "SELECT\n 'testa 1' NOT ILIKE ALL (\n ARRAY['testa%', 'testb%']\n )" - }, - { - "dialect": "postgres", - "sql": "SELECT 'testa 1' NOT ILIKE ANY(ARRAY['testa%', 'testb%'])", - "formatted": "SELECT\n 'testa 1' NOT ILIKE ANY(\n ARRAY['testa%', 'testb%']\n )" - }, - { - "dialect": "postgres", - "sql": "SELECT NOT 'testa 1' LIKE ALL (ARRAY['testa%', 'testb%'])", - "formatted": "SELECT\n NOT 'testa 1' LIKE ALL (\n ARRAY['testa%', 'testb%']\n )" - }, - { - "dialect": "postgres", - "sql": "SELECT NOT ('testa 1' LIKE ALL (ARRAY['testa%', 'testb%']))", - "formatted": "SELECT\n NOT (\n 'testa 1' LIKE ALL (\n ARRAY['testa%', 'testb%']\n )\n )" - }, - { - "dialect": "postgres", - "sql": "SELECT to_timestamp(123)::time without time zone", - "formatted": "SELECT to_timestamp(123)::time without time zone" - }, - { - "dialect": "postgres", - "sql": "SELECT SUM(x) OVER (PARTITION BY a ORDER BY d ROWS 1 PRECEDING)", - "formatted": "SELECT SUM(x) OVER (PARTITION BY a ORDER BY d ROWS 1 PRECEDING)" - }, - { - "dialect": "postgres", - "sql": "SELECT SUBSTRING(2022::CHAR(4) || LPAD(3::CHAR(2), 2, '0') FROM 3 FOR 4)", - "formatted": "SELECT SUBSTRING(2022::CHAR(4) || LPAD(3::CHAR(2), 2, '0') FROM 3 FOR 4)" - }, - { - "dialect": "postgres", - "sql": "SELECT m.name FROM manufacturers AS m LEFT JOIN LATERAL GET_PRODUCT_NAMES(m.id) pname ON TRUE WHERE pname IS NULL", - "formatted": "SELECT m.name\nFROM manufacturers AS m\nLEFT JOIN LATERAL GET_PRODUCT_NAMES(m.id) pname\n ON TRUE\nWHERE pname IS NULL", - "previously_formatted": "SELECT\n m.name\nFROM manufacturers AS m\nLEFT JOIN LATERAL GET_PRODUCT_NAMES(m.id) AS pname\n ON TRUE\nWHERE\n pname IS NULL" - }, - { - "dialect": "postgres", - "sql": "SELECT p1.id, p2.id, v1, v2 FROM polygons AS p1, polygons AS p2, LATERAL VERTICES(p1.poly) v1, LATERAL VERTICES(p2.poly) v2 WHERE (v1 <-> v2) < 10 AND p1.id <> p2.id", - "formatted": "SELECT\n p1.id,\n p2.id,\n v1,\n v2\nFROM polygons AS p1, polygons AS p2, LATERAL VERTICES(p1.poly) v1, LATERAL VERTICES(p2.poly) v2\nWHERE\n (\n v1 <-> v2\n ) < 10 AND p1.id <> p2.id", - "previously_formatted": "SELECT\n p1.id,\n p2.id,\n v1,\n v2\nFROM polygons AS p1, polygons AS p2, LATERAL VERTICES(p1.poly) AS v1, LATERAL VERTICES(p2.poly) AS v2\nWHERE\n (\n v1 <-> v2\n ) < 10 AND p1.id <> p2.id" - }, - { - "dialect": "postgres", - "sql": "SELECT id, email, CAST(deleted AS TEXT) FROM users WHERE deleted NOTNULL", - "formatted": "SELECT\n id,\n email,\n CAST(deleted AS TEXT)\nFROM users\nWHERE deleted NOTNULL" - }, - { - "dialect": "postgres", - "sql": "SELECT id, email, CAST(deleted AS TEXT) FROM users WHERE NOT deleted ISNULL", - "formatted": "SELECT\n id,\n email,\n CAST(deleted AS TEXT)\nFROM users\nWHERE NOT deleted ISNULL" - }, - { - "dialect": "postgres", - "sql": "SELECT JSON_ARRAY_ELEMENTS((foo->'sections')::JSON) AS sections", - "formatted": "SELECT\n JSON_ARRAY_ELEMENTS((\n foo -> 'sections'\n )::JSON) AS sections" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM t1*", - "formatted": "SELECT *\nFROM t1*" - }, - { - "dialect": "postgres", - "sql": "SELECT SUBSTRING('afafa' for 1)", - "formatted": "SELECT SUBSTRING('afafa' FOR 1)" - }, - { - "dialect": "postgres", - "sql": "\n WITH\n json_data AS (SELECT '{\"field_id\": [1, 2, 3]}'::JSON AS data),\n field_ids AS (SELECT 'field_id' AS field_id)\n\n SELECT\n JSON_ARRAY_ELEMENTS(json_data.data -> field_ids.field_id) AS element\n FROM json_data, field_ids\n ", - "formatted": "WITH json_data AS (\n SELECT '{\"field_id\": [1, 2, 3]}'::JSON AS data\n),\n\nfield_ids AS (\n SELECT 'field_id' AS field_id\n)\n\nSELECT JSON_ARRAY_ELEMENTS(json_data.data -> field_ids.field_id) AS element\nFROM json_data, field_ids" - }, - { - "dialect": "postgres", - "sql": "SELECT TRIM(LEADING ' XXX ' COLLATE \"de_DE\")", - "formatted": "SELECT TRIM(LEADING ' XXX ' COLLATE \"de_DE\")" - }, - { - "dialect": "postgres", - "sql": "SELECT TRIM(TRAILING ' XXX ' COLLATE \"de_DE\")", - "formatted": "SELECT TRIM(TRAILING ' XXX ' COLLATE \"de_DE\")" - }, - { - "dialect": "postgres", - "sql": "SELECT js, js IS JSON AS \"json?\", js IS JSON VALUE AS \"scalar?\", js IS JSON SCALAR AS \"scalar?\", js IS JSON OBJECT AS \"object?\", js IS JSON ARRAY AS \"array?\" FROM t", - "formatted": "SELECT\n js,\n js IS JSON AS \"json?\",\n js IS JSON VALUE AS \"scalar?\",\n js IS JSON SCALAR AS \"scalar?\",\n js IS JSON OBJECT AS \"object?\",\n js IS JSON ARRAY AS \"array?\"\nFROM t" - }, - { - "dialect": "postgres", - "sql": "SELECT js, js IS JSON ARRAY WITH UNIQUE KEYS AS \"array w. UK?\", js IS JSON ARRAY WITHOUT UNIQUE KEYS AS \"array w/o UK?\", js IS JSON ARRAY UNIQUE KEYS AS \"array w UK 2?\" FROM t", - "formatted": "SELECT\n js,\n js IS JSON ARRAY WITH UNIQUE KEYS AS \"array w. UK?\",\n js IS JSON ARRAY WITHOUT UNIQUE KEYS AS \"array w/o UK?\",\n js IS JSON ARRAY UNIQUE KEYS AS \"array w UK 2?\"\nFROM t" - }, - { - "dialect": "postgres", - "sql": "SELECT 1 FROM ((VALUES (1)) AS vals(id) LEFT OUTER JOIN tbl ON vals.id = tbl.id)", - "formatted": "SELECT 1\nFROM ((VALUES (1)) AS vals(id)\nLEFT OUTER JOIN tbl\n ON vals.id = tbl.id)" - }, - { - "dialect": "postgres", - "sql": "SELECT OVERLAY(a PLACING b FROM 1)", - "formatted": "SELECT OVERLAY(a PLACING b FROM 1)" - }, - { - "dialect": "postgres", - "sql": "SELECT OVERLAY(a PLACING b FROM 1 FOR 1)", - "formatted": "SELECT OVERLAY(a PLACING b FROM 1 FOR 1)" - }, - { - "dialect": "postgres", - "sql": "SELECT PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY a) FILTER(WHERE CAST(b AS BOOLEAN)) AS mean_value FROM (VALUES (0, 't')) AS fake_data(a, b)", - "formatted": "SELECT PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY a) FILTER(WHERE CAST(b AS BOOLEAN)) AS mean_value\nFROM (VALUES (0, 't')) AS fake_data(a, b)" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM foo WHERE id = %s", - "formatted": "SELECT *\nFROM foo\nWHERE id = %s" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM foo WHERE id = %(id_param)s", - "formatted": "SELECT *\nFROM foo\nWHERE id = %(id_param)s" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM foo WHERE id = ?", - "formatted": "SELECT *\nFROM foo\nWHERE id = ?" - }, - { - "dialect": "postgres", - "sql": "SELECT '{\"a\":1, \"b\":2, \"c\":3}'::jsonb ?| array['b', 'c']", - "formatted": "SELECT '{\"a\":1, \"b\":2, \"c\":3}'::jsonb ?| ARRAY['b', 'c']" - }, - { - "dialect": "postgres", - "sql": "SELECT '[\"a\", \"b\"]'::jsonb ?& array['a', 'b']", - "formatted": "SELECT '[\"a\", \"b\"]'::jsonb ?& ARRAY['a', 'b']" - }, - { - "dialect": "postgres", - "sql": "SELECT '[\"a\", {\"b\":1}]'::jsonb #- '{1,b}'", - "formatted": "SELECT '[\"a\", {\"b\":1}]'::jsonb #- '{1,b}'" - }, - { - "dialect": "postgres", - "sql": "SELECT JSON_AGG(DISTINCT name) FROM users", - "formatted": "SELECT JSON_AGG(DISTINCT name)\nFROM users" - }, - { - "dialect": "postgres", - "sql": "SELECT JSON_AGG(c1 ORDER BY c1) FROM (VALUES ('c'), ('b'), ('a')) AS t(c1)", - "formatted": "SELECT JSON_AGG(c1 ORDER BY c1)\nFROM (VALUES ('c'), ('b'), ('a')) AS t(c1)" - }, - { - "dialect": "postgres", - "sql": "SELECT JSON_AGG(DISTINCT c1 ORDER BY c1) FROM (VALUES ('c'), ('b'), ('a')) AS t(c1)", - "formatted": "SELECT JSON_AGG(DISTINCT c1 ORDER BY c1)\nFROM (VALUES ('c'), ('b'), ('a')) AS t(c1)" - }, - { - "dialect": "postgres", - "sql": "SELECT NUMRANGE(1.1, 2.2) -|- NUMRANGE(2.2, 3.3)", - "formatted": "SELECT NUMRANGE(1.1, 2.2) -|- NUMRANGE(2.2, 3.3)" - }, - { - "dialect": "postgres", - "sql": "SELECT SLOPE(point '(4,4)', point '(0,0)')", - "formatted": "SELECT SLOPE(point '(4,4)', point '(0,0)')" - }, - { - "dialect": "postgres", - "sql": "SELECT MLEAST(VARIADIC ARRAY[10, -1, 5, 4.4])", - "formatted": "SELECT MLEAST(VARIADIC ARRAY[10, -1, 5, 4.4])" - }, - { - "dialect": "postgres", - "sql": "SELECT MLEAST(VARIADIC ARRAY[]::numeric[])", - "formatted": null - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM schema_name.table_name st WHERE JSON_EXTRACT_PATH_TEXT((st.data)::json, variadic array['test'::text]) = 'test'::text", - "formatted": "SELECT *\nFROM schema_name.table_name st\nWHERE\n JSON_EXTRACT_PATH_TEXT(\n (\n st.data\n )::json,\n VARIADIC ARRAY['test'::text]\n ) = 'test'::text", - "previously_formatted": "SELECT\n *\nFROM schema_name.table_name AS st\nWHERE\n JSON_EXTRACT_PATH_TEXT(\n (\n st.data\n )::json,\n VARIADIC ARRAY['test'::text]\n ) = 'test'::text" - }, - { - "dialect": "postgres", - "sql": "WITH t(c) AS (SELECT 1) SELECT * INTO UNLOGGED foo FROM (SELECT c AS c FROM t) AS temp", - "formatted": "WITH t(c) AS (\n SELECT 1\n)\n\nSELECT *\nINTO UNLOGGED foo\nFROM (\n SELECT c AS c\n FROM t\n) AS temp" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM UNNEST(ARRAY[1, 2], ARRAY['foo', 'bar', 'baz']) AS x(a, b)", - "formatted": "SELECT *\nFROM UNNEST(ARRAY[1, 2], ARRAY['foo', 'bar', 'baz']) AS x(a, b)" - }, - { - "dialect": "postgres", - "sql": "SELECT TRIM(ARRAY_TO_STRING(ARRAY(SELECT val FROM UNNEST(ARRAY['a', 'b']) WITH ORDINALITY AS u(val, rn)), ' '))", - "formatted": "SELECT\n TRIM(ARRAY_TO_STRING(\n ARRAY(\n SELECT val\n FROM UNNEST(ARRAY['a', 'b']) WITH ORDINALITY AS u(val, rn)\n ),\n ' '\n ))" - }, - { - "dialect": "postgres", - "sql": "SELECT a, LOGICAL_OR(b) FROM table GROUP BY a", - "formatted": "SELECT\n a,\n LOGICAL_OR(b)\nFROM table\nGROUP BY\n a" - }, - { - "dialect": "postgres", - "sql": "SELECT CONCAT('abcde', 2, NULL, 22)", - "formatted": "SELECT CONCAT('abcde', 2, NULL, 22)" - }, - { - "dialect": "postgres", - "sql": "SELECT u & 5 FROM t", - "formatted": "SELECT u & 5\nFROM t" - }, - { - "dialect": "postgres", - "sql": "SELECT (U&'\\FE01' || 'Test literal') AS label FROM data", - "formatted": "SELECT\n (\n U&'\\FE01' || 'Test literal'\n ) AS label\nFROM data" - }, - { - "dialect": "postgres", - "sql": "SELECT U&'d!0061t!+000061' UESCAPE '!' AS label", - "formatted": "SELECT U&'d!0061t!+000061' UESCAPE '!' AS label" - }, - { - "dialect": "postgres", - "sql": "SELECT u&'\\0441\\043B\\043E\\043D'", - "formatted": "SELECT u&'\\0441\\043B\\043E\\043D'" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM ROWS FROM (FUNC1(col1, col2))", - "formatted": "SELECT *\nFROM ROWS FROM (FUNC1(col1, col2))" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM ROWS FROM (FUNC1(col1) AS alias1(\"col1\" TEXT), FUNC2(col2) AS alias2(\"col2\" INT)) WITH ORDINALITY", - "formatted": "SELECT *\nFROM ROWS FROM (FUNC1(col1) AS alias1(\"col1\" TEXT), FUNC2(col2) AS alias2(\"col2\" INT)) WITH ORDINALITY" - }, - { - "dialect": "postgres", - "sql": "SELECT * FROM table1, ROWS FROM (FUNC1(col1) AS alias1(\"col1\" TEXT)) WITH ORDINALITY AS alias3(\"col3\" INT, \"col4\" TEXT)", - "formatted": "SELECT *\nFROM table1, ROWS FROM (FUNC1(col1) AS alias1(\"col1\" TEXT)) WITH ORDINALITY AS alias3(\"col3\" INT, \"col4\" TEXT)" - }, - { - "dialect": "postgres", - "sql": "SELECT ARRAY_LENGTH(ARRAY[1, 2, 3], 1)", - "formatted": "SELECT ARRAY_LENGTH(ARRAY[1, 2, 3], 1)" - }, - { - "dialect": "postgres", - "sql": "SELECT XMLELEMENT(NAME foo)", - "formatted": "SELECT XMLELEMENT(NAME foo)" - }, - { - "dialect": "postgres", - "sql": "SELECT XMLELEMENT(NAME foo, XMLATTRIBUTES('xyz' AS bar))", - "formatted": "SELECT XMLELEMENT(NAME foo, XMLATTRIBUTES('xyz' AS bar))" - }, - { - "dialect": "postgres", - "sql": "SELECT XMLELEMENT(NAME test, XMLATTRIBUTES(a, b)) FROM test", - "formatted": "SELECT XMLELEMENT(NAME test, XMLATTRIBUTES(a, b))\nFROM test" - }, - { - "dialect": "postgres", - "sql": "SELECT XMLELEMENT(NAME foo, XMLATTRIBUTES(CURRENT_DATE AS bar), 'cont', 'ent')", - "formatted": "SELECT XMLELEMENT(NAME foo, XMLATTRIBUTES(CURRENT_DATE AS bar), 'cont', 'ent')" - }, - { - "dialect": "postgres", - "sql": "SELECT XMLELEMENT(NAME \"foo$bar\", XMLATTRIBUTES('xyz' AS \"a&b\"))", - "formatted": "SELECT XMLELEMENT(NAME \"foo$bar\", XMLATTRIBUTES('xyz' AS \"a&b\"))" - }, - { - "dialect": "postgres", - "sql": "SELECT XMLELEMENT(NAME foo, XMLATTRIBUTES('xyz' AS bar), XMLELEMENT(NAME abc), XMLCOMMENT('test'), XMLELEMENT(NAME xyz))", - "formatted": "SELECT XMLELEMENT(NAME foo, XMLATTRIBUTES('xyz' AS bar), XMLELEMENT(NAME abc), XMLCOMMENT('test'), XMLELEMENT(NAME xyz))" - }, - { - "dialect": "postgres", - "sql": "WITH RECURSIVE search_graph(id, link, data, depth) AS (SELECT g.id, g.link, g.data, 1 FROM graph AS g UNION ALL SELECT g.id, g.link, g.data, sg.depth + 1 FROM graph AS g, search_graph AS sg WHERE g.id = sg.link) CYCLE id SET is_cycle USING path SELECT * FROM search_graph", - "formatted": "WITH RECURSIVE search_graph(id, link, data, depth) AS (\n SELECT\n g.id,\n g.link,\n g.data,\n 1\n FROM graph AS g\n UNION ALL\n SELECT\n g.id,\n g.link,\n g.data,\n sg.depth + 1\n FROM graph AS g, search_graph AS sg\n WHERE g.id = sg.link\n)\n\nCYCLE id SET is_cycle USING path\nSELECT *\nFROM search_graph" - }, - { - "dialect": "postgres", - "sql": "SELECT a <<->> b", - "formatted": "SELECT a <<->> b" - }, - { - "dialect": "postgres", - "sql": "SELECT r IS NOT NULL FROM t", - "formatted": "SELECT r IS NOT NULL\nFROM t" - }, - { - "dialect": "postgres", - "sql": "SELECT NOT r IS NULL FROM t", - "formatted": "SELECT NOT r IS NULL\nFROM t" - }, - { - "dialect": "postgres", - "sql": "SELECT NOT r IS NOT NULL FROM t", - "formatted": "SELECT NOT r IS NOT NULL\nFROM t" - }, - { - "dialect": "postgres", - "sql": "SELECT r NOTNULL FROM t", - "formatted": "SELECT r NOTNULL\nFROM t" - }, - { - "dialect": "postgres", - "sql": "SELECT r ISNULL FROM t", - "formatted": "SELECT r ISNULL\nFROM t" - }, - { - "dialect": "postgres", - "sql": "SELECT ARRAY[1, 2, 3] <@ ARRAY[1, 2]", - "formatted": "SELECT ARRAY[1, 2, 3] <@ ARRAY[1, 2]" - }, - { - "dialect": "snowflake", - "sql": "WITH t AS (SELECT PARSE_JSON('{\"level1\": {\"level2\": {\"level3\": \"value\"}}}') AS data) SELECT data: level1 : level2 : level3::VARIANT FROM t", - "formatted": "WITH t AS (\n SELECT PARSE_JSON('{\"level1\": {\"level2\": {\"level3\": \"value\"}}}') AS data\n)\n\nSELECT data:level1:level2:level3::VARIANT\nFROM t" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM x ASOF JOIN y OFFSET MATCH_CONDITION (x.a > y.a)", - "formatted": "SELECT *\nFROM x\nASOF JOIN y OFFSET MATCH_CONDITION (x.a > y.a)", - "previously_formatted": "SELECT\n *\nFROM x\nASOF JOIN y AS OFFSET MATCH_CONDITION (x.a > y.a)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM x ASOF JOIN y LIMIT MATCH_CONDITION (x.a > y.a)", - "formatted": "SELECT *\nFROM x\nASOF JOIN y LIMIT MATCH_CONDITION (x.a > y.a)", - "previously_formatted": "SELECT\n *\nFROM x\nASOF JOIN y AS LIMIT MATCH_CONDITION (x.a > y.a)" - }, - { - "dialect": "snowflake", - "sql": "SELECT session", - "formatted": "SELECT session" - }, - { - "dialect": "snowflake", - "sql": "SELECT DATE_PART(EPOCH_MILLISECOND, CURRENT_TIMESTAMP()) AS a", - "formatted": "SELECT DATE_PART(EPOCH_MILLISECOND, CURRENT_TIMESTAMP()) AS a" - }, - { - "dialect": "snowflake", - "sql": "SELECT GET(a, b)", - "formatted": "SELECT GET(a, b)" - }, - { - "dialect": "snowflake", - "sql": "SELECT HASH_AGG(a, b, c, d)", - "formatted": "SELECT HASH_AGG(a, b, c, d)" - }, - { - "dialect": "snowflake", - "sql": "SELECT GREATEST(1, 2, 3, NULL)", - "formatted": "SELECT GREATEST(1, 2, 3, NULL)" - }, - { - "dialect": "snowflake", - "sql": "SELECT GREATEST_IGNORE_NULLS(1, 2, 3, NULL)", - "formatted": "SELECT GREATEST_IGNORE_NULLS(1, 2, 3, NULL)" - }, - { - "dialect": "snowflake", - "sql": "SELECT LEAST(5, NULL, 7, 3)", - "formatted": "SELECT LEAST(5, NULL, 7, 3)" - }, - { - "dialect": "snowflake", - "sql": "SELECT LEAST_IGNORE_NULLS(5, NULL, 7, 3)", - "formatted": "SELECT LEAST_IGNORE_NULLS(5, NULL, 7, 3)" - }, - { - "dialect": "snowflake", - "sql": "SELECT MAX(x)", - "formatted": "SELECT MAX(x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT COUNT(x)", - "formatted": "SELECT COUNT(x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT MIN(amount)", - "formatted": "SELECT MIN(amount)" - }, - { - "dialect": "snowflake", - "sql": "SELECT MODE(x)", - "formatted": "SELECT MODE(x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT MODE(status) OVER (PARTITION BY region) FROM orders", - "formatted": "SELECT MODE(status) OVER (PARTITION BY region)\nFROM orders" - }, - { - "dialect": "snowflake", - "sql": "SELECT TAN(x)", - "formatted": "SELECT TAN(x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT COS(x)", - "formatted": "SELECT COS(x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT SINH(1.5)", - "formatted": "SELECT SINH(1.5)" - }, - { - "dialect": "snowflake", - "sql": "SELECT MOD(x, y)", - "formatted": "SELECT MOD(x, y)" - }, - { - "dialect": "snowflake", - "sql": "SELECT ROUND(x)", - "formatted": "SELECT ROUND(x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT ROUND(123.456, -1)", - "formatted": "SELECT ROUND(123.456, -1)" - }, - { - "dialect": "snowflake", - "sql": "SELECT ROUND(123.456, 2, 'HALF_AWAY_FROM_ZERO')", - "formatted": "SELECT ROUND(123.456, 2, 'HALF_AWAY_FROM_ZERO')" - }, - { - "dialect": "snowflake", - "sql": "SELECT FLOOR(x)", - "formatted": "SELECT FLOOR(x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT FLOOR(135.135, 1)", - "formatted": "SELECT FLOOR(135.135, 1)" - }, - { - "dialect": "snowflake", - "sql": "SELECT FLOOR(x, -1)", - "formatted": "SELECT FLOOR(x, -1)" - }, - { - "dialect": "snowflake", - "sql": "SELECT PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY salary) FROM employees", - "formatted": "SELECT PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY salary)\nFROM employees" - }, - { - "dialect": "snowflake", - "sql": "SELECT APPROX_TOP_K(col) FROM t", - "formatted": "SELECT APPROX_TOP_K(col)\nFROM t" - }, - { - "dialect": "snowflake", - "sql": "SELECT APPROX_TOP_K(category, 3) FROM t", - "formatted": "SELECT APPROX_TOP_K(category, 3)\nFROM t" - }, - { - "dialect": "snowflake", - "sql": "SELECT MINHASH(5, col)", - "formatted": "SELECT MINHASH(5, col)" - }, - { - "dialect": "snowflake", - "sql": "SELECT MINHASH(5, col1, col2)", - "formatted": "SELECT MINHASH(5, col1, col2)" - }, - { - "dialect": "snowflake", - "sql": "SELECT MINHASH(5, *)", - "formatted": "SELECT MINHASH(5, *)" - }, - { - "dialect": "snowflake", - "sql": "SELECT MINHASH_COMBINE(minhash_col)", - "formatted": "SELECT MINHASH_COMBINE(minhash_col)" - }, - { - "dialect": "snowflake", - "sql": "SELECT APPROXIMATE_SIMILARITY(minhash_col)", - "formatted": "SELECT APPROXIMATE_SIMILARITY(minhash_col)" - }, - { - "dialect": "snowflake", - "sql": "SELECT APPROXIMATE_JACCARD_INDEX(minhash_col)", - "formatted": "SELECT APPROXIMATE_JACCARD_INDEX(minhash_col)" - }, - { - "dialect": "snowflake", - "sql": "SELECT APPROX_PERCENTILE_ACCUMULATE(col)", - "formatted": "SELECT APPROX_PERCENTILE_ACCUMULATE(col)" - }, - { - "dialect": "snowflake", - "sql": "SELECT APPROX_PERCENTILE_ESTIMATE(state, 0.5)", - "formatted": "SELECT APPROX_PERCENTILE_ESTIMATE(state, 0.5)" - }, - { - "dialect": "snowflake", - "sql": "SELECT APPROX_TOP_K_ACCUMULATE(col, 10)", - "formatted": "SELECT APPROX_TOP_K_ACCUMULATE(col, 10)" - }, - { - "dialect": "snowflake", - "sql": "SELECT APPROX_TOP_K_COMBINE(state, 2)", - "formatted": "SELECT APPROX_TOP_K_COMBINE(state, 2)" - }, - { - "dialect": "snowflake", - "sql": "SELECT APPROX_TOP_K_COMBINE(state)", - "formatted": "SELECT APPROX_TOP_K_COMBINE(state)" - }, - { - "dialect": "snowflake", - "sql": "SELECT APPROX_TOP_K_ESTIMATE(state_column, 4)", - "formatted": "SELECT APPROX_TOP_K_ESTIMATE(state_column, 4)" - }, - { - "dialect": "snowflake", - "sql": "SELECT APPROX_TOP_K_ESTIMATE(state_column)", - "formatted": "SELECT APPROX_TOP_K_ESTIMATE(state_column)" - }, - { - "dialect": "snowflake", - "sql": "SELECT APPROX_PERCENTILE_COMBINE(state_column)", - "formatted": "SELECT APPROX_PERCENTILE_COMBINE(state_column)" - }, - { - "dialect": "snowflake", - "sql": "SELECT EQUAL_NULL(1, 2)", - "formatted": "SELECT EQUAL_NULL(1, 2)" - }, - { - "dialect": "snowflake", - "sql": "SELECT EXP(1)", - "formatted": "SELECT EXP(1)" - }, - { - "dialect": "snowflake", - "sql": "SELECT FACTORIAL(5)", - "formatted": "SELECT FACTORIAL(5)" - }, - { - "dialect": "snowflake", - "sql": "SELECT BIT_LENGTH('abc')", - "formatted": "SELECT BIT_LENGTH('abc')" - }, - { - "dialect": "snowflake", - "sql": "SELECT BIT_LENGTH(x'A1B2')", - "formatted": "SELECT BIT_LENGTH(x'A1B2')" - }, - { - "dialect": "snowflake", - "sql": "SELECT BITMAP_BUCKET_NUMBER(32769)", - "formatted": "SELECT BITMAP_BUCKET_NUMBER(32769)" - }, - { - "dialect": "snowflake", - "sql": "SELECT BITMAP_CONSTRUCT_AGG(value)", - "formatted": "SELECT BITMAP_CONSTRUCT_AGG(value)" - }, - { - "dialect": "snowflake", - "sql": "SELECT BITMAP_COUNT(BITMAP_CONSTRUCT_AGG(value)) FROM TABLE(FLATTEN(INPUT => ARRAY_CONSTRUCT(1, 2, 3, 5)))", - "formatted": "SELECT BITMAP_COUNT(BITMAP_CONSTRUCT_AGG(value))\nFROM TABLE(FLATTEN(INPUT => ARRAY_CONSTRUCT(1, 2, 3, 5)))" - }, - { - "dialect": "snowflake", - "sql": "SELECT BOOLAND(1, -2)", - "formatted": "SELECT BOOLAND(1, -2)" - }, - { - "dialect": "snowflake", - "sql": "SELECT BOOLXOR(2, 0)", - "formatted": "SELECT BOOLXOR(2, 0)" - }, - { - "dialect": "snowflake", - "sql": "SELECT BOOLOR(1, 0)", - "formatted": "SELECT BOOLOR(1, 0)" - }, - { - "dialect": "snowflake", - "sql": "SELECT TO_BOOLEAN('true')", - "formatted": "SELECT TO_BOOLEAN('true')" - }, - { - "dialect": "snowflake", - "sql": "SELECT TO_BOOLEAN(1)", - "formatted": "SELECT TO_BOOLEAN(1)" - }, - { - "dialect": "snowflake", - "sql": "SELECT TO_VARIANT(123)", - "formatted": "SELECT TO_VARIANT(123)" - }, - { - "dialect": "snowflake", - "sql": "SELECT IS_NULL_VALUE(GET_PATH(payload, 'field'))", - "formatted": "SELECT IS_NULL_VALUE(GET_PATH(payload, 'field'))" - }, - { - "dialect": "snowflake", - "sql": "SELECT RTRIMMED_LENGTH(' ABCD ')", - "formatted": "SELECT RTRIMMED_LENGTH(' ABCD ')" - }, - { - "dialect": "snowflake", - "sql": "SELECT HEX_DECODE_STRING('48656C6C6F')", - "formatted": "SELECT HEX_DECODE_STRING('48656C6C6F')" - }, - { - "dialect": "snowflake", - "sql": "SELECT HEX_ENCODE('Hello World')", - "formatted": "SELECT HEX_ENCODE('Hello World')" - }, - { - "dialect": "snowflake", - "sql": "SELECT HEX_ENCODE('Hello World', 1)", - "formatted": "SELECT HEX_ENCODE('Hello World', 1)" - }, - { - "dialect": "snowflake", - "sql": "SELECT HEX_ENCODE('Hello World', 0)", - "formatted": "SELECT HEX_ENCODE('Hello World', 0)" - }, - { - "dialect": "snowflake", - "sql": "SELECT IFNULL(col1, col2)", - "formatted": "SELECT IFNULL(col1, col2)" - }, - { - "dialect": "snowflake", - "sql": "SELECT NEXT_DAY('2025-10-15', 'FRIDAY')", - "formatted": "SELECT NEXT_DAY('2025-10-15', 'FRIDAY')" - }, - { - "dialect": "snowflake", - "sql": "SELECT NVL2(col1, col2, col3)", - "formatted": "SELECT NVL2(col1, col2, col3)" - }, - { - "dialect": "snowflake", - "sql": "SELECT NVL(col1, col2)", - "formatted": "SELECT NVL(col1, col2)" - }, - { - "dialect": "snowflake", - "sql": "SELECT CHR(8364)", - "formatted": "SELECT CHR(8364)" - }, - { - "dialect": "snowflake", - "sql": "SELECT CHECK_JSON('{\"key\": \"value\"}')", - "formatted": "SELECT CHECK_JSON('{\"key\": \"value\"}')" - }, - { - "dialect": "snowflake", - "sql": "SELECT CHECK_XML('value')", - "formatted": "SELECT CHECK_XML('value')" - }, - { - "dialect": "snowflake", - "sql": "SELECT CHECK_XML('value', TRUE)", - "formatted": "SELECT CHECK_XML('value', TRUE)" - }, - { - "dialect": "snowflake", - "sql": "SELECT COMPRESS('Hello World', 'ZLIB')", - "formatted": "SELECT COMPRESS('Hello World', 'ZLIB')" - }, - { - "dialect": "snowflake", - "sql": "SELECT DECOMPRESS_BINARY('compressed_data', 'SNAPPY')", - "formatted": "SELECT DECOMPRESS_BINARY('compressed_data', 'SNAPPY')" - }, - { - "dialect": "snowflake", - "sql": "SELECT DECOMPRESS_STRING('compressed_data', 'ZSTD')", - "formatted": "SELECT DECOMPRESS_STRING('compressed_data', 'ZSTD')" - }, - { - "dialect": "snowflake", - "sql": "SELECT LPAD('Hello', 10, '*')", - "formatted": "SELECT LPAD('Hello', 10, '*')" - }, - { - "dialect": "snowflake", - "sql": "SELECT LPAD(tbl.bin_col, 10)", - "formatted": "SELECT LPAD(tbl.bin_col, 10)" - }, - { - "dialect": "snowflake", - "sql": "SELECT RPAD('Hello', 10, '*')", - "formatted": "SELECT RPAD('Hello', 10, '*')" - }, - { - "dialect": "snowflake", - "sql": "SELECT RPAD(tbl.bin_col, 10)", - "formatted": "SELECT RPAD(tbl.bin_col, 10)" - }, - { - "dialect": "snowflake", - "sql": "SELECT RPAD(TO_BINARY('Hi', 'UTF8'), 10, TO_BINARY('_', 'UTF8'))", - "formatted": "SELECT\n RPAD(\n TO_BINARY('Hi', 'UTF8'),\n 10,\n TO_BINARY('_', 'UTF8')\n )" - }, - { - "dialect": "snowflake", - "sql": "SELECT SOUNDEX(column_name)", - "formatted": "SELECT SOUNDEX(column_name)" - }, - { - "dialect": "snowflake", - "sql": "SELECT SOUNDEX_P123(column_name)", - "formatted": "SELECT SOUNDEX_P123(column_name)" - }, - { - "dialect": "snowflake", - "sql": "SELECT ABS(x)", - "formatted": "SELECT ABS(x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT ASIN(0.5)", - "formatted": "SELECT ASIN(0.5)" - }, - { - "dialect": "snowflake", - "sql": "SELECT ASINH(0.5)", - "formatted": "SELECT ASINH(0.5)" - }, - { - "dialect": "snowflake", - "sql": "SELECT ATAN(0.5)", - "formatted": "SELECT ATAN(0.5)" - }, - { - "dialect": "snowflake", - "sql": "SELECT ATAN2(0.5, 0.3)", - "formatted": "SELECT ATAN2(0.5, 0.3)" - }, - { - "dialect": "snowflake", - "sql": "SELECT ATANH(0.5)", - "formatted": "SELECT ATANH(0.5)" - }, - { - "dialect": "snowflake", - "sql": "SELECT CBRT(27.0)", - "formatted": "SELECT CBRT(27.0)" - }, - { - "dialect": "snowflake", - "sql": "SELECT POW(2, 3)", - "formatted": "SELECT POW(2, 3)" - }, - { - "dialect": "snowflake", - "sql": "SELECT POW(2.5, 3.0)", - "formatted": "SELECT POW(2.5, 3.0)" - }, - { - "dialect": "snowflake", - "sql": "SELECT SQUARE(2.5)", - "formatted": "SELECT SQUARE(2.5)" - }, - { - "dialect": "snowflake", - "sql": "SELECT SIGN(x)", - "formatted": "SELECT SIGN(x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT COSH(1.5)", - "formatted": "SELECT COSH(1.5)" - }, - { - "dialect": "snowflake", - "sql": "SELECT TANH(0.5)", - "formatted": "SELECT TANH(0.5)" - }, - { - "dialect": "snowflake", - "sql": "SELECT TRANSLATE(column_name, 'abc', '123')", - "formatted": "SELECT TRANSLATE(column_name, 'abc', '123')" - }, - { - "dialect": "snowflake", - "sql": "SELECT UNICODE(column_name)", - "formatted": "SELECT UNICODE(column_name)" - }, - { - "dialect": "snowflake", - "sql": "SELECT WIDTH_BUCKET(col, 0, 100, 10)", - "formatted": "SELECT WIDTH_BUCKET(col, 0, 100, 10)" - }, - { - "dialect": "snowflake", - "sql": "SELECT PI()", - "formatted": "SELECT PI()" - }, - { - "dialect": "snowflake", - "sql": "SELECT DEGREES(PI() / 3)", - "formatted": "SELECT DEGREES(PI() / 3)" - }, - { - "dialect": "snowflake", - "sql": "SELECT DEGREES(1)", - "formatted": "SELECT DEGREES(1)" - }, - { - "dialect": "snowflake", - "sql": "SELECT RADIANS(180)", - "formatted": "SELECT RADIANS(180)" - }, - { - "dialect": "snowflake", - "sql": "SELECT REGR_AVGX(y, x)", - "formatted": "SELECT REGR_AVGX(y, x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT REGR_AVGY(y, x)", - "formatted": "SELECT REGR_AVGY(y, x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT REGR_COUNT(y, x)", - "formatted": "SELECT REGR_COUNT(y, x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT REGR_INTERCEPT(y, x)", - "formatted": "SELECT REGR_INTERCEPT(y, x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT REGR_R2(y, x)", - "formatted": "SELECT REGR_R2(y, x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT REGR_SXX(y, x)", - "formatted": "SELECT REGR_SXX(y, x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT REGR_SXY(y, x)", - "formatted": "SELECT REGR_SXY(y, x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT REGR_SYY(y, x)", - "formatted": "SELECT REGR_SYY(y, x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT REGR_SLOPE(y, x)", - "formatted": "SELECT REGR_SLOPE(y, x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT RANDSTR(123, 456)", - "formatted": "SELECT RANDSTR(123, 456)" - }, - { - "dialect": "snowflake", - "sql": "SELECT RANDSTR(123, RANDOM())", - "formatted": "SELECT RANDSTR(123, RANDOM())" - }, - { - "dialect": "snowflake", - "sql": "SELECT NORMAL(0, 1, RANDOM())", - "formatted": "SELECT NORMAL(0, 1, RANDOM())" - }, - { - "dialect": "snowflake", - "sql": "SELECT GROUPING_ID(a, b) AS g_id FROM x GROUP BY ROLLUP (a, b)", - "formatted": "SELECT GROUPING_ID(a, b) AS g_id\nFROM x\nGROUP BY\n ROLLUP (\n a,\n b\n )" - }, - { - "dialect": "snowflake", - "sql": "SELECT XMLGET(object_col, 'level2')", - "formatted": "SELECT XMLGET(object_col, 'level2')" - }, - { - "dialect": "snowflake", - "sql": "SELECT XMLGET(object_col, 'level3', 1)", - "formatted": "SELECT XMLGET(object_col, 'level3', 1)" - }, - { - "dialect": "snowflake", - "sql": "SELECT {*} FROM my_table", - "formatted": "SELECT {*}\nFROM my_table" - }, - { - "dialect": "snowflake", - "sql": "SELECT {my_table.*} FROM my_table", - "formatted": "SELECT {my_table.*}\nFROM my_table" - }, - { - "dialect": "snowflake", - "sql": "SELECT {* ILIKE 'col1%'} FROM my_table", - "formatted": "SELECT {* ILIKE 'col1%'}\nFROM my_table" - }, - { - "dialect": "snowflake", - "sql": "SELECT {* EXCLUDE (col1)} FROM my_table", - "formatted": "SELECT {* EXCLUDE (col1)}\nFROM my_table" - }, - { - "dialect": "snowflake", - "sql": "SELECT {* EXCLUDE (col1, col2)} FROM my_table", - "formatted": "SELECT {* EXCLUDE (col1, col2)}\nFROM my_table" - }, - { - "dialect": "snowflake", - "sql": "SELECT a, b, COUNT(*) FROM x GROUP BY ALL LIMIT 100", - "formatted": "SELECT\n a,\n b,\n COUNT(*)\nFROM x\nGROUP BY ALL\nLIMIT 100" - }, - { - "dialect": "snowflake", - "sql": "SELECT STAR(tbl, exclude := [foo])", - "formatted": "SELECT STAR(tbl, exclude := [foo])" - }, - { - "dialect": "snowflake", - "sql": "SELECT CAST([1, 2, 3] AS VECTOR(FLOAT, 3))", - "formatted": "SELECT CAST([1, 2, 3] AS VECTOR(FLOAT, 3))" - }, - { - "dialect": "snowflake", - "sql": "SELECT VECTOR_COSINE_SIMILARITY(a, b)", - "formatted": "SELECT VECTOR_COSINE_SIMILARITY(a, b)" - }, - { - "dialect": "snowflake", - "sql": "SELECT VECTOR_INNER_PRODUCT(a, b)", - "formatted": "SELECT VECTOR_INNER_PRODUCT(a, b)" - }, - { - "dialect": "snowflake", - "sql": "SELECT VECTOR_L1_DISTANCE(a, b)", - "formatted": "SELECT VECTOR_L1_DISTANCE(a, b)" - }, - { - "dialect": "snowflake", - "sql": "SELECT VECTOR_L2_DISTANCE(a, b)", - "formatted": "SELECT VECTOR_L2_DISTANCE(a, b)" - }, - { - "dialect": "snowflake", - "sql": "SELECT CONNECT_BY_ROOT test AS test_column_alias", - "formatted": "SELECT CONNECT_BY_ROOT test AS test_column_alias" - }, - { - "dialect": "snowflake", - "sql": "SELECT rename, replace", - "formatted": "SELECT\n rename,\n replace" - }, - { - "dialect": "snowflake", - "sql": "SELECT TIMEADD(HOUR, 2, CAST('09:05:03' AS TIME))", - "formatted": "SELECT TIMEADD(HOUR, 2, CAST('09:05:03' AS TIME))" - }, - { - "dialect": "snowflake", - "sql": "SELECT CAST(OBJECT_CONSTRUCT('a', 1) AS MAP(VARCHAR, INT))", - "formatted": "SELECT CAST(OBJECT_CONSTRUCT('a', 1) AS MAP(VARCHAR, INT))" - }, - { - "dialect": "snowflake", - "sql": "SELECT MAP_CAT(CAST(col AS MAP(VARCHAR, VARCHAR)), CAST(col AS MAP(VARCHAR, VARCHAR)))", - "formatted": "SELECT MAP_CAT(CAST(col AS MAP(VARCHAR, VARCHAR)), CAST(col AS MAP(VARCHAR, VARCHAR)))" - }, - { - "dialect": "snowflake", - "sql": "SELECT MAP_CONTAINS_KEY('k1', CAST(col AS MAP(VARCHAR, VARCHAR)))", - "formatted": "SELECT MAP_CONTAINS_KEY('k1', CAST(col AS MAP(VARCHAR, VARCHAR)))" - }, - { - "dialect": "snowflake", - "sql": "SELECT MAP_DELETE(CAST(col AS MAP(VARCHAR, VARCHAR)), 'k1')", - "formatted": "SELECT MAP_DELETE(CAST(col AS MAP(VARCHAR, VARCHAR)), 'k1')" - }, - { - "dialect": "snowflake", - "sql": "SELECT MAP_INSERT(CAST(col AS MAP(VARCHAR, VARCHAR)), 'b', '2')", - "formatted": "SELECT MAP_INSERT(CAST(col AS MAP(VARCHAR, VARCHAR)), 'b', '2')" - }, - { - "dialect": "snowflake", - "sql": "SELECT MAP_KEYS(CAST(col AS MAP(VARCHAR, VARCHAR)))", - "formatted": "SELECT MAP_KEYS(CAST(col AS MAP(VARCHAR, VARCHAR)))" - }, - { - "dialect": "snowflake", - "sql": "SELECT MAP_PICK(CAST(col AS MAP(VARCHAR, VARCHAR)), 'a', 'c')", - "formatted": "SELECT MAP_PICK(CAST(col AS MAP(VARCHAR, VARCHAR)), 'a', 'c')" - }, - { - "dialect": "snowflake", - "sql": "SELECT MAP_SIZE(CAST(col AS MAP(VARCHAR, VARCHAR)))", - "formatted": "SELECT MAP_SIZE(CAST(col AS MAP(VARCHAR, VARCHAR)))" - }, - { - "dialect": "snowflake", - "sql": "SELECT CAST(OBJECT_CONSTRUCT('a', 1) AS OBJECT(a CHAR NOT NULL))", - "formatted": "SELECT CAST(OBJECT_CONSTRUCT('a', 1) AS OBJECT(a CHAR NOT NULL))" - }, - { - "dialect": "snowflake", - "sql": "SELECT CAST([1, 2, 3] AS ARRAY(INT))", - "formatted": "SELECT CAST([1, 2, 3] AS ARRAY(INT))" - }, - { - "dialect": "snowflake", - "sql": "SELECT CAST(obj AS OBJECT(x CHAR) RENAME FIELDS)", - "formatted": "SELECT CAST(obj AS OBJECT(x CHAR) RENAME FIELDS)" - }, - { - "dialect": "snowflake", - "sql": "SELECT CAST(obj AS OBJECT(x CHAR, y VARCHAR) ADD FIELDS)", - "formatted": "SELECT CAST(obj AS OBJECT(x CHAR, y VARCHAR) ADD FIELDS)" - }, - { - "dialect": "snowflake", - "sql": "SELECT TO_TIMESTAMP(x) FROM t", - "formatted": "SELECT TO_TIMESTAMP(x)\nFROM t" - }, - { - "dialect": "snowflake", - "sql": "SELECT TO_TIMESTAMP_NTZ(x) FROM t", - "formatted": "SELECT TO_TIMESTAMP_NTZ(x)\nFROM t" - }, - { - "dialect": "snowflake", - "sql": "SELECT TO_TIMESTAMP_LTZ(x) FROM t", - "formatted": "SELECT TO_TIMESTAMP_LTZ(x)\nFROM t" - }, - { - "dialect": "snowflake", - "sql": "SELECT TO_TIMESTAMP_TZ(x) FROM t", - "formatted": "SELECT TO_TIMESTAMP_TZ(x)\nFROM t" - }, - { - "dialect": "snowflake", - "sql": "SELECT TIMESTAMP_FROM_PARTS(2024, 5, 9, 14, 30, 45)", - "formatted": "SELECT TIMESTAMP_FROM_PARTS(2024, 5, 9, 14, 30, 45)" - }, - { - "dialect": "snowflake", - "sql": "SELECT TIMESTAMP_FROM_PARTS(2024, 5, 9, 14, 30, 45, 123)", - "formatted": "SELECT TIMESTAMP_FROM_PARTS(2024, 5, 9, 14, 30, 45, 123)" - }, - { - "dialect": "snowflake", - "sql": "SELECT TIMESTAMP_LTZ_FROM_PARTS(2013, 4, 5, 12, 00, 00)", - "formatted": "SELECT TIMESTAMP_LTZ_FROM_PARTS(2013, 4, 5, 12, 00, 00)" - }, - { - "dialect": "snowflake", - "sql": "SELECT TIMESTAMP_TZ_FROM_PARTS(2013, 4, 5, 12, 00, 00)", - "formatted": "SELECT TIMESTAMP_TZ_FROM_PARTS(2013, 4, 5, 12, 00, 00)" - }, - { - "dialect": "snowflake", - "sql": "SELECT TIMESTAMP_TZ_FROM_PARTS(2013, 4, 5, 12, 00, 00, 0, 'America/Los_Angeles')", - "formatted": "SELECT TIMESTAMP_TZ_FROM_PARTS(2013, 4, 5, 12, 00, 00, 0, 'America/Los_Angeles')" - }, - { - "dialect": "snowflake", - "sql": "SELECT TIMESTAMP_FROM_PARTS(CAST('2024-05-09' AS DATE), CAST('14:30:45' AS TIME))", - "formatted": "SELECT\n TIMESTAMP_FROM_PARTS(\n CAST('2024-05-09' AS DATE),\n CAST('14:30:45' AS TIME)\n )" - }, - { - "dialect": "snowflake", - "sql": "SELECT TIMESTAMP_NTZ_FROM_PARTS(TO_DATE('2013-04-05'), TO_TIME('12:00:00'))", - "formatted": "SELECT TIMESTAMP_NTZ_FROM_PARTS(TO_DATE('2013-04-05'), TO_TIME('12:00:00'))" - }, - { - "dialect": "snowflake", - "sql": "SELECT TIMESTAMP_NTZ_FROM_PARTS(2013, 4, 5, 12, 00, 00, 987654321)", - "formatted": "SELECT TIMESTAMP_NTZ_FROM_PARTS(2013, 4, 5, 12, 00, 00, 987654321)" - }, - { - "dialect": "snowflake", - "sql": "SELECT DATE_FROM_PARTS(1977, 8, 7)", - "formatted": "SELECT DATE_FROM_PARTS(1977, 8, 7)" - }, - { - "dialect": "snowflake", - "sql": "SELECT GET_PATH(v, 'attr[0].name') FROM vartab", - "formatted": "SELECT GET_PATH(v, 'attr[0].name')\nFROM vartab" - }, - { - "dialect": "snowflake", - "sql": "SELECT TO_ARRAY(CAST(x AS ARRAY))", - "formatted": "SELECT TO_ARRAY(CAST(x AS ARRAY))" - }, - { - "dialect": "snowflake", - "sql": "SELECT TO_ARRAY(CAST(['test'] AS VARIANT))", - "formatted": "SELECT TO_ARRAY(CAST(['test'] AS VARIANT))" - }, - { - "dialect": "snowflake", - "sql": "SELECT ARRAY_UNIQUE_AGG(x)", - "formatted": "SELECT ARRAY_UNIQUE_AGG(x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT ARRAY_APPEND([1, 2, 3], 4)", - "formatted": "SELECT ARRAY_APPEND([1, 2, 3], 4)" - }, - { - "dialect": "snowflake", - "sql": "SELECT ARRAY_CAT([1, 2], [3, 4])", - "formatted": "SELECT ARRAY_CAT([1, 2], [3, 4])" - }, - { - "dialect": "snowflake", - "sql": "SELECT ARRAY_PREPEND([2, 3, 4], 1)", - "formatted": "SELECT ARRAY_PREPEND([2, 3, 4], 1)" - }, - { - "dialect": "snowflake", - "sql": "SELECT ARRAY_REMOVE([1, 2, 3], 2)", - "formatted": "SELECT ARRAY_REMOVE([1, 2, 3], 2)" - }, - { - "dialect": "snowflake", - "sql": "SELECT ARRAYS_ZIP([1, 2, 3])", - "formatted": "SELECT ARRAYS_ZIP([1, 2, 3])" - }, - { - "dialect": "snowflake", - "sql": "SELECT ARRAYS_ZIP([1, 2, 3], ['a', 'b', 'c'], [10, 20, 30])", - "formatted": "SELECT\n ARRAYS_ZIP(\n [1, 2, 3],\n ['a', 'b', 'c'],\n [10, 20, 30]\n )" - }, - { - "dialect": "snowflake", - "sql": "SELECT AI_AGG(review, 'Summarize the reviews')", - "formatted": "SELECT AI_AGG(review, 'Summarize the reviews')" - }, - { - "dialect": "snowflake", - "sql": "SELECT AI_SUMMARIZE_AGG(review)", - "formatted": "SELECT AI_SUMMARIZE_AGG(review)" - }, - { - "dialect": "snowflake", - "sql": "SELECT AI_CLASSIFY('text', ['travel', 'cooking'])", - "formatted": "SELECT AI_CLASSIFY('text', ['travel', 'cooking'])" - }, - { - "dialect": "snowflake", - "sql": "SELECT OBJECT_CONSTRUCT()", - "formatted": "SELECT OBJECT_CONSTRUCT()" - }, - { - "dialect": "snowflake", - "sql": "SELECT CURRENT_ACCOUNT()", - "formatted": "SELECT CURRENT_ACCOUNT()" - }, - { - "dialect": "snowflake", - "sql": "SELECT CURRENT_ACCOUNT_NAME()", - "formatted": "SELECT CURRENT_ACCOUNT_NAME()" - }, - { - "dialect": "snowflake", - "sql": "SELECT CURRENT_AVAILABLE_ROLES()", - "formatted": "SELECT CURRENT_AVAILABLE_ROLES()" - }, - { - "dialect": "snowflake", - "sql": "SELECT CURRENT_CLIENT()", - "formatted": "SELECT CURRENT_CLIENT()" - }, - { - "dialect": "snowflake", - "sql": "SELECT CURRENT_IP_ADDRESS()", - "formatted": "SELECT CURRENT_IP_ADDRESS()" - }, - { - "dialect": "snowflake", - "sql": "SELECT CURRENT_DATABASE()", - "formatted": "SELECT CURRENT_DATABASE()" - }, - { - "dialect": "snowflake", - "sql": "SELECT CURRENT_SCHEMAS()", - "formatted": "SELECT CURRENT_SCHEMAS()" - }, - { - "dialect": "snowflake", - "sql": "SELECT CURRENT_SECONDARY_ROLES()", - "formatted": "SELECT CURRENT_SECONDARY_ROLES()" - }, - { - "dialect": "snowflake", - "sql": "SELECT CURRENT_SESSION()", - "formatted": "SELECT CURRENT_SESSION()" - }, - { - "dialect": "snowflake", - "sql": "SELECT CURRENT_STATEMENT()", - "formatted": "SELECT CURRENT_STATEMENT()" - }, - { - "dialect": "snowflake", - "sql": "SELECT CURRENT_VERSION()", - "formatted": "SELECT CURRENT_VERSION()" - }, - { - "dialect": "snowflake", - "sql": "SELECT CURRENT_TRANSACTION()", - "formatted": "SELECT CURRENT_TRANSACTION()" - }, - { - "dialect": "snowflake", - "sql": "SELECT CURRENT_WAREHOUSE()", - "formatted": "SELECT CURRENT_WAREHOUSE()" - }, - { - "dialect": "snowflake", - "sql": "SELECT CURRENT_ORGANIZATION_USER()", - "formatted": "SELECT CURRENT_ORGANIZATION_USER()" - }, - { - "dialect": "snowflake", - "sql": "SELECT CURRENT_REGION()", - "formatted": "SELECT CURRENT_REGION()" - }, - { - "dialect": "snowflake", - "sql": "SELECT CURRENT_ROLE()", - "formatted": "SELECT CURRENT_ROLE()" - }, - { - "dialect": "snowflake", - "sql": "SELECT CURRENT_ROLE_TYPE()", - "formatted": "SELECT CURRENT_ROLE_TYPE()" - }, - { - "dialect": "snowflake", - "sql": "SELECT DAY(CURRENT_TIMESTAMP())", - "formatted": "SELECT DAY(CURRENT_TIMESTAMP())" - }, - { - "dialect": "snowflake", - "sql": "SELECT DAYOFMONTH(CURRENT_TIMESTAMP())", - "formatted": "SELECT DAYOFMONTH(CURRENT_TIMESTAMP())" - }, - { - "dialect": "snowflake", - "sql": "SELECT DAYOFYEAR(CURRENT_TIMESTAMP())", - "formatted": "SELECT DAYOFYEAR(CURRENT_TIMESTAMP())" - }, - { - "dialect": "snowflake", - "sql": "SELECT MONTH(CURRENT_TIMESTAMP())", - "formatted": "SELECT MONTH(CURRENT_TIMESTAMP())" - }, - { - "dialect": "snowflake", - "sql": "SELECT QUARTER(CURRENT_TIMESTAMP())", - "formatted": "SELECT QUARTER(CURRENT_TIMESTAMP())" - }, - { - "dialect": "snowflake", - "sql": "SELECT WEEK(CURRENT_TIMESTAMP())", - "formatted": "SELECT WEEK(CURRENT_TIMESTAMP())" - }, - { - "dialect": "snowflake", - "sql": "SELECT WEEKISO(CURRENT_TIMESTAMP())", - "formatted": "SELECT WEEKISO(CURRENT_TIMESTAMP())" - }, - { - "dialect": "snowflake", - "sql": "SELECT YEAR(CURRENT_TIMESTAMP())", - "formatted": "SELECT YEAR(CURRENT_TIMESTAMP())" - }, - { - "dialect": "snowflake", - "sql": "SELECT YEAROFWEEK(CURRENT_TIMESTAMP())", - "formatted": "SELECT YEAROFWEEK(CURRENT_TIMESTAMP())" - }, - { - "dialect": "snowflake", - "sql": "SELECT YEAROFWEEKISO(CURRENT_TIMESTAMP())", - "formatted": "SELECT YEAROFWEEKISO(CURRENT_TIMESTAMP())" - }, - { - "dialect": "snowflake", - "sql": "SELECT SUM(amount) FROM mytable GROUP BY ALL", - "formatted": "SELECT SUM(amount)\nFROM mytable\nGROUP BY ALL" - }, - { - "dialect": "snowflake", - "sql": "SELECT STDDEV(x)", - "formatted": "SELECT STDDEV(x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT STDDEV(x) OVER (PARTITION BY 1)", - "formatted": "SELECT STDDEV(x) OVER (PARTITION BY 1)" - }, - { - "dialect": "snowflake", - "sql": "SELECT STDDEV_POP(x)", - "formatted": "SELECT STDDEV_POP(x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT STDDEV_POP(x) OVER (PARTITION BY 1)", - "formatted": "SELECT STDDEV_POP(x) OVER (PARTITION BY 1)" - }, - { - "dialect": "snowflake", - "sql": "SELECT STDDEV_SAMP(x)", - "formatted": "SELECT STDDEV_SAMP(x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT STDDEV_SAMP(x) OVER (PARTITION BY 1)", - "formatted": "SELECT STDDEV_SAMP(x) OVER (PARTITION BY 1)" - }, - { - "dialect": "snowflake", - "sql": "SELECT KURTOSIS(x)", - "formatted": "SELECT KURTOSIS(x)" - }, - { - "dialect": "snowflake", - "sql": "SELECT KURTOSIS(x) OVER (PARTITION BY 1)", - "formatted": "SELECT KURTOSIS(x) OVER (PARTITION BY 1)" - }, - { - "dialect": "snowflake", - "sql": "WITH x AS (SELECT 1 AS foo) SELECT foo FROM IDENTIFIER('x')", - "formatted": "WITH x AS (\n SELECT 1 AS foo\n)\n\nSELECT foo\nFROM IDENTIFIER('x')" - }, - { - "dialect": "snowflake", - "sql": "WITH x AS (SELECT 1 AS foo) SELECT IDENTIFIER('foo') FROM x", - "formatted": "WITH x AS (\n SELECT 1 AS foo\n)\n\nSELECT IDENTIFIER('foo')\nFROM x" - }, - { - "dialect": "snowflake", - "sql": "SELECT IDENTIFIER($my_function_name)()", - "formatted": "SELECT IDENTIFIER($my_function_name)()" - }, - { - "dialect": "snowflake", - "sql": "SELECT IDENTIFIER('speed_of_light')()", - "formatted": "SELECT IDENTIFIER('speed_of_light')()" - }, - { - "dialect": "snowflake", - "sql": "SELECT CAST('2021-01-01' AS DATE) + INTERVAL '1 DAY'", - "formatted": "SELECT CAST('2021-01-01' AS DATE) + INTERVAL '1 DAY'" - }, - { - "dialect": "snowflake", - "sql": "SELECT HLL(*)", - "formatted": "SELECT HLL(*)" - }, - { - "dialect": "snowflake", - "sql": "SELECT HLL(a)", - "formatted": "SELECT HLL(a)" - }, - { - "dialect": "snowflake", - "sql": "SELECT HLL(DISTINCT t.a)", - "formatted": "SELECT HLL(DISTINCT t.a)" - }, - { - "dialect": "snowflake", - "sql": "SELECT HLL(a, b, c)", - "formatted": "SELECT HLL(a, b, c)" - }, - { - "dialect": "snowflake", - "sql": "SELECT HLL(DISTINCT a, b, c)", - "formatted": "SELECT HLL(DISTINCT a, b, c)" - }, - { - "dialect": "snowflake", - "sql": "SELECT REGEXP_LIKE(a, b, c)", - "formatted": "SELECT REGEXP_LIKE(a, b, c)" - }, - { - "dialect": "snowflake", - "sql": "SELECT CONVERT_TIMEZONE('UTC', 'America/Los_Angeles', col)", - "formatted": "SELECT CONVERT_TIMEZONE('UTC', 'America/Los_Angeles', col)" - }, - { - "dialect": "snowflake", - "sql": "SELECT CURRENT_ORGANIZATION_NAME()", - "formatted": "SELECT CURRENT_ORGANIZATION_NAME()" - }, - { - "dialect": "snowflake", - "sql": "SELECT MATCH_CONDITION", - "formatted": "SELECT MATCH_CONDITION" - }, - { - "dialect": "snowflake", - "sql": "SELECT OBJECT_AGG(key, value) FROM tbl", - "formatted": "SELECT OBJECT_AGG(key, value)\nFROM tbl" - }, - { - "dialect": "snowflake", - "sql": "SELECT TO_CHAR(CAST('12:05:05' AS TIME))", - "formatted": "SELECT TO_CHAR(CAST('12:05:05' AS TIME))" - }, - { - "dialect": "snowflake", - "sql": "SELECT TRIM(COALESCE(TO_CHAR(CAST(c AS TIME)), '')) FROM t", - "formatted": "SELECT TRIM(COALESCE(TO_CHAR(CAST(c AS TIME)), ''))\nFROM t" - }, - { - "dialect": "snowflake", - "sql": "SELECT GET_PATH(PARSE_JSON(foo), 'bar')", - "formatted": "SELECT GET_PATH(PARSE_JSON(foo), 'bar')" - }, - { - "dialect": "snowflake", - "sql": "SELECT PARSE_IP('192.168.1.1', 'INET')", - "formatted": "SELECT PARSE_IP('192.168.1.1', 'INET')" - }, - { - "dialect": "snowflake", - "sql": "SELECT PARSE_IP('192.168.1.1', 'INET', 0)", - "formatted": "SELECT PARSE_IP('192.168.1.1', 'INET', 0)" - }, - { - "dialect": "snowflake", - "sql": "SELECT GET_PATH(foo, 'bar')", - "formatted": "SELECT GET_PATH(foo, 'bar')" - }, - { - "dialect": "snowflake", - "sql": "SELECT a, exclude, b FROM xxx", - "formatted": "SELECT\n a,\n exclude,\n b\nFROM xxx" - }, - { - "dialect": "snowflake", - "sql": "SELECT ARRAY_SORT(x, TRUE, FALSE)", - "formatted": "SELECT ARRAY_SORT(x, TRUE, FALSE)" - }, - { - "dialect": "snowflake", - "sql": "SELECT BOOLXOR_AGG(col) FROM tbl", - "formatted": "SELECT BOOLXOR_AGG(col)\nFROM tbl" - }, - { - "dialect": "snowflake", - "sql": "SELECT PERCENTILE_DISC(0.9) WITHIN GROUP (ORDER BY col) OVER (PARTITION BY category)", - "formatted": "SELECT PERCENTILE_DISC(0.9) WITHIN GROUP (ORDER BY col) OVER (PARTITION BY category)" - }, - { - "dialect": "snowflake", - "sql": "SELECT AI_CLASSIFY('text', ['travel', 'cooking'], OBJECT_CONSTRUCT('output_mode', 'multi'))", - "formatted": "SELECT\n AI_CLASSIFY(\n 'text',\n ['travel', 'cooking'],\n OBJECT_CONSTRUCT('output_mode', 'multi')\n )" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM table AT (TIMESTAMP => '2024-07-24') UNPIVOT(a FOR b IN (c)) AS pivot_table", - "formatted": "SELECT *\nFROM table AT (TIMESTAMP => '2024-07-24') UNPIVOT(a FOR b IN (c)) AS pivot_table" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN ('2023_Q1', '2023_Q2', '2023_Q3', '2023_Q4', '2024_Q1') DEFAULT ON NULL (0)) ORDER BY empid", - "formatted": "SELECT *\nFROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN ('2023_Q1', '2023_Q2', '2023_Q3', '2023_Q4', '2024_Q1') DEFAULT ON NULL (0))\nORDER BY\n empid" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN (SELECT DISTINCT quarter FROM ad_campaign_types_by_quarter WHERE television = TRUE ORDER BY quarter)) ORDER BY empid", - "formatted": "SELECT *\nFROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN (SELECT DISTINCT\n quarter\nFROM ad_campaign_types_by_quarter\nWHERE television = TRUE\nORDER BY\n quarter))\nORDER BY\n empid" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN (ANY ORDER BY quarter)) ORDER BY empid", - "formatted": "SELECT *\nFROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN (ANY ORDER BY quarter))\nORDER BY\n empid" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN (ANY)) ORDER BY empid", - "formatted": "SELECT *\nFROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN (ANY))\nORDER BY\n empid" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM DATA AS DATA_L ASOF JOIN DATA AS DATA_R MATCH_CONDITION (DATA_L.VAL > DATA_R.VAL) ON DATA_L.ID = DATA_R.ID", - "formatted": "SELECT *\nFROM DATA AS DATA_L\nASOF JOIN DATA AS DATA_R MATCH_CONDITION (DATA_L.VAL > DATA_R.VAL)\n ON DATA_L.ID = DATA_R.ID" - }, - { - "dialect": "snowflake", - "sql": "SELECT TO_TIMESTAMP('2025-01-16T14:45:30.123+0500', 'yyyy-mm-DDThh24:mi:ss.ff9tzhtzm')", - "formatted": "SELECT TO_TIMESTAMP('2025-01-16T14:45:30.123+0500', 'yyyy-mm-DDThh24:mi:ss.ff9tzhtzm')" - }, - { - "dialect": "snowflake", - "sql": "SELECT * REPLACE (CAST(col AS TEXT) AS scol) FROM t", - "formatted": "SELECT * REPLACE (CAST(col AS TEXT) AS scol)\nFROM t" - }, - { - "dialect": "snowflake", - "sql": "SELECT 1 put", - "formatted": "SELECT 1 put", - "previously_formatted": "SELECT\n 1 AS put" - }, - { - "dialect": "snowflake", - "sql": "SELECT 1 get", - "formatted": "SELECT 1 get", - "previously_formatted": "SELECT\n 1 AS get" - }, - { - "dialect": "snowflake", - "sql": "WITH t (SELECT 1 AS c) SELECT c FROM t", - "formatted": "WITH t (\n SELECT 1 AS c\n)\nSELECT c\nFROM t", - "previously_formatted": "WITH t AS (\n SELECT\n 1 AS c\n)\nSELECT\n c\nFROM t" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM s WHERE c NOT IN (1, 2, 3)", - "formatted": "SELECT *\nFROM s\nWHERE c NOT IN (1, 2, 3)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM s WHERE c NOT IN (SELECT * FROM t)", - "formatted": "SELECT *\nFROM s\nWHERE\n c NOT IN (\n SELECT *\n FROM t\n )" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM t1 INNER JOIN t2 USING (t1.col)", - "formatted": "SELECT *\nFROM t1\nINNER JOIN t2\n USING (t1.col)" - }, - { - "dialect": "snowflake", - "sql": "SELECT a:from::STRING, a:from || ' test' ", - "formatted": "SELECT\n a:from::STRING,\n a:from || ' test'" - }, - { - "dialect": "snowflake", - "sql": "SELECT a:select", - "formatted": "SELECT a:select" - }, - { - "dialect": "snowflake", - "sql": "SELECT GET_PATH(PARSE_JSON('{\"y\": [{\"z\": 1}]}'), 'y[0]:z')", - "formatted": "SELECT GET_PATH(PARSE_JSON('{\"y\": [{\"z\": 1}]}'), 'y[0]:z')" - }, - { - "dialect": "snowflake", - "sql": "SELECT p FROM t WHERE p:val NOT IN ('2')", - "formatted": "SELECT p\nFROM t\nWHERE p:val NOT IN ('2')" - }, - { - "dialect": "snowflake", - "sql": "SELECT PARSE_JSON('{\"x\": \"hello\"}'):x LIKE 'hello'", - "formatted": "SELECT PARSE_JSON('{\"x\": \"hello\"}'):x LIKE 'hello'" - }, - { - "dialect": "snowflake", - "sql": "SELECT data:x LIKE 'hello' FROM some_table", - "formatted": "SELECT data:x LIKE 'hello'\nFROM some_table" - }, - { - "dialect": "snowflake", - "sql": "SELECT SUM({ fn CONVERT(123, SQL_DOUBLE) })", - "formatted": "SELECT SUM({ fn CONVERT(123, SQL_DOUBLE) })" - }, - { - "dialect": "snowflake", - "sql": "SELECT SUM({ fn CONVERT(123, SQL_VARCHAR) })", - "formatted": "SELECT SUM({ fn CONVERT(123, SQL_VARCHAR) })" - }, - { - "dialect": "snowflake", - "sql": "SELECT TIMESTAMPFROMPARTS(d, t)", - "formatted": "SELECT TIMESTAMPFROMPARTS(d, t)" - }, - { - "dialect": "snowflake", - "sql": "SELECT v:attr[0].name FROM vartab", - "formatted": "SELECT v:attr[0].name\nFROM vartab" - }, - { - "dialect": "snowflake", - "sql": "SELECT v:\"fruit\" FROM vartab", - "formatted": "SELECT v:\"fruit\"\nFROM vartab" - }, - { - "dialect": "snowflake", - "sql": "SELECT PARSE_JSON('{\"food\":{\"fruit\":\"banana\"}}'):food.fruit::VARCHAR", - "formatted": "SELECT PARSE_JSON('{\"food\":{\"fruit\":\"banana\"}}'):food.fruit::VARCHAR" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM t, UNNEST(x) WITH ORDINALITY", - "formatted": "SELECT *\nFROM t, UNNEST(x) WITH ORDINALITY" - }, - { - "dialect": "snowflake", - "sql": "SELECT state, city, SUM(retail_price * quantity) AS gross_revenue FROM sales GROUP BY ALL", - "formatted": "SELECT\n state,\n city,\n SUM(retail_price * quantity) AS gross_revenue\nFROM sales\nGROUP BY ALL" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM foo window", - "formatted": "SELECT *\nFROM foo window", - "previously_formatted": "SELECT\n *\nFROM foo AS window" - }, - { - "dialect": "snowflake", - "sql": "SELECT RLIKE(a, $$regular expression with \\ characters: \\d{2}-\\d{3}-\\d{4}$$, 'i') FROM log_source", - "formatted": "SELECT RLIKE(a, $$regular expression with \\ characters: \\d{2}-\\d{3}-\\d{4}$$, 'i')\nFROM log_source" - }, - { - "dialect": "snowflake", - "sql": "SELECT $$a ' \\ \\t \\x21 z $ $$", - "formatted": "SELECT $$a ' \\ \\t \\x21 z $ $$" - }, - { - "dialect": "snowflake", - "sql": "SELECT $$a\\$b$$", - "formatted": "SELECT $$a\\$b$$" - }, - { - "dialect": "snowflake", - "sql": "SELECT $$a\\$$", - "formatted": "SELECT $$a\\$$" - }, - { - "dialect": "snowflake", - "sql": "SELECT {'test': 'best'}::VARIANT", - "formatted": "SELECT {'test': 'best'}::VARIANT" - }, - { - "dialect": "snowflake", - "sql": "SELECT {fn DAYNAME('2022-5-13')}", - "formatted": "SELECT {fn\n DAYNAME('2022-5-13') }" - }, - { - "dialect": "snowflake", - "sql": "SELECT {fn LOG(5)}", - "formatted": "SELECT {fn\n LOG(5) }" - }, - { - "dialect": "snowflake", - "sql": "SELECT {fn CEILING(5.3)}", - "formatted": "SELECT {fn\n CEILING(5.3) }" - }, - { - "dialect": "snowflake", - "sql": "SELECT CEIL(3.14)", - "formatted": "SELECT CEIL(3.14)" - }, - { - "dialect": "snowflake", - "sql": "SELECT CEIL(3.14, 1)", - "formatted": "SELECT CEIL(3.14, 1)" - }, - { - "dialect": "snowflake", - "sql": "SELECT DAYOFWEEK('2016-01-02T23:39:20.123-07:00'::TIMESTAMP)", - "formatted": "SELECT DAYOFWEEK('2016-01-02T23:39:20.123-07:00'::TIMESTAMP)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM xxx WHERE col ilike '%Don''t%'", - "formatted": "SELECT *\nFROM xxx\nWHERE col ILIKE '%Don''t%'" - }, - { - "dialect": "snowflake", - "sql": "SELECT * EXCLUDE a, b FROM xxx", - "formatted": "SELECT\n * EXCLUDE a,\n b\nFROM xxx" - }, - { - "dialect": "snowflake", - "sql": "SELECT * RENAME a AS b, c AS d FROM xxx", - "formatted": "SELECT\n * RENAME a AS b,\n c AS d\nFROM xxx" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM xxx, yyy, zzz,", - "formatted": "SELECT *\nFROM xxx, yyy, zzz," - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM xxx, yyy, zzz, WHERE foo = bar", - "formatted": "SELECT *\nFROM xxx, yyy, zzz,\nWHERE foo = bar" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM xxx, yyy, zzz", - "formatted": "SELECT *\nFROM xxx, yyy, zzz" - }, - { - "dialect": "snowflake", - "sql": "SELECT TIMESTAMPNTZFROMPARTS(2013, 4, 5, 12, 00, 00)", - "formatted": "SELECT TIMESTAMPNTZFROMPARTS(2013, 4, 5, 12, 00, 00)" - }, - { - "dialect": "snowflake", - "sql": "SELECT NTH_VALUE(a, 2) FROM t", - "formatted": "SELECT NTH_VALUE(a, 2)\nFROM t" - }, - { - "dialect": "snowflake", - "sql": "SELECT ARRAY_CONSTRUCT('foo')::VARIANT[0]", - "formatted": "SELECT ARRAY_CONSTRUCT('foo')::VARIANT[0]" - }, - { - "dialect": "snowflake", - "sql": "SELECT UUID_STRING(), UUID_STRING('fe971b24-9572-4005-b22f-351e9c09274d', 'foo')", - "formatted": "SELECT\n UUID_STRING(),\n UUID_STRING('fe971b24-9572-4005-b22f-351e9c09274d', 'foo')" - }, - { - "dialect": "snowflake", - "sql": "SELECT BITNOT(a)", - "formatted": "SELECT BITNOT(a)" - }, - { - "dialect": "snowflake", - "sql": "SELECT BIT_NOT(a)", - "formatted": "SELECT BIT_NOT(a)" - }, - { - "dialect": "snowflake", - "sql": "SELECT BITAND(a, b)", - "formatted": "SELECT BITAND(a, b)" - }, - { - "dialect": "snowflake", - "sql": "SELECT BITAND(a, b, 'LEFT')", - "formatted": "SELECT BITAND(a, b, 'LEFT')" - }, - { - "dialect": "snowflake", - "sql": "SELECT BIT_AND(a, b)", - "formatted": "SELECT BIT_AND(a, b)" - }, - { - "dialect": "snowflake", - "sql": "SELECT BIT_AND(a, b, 'LEFT')", - "formatted": "SELECT BIT_AND(a, b, 'LEFT')" - }, - { - "dialect": "snowflake", - "sql": "SELECT BITOR(a, b)", - "formatted": "SELECT BITOR(a, b)" - }, - { - "dialect": "snowflake", - "sql": "SELECT BITOR(a, b, 'LEFT')", - "formatted": "SELECT BITOR(a, b, 'LEFT')" - }, - { - "dialect": "snowflake", - "sql": "SELECT BIT_OR(a, b)", - "formatted": "SELECT BIT_OR(a, b)" - }, - { - "dialect": "snowflake", - "sql": "SELECT BIT_OR(a, b, 'RIGHT')", - "formatted": "SELECT BIT_OR(a, b, 'RIGHT')" - }, - { - "dialect": "snowflake", - "sql": "SELECT BITXOR(a, b)", - "formatted": "SELECT BITXOR(a, b)" - }, - { - "dialect": "snowflake", - "sql": "SELECT BITXOR(a, b, 'LEFT')", - "formatted": "SELECT BITXOR(a, b, 'LEFT')" - }, - { - "dialect": "snowflake", - "sql": "SELECT BIT_XOR(a, b)", - "formatted": "SELECT BIT_XOR(a, b)" - }, - { - "dialect": "snowflake", - "sql": "SELECT BIT_XOR(a, b, 'LEFT')", - "formatted": "SELECT BIT_XOR(a, b, 'LEFT')" - }, - { - "dialect": "snowflake", - "sql": "SELECT HOUR(CAST('08:50:57' AS TIME))", - "formatted": "SELECT HOUR(CAST('08:50:57' AS TIME))" - }, - { - "dialect": "snowflake", - "sql": "SELECT MINUTE(CAST('08:50:57' AS TIME))", - "formatted": "SELECT MINUTE(CAST('08:50:57' AS TIME))" - }, - { - "dialect": "snowflake", - "sql": "SELECT SECOND(CAST('08:50:57' AS TIME))", - "formatted": "SELECT SECOND(CAST('08:50:57' AS TIME))" - }, - { - "dialect": "snowflake", - "sql": "SELECT HOUR(CAST('2024-05-09 08:50:57' AS TIMESTAMP))", - "formatted": "SELECT HOUR(CAST('2024-05-09 08:50:57' AS TIMESTAMP))" - }, - { - "dialect": "snowflake", - "sql": "SELECT MONTHNAME(CAST('2024-05-09' AS DATE))", - "formatted": "SELECT MONTHNAME(CAST('2024-05-09' AS DATE))" - }, - { - "dialect": "snowflake", - "sql": "SELECT PREVIOUS_DAY(CAST('2024-05-09' AS DATE), 'MONDAY')", - "formatted": "SELECT PREVIOUS_DAY(CAST('2024-05-09' AS DATE), 'MONDAY')" - }, - { - "dialect": "snowflake", - "sql": "SELECT TIME_FROM_PARTS(14, 30, 45)", - "formatted": "SELECT TIME_FROM_PARTS(14, 30, 45)" - }, - { - "dialect": "snowflake", - "sql": "SELECT TIME_FROM_PARTS(14, 30, 45, 123)", - "formatted": "SELECT TIME_FROM_PARTS(14, 30, 45, 123)" - }, - { - "dialect": "snowflake", - "sql": "SELECT MONTHS_BETWEEN(CAST('2019-03-15' AS DATE), CAST('2019-02-15' AS DATE))", - "formatted": "SELECT MONTHS_BETWEEN(CAST('2019-03-15' AS DATE), CAST('2019-02-15' AS DATE))" - }, - { - "dialect": "snowflake", - "sql": "SELECT MONTHS_BETWEEN(CAST('2019-03-01 02:00:00' AS TIMESTAMP), CAST('2019-02-15 01:00:00' AS TIMESTAMP))", - "formatted": "SELECT MONTHS_BETWEEN(CAST('2019-03-01 02:00:00' AS TIMESTAMP), CAST('2019-02-15 01:00:00' AS TIMESTAMP))" - }, - { - "dialect": "snowflake", - "sql": "SELECT TIME_SLICE(CAST('2024-05-09 08:50:57.891' AS TIMESTAMP), 15, 'MINUTE')", - "formatted": "SELECT TIME_SLICE(CAST('2024-05-09 08:50:57.891' AS TIMESTAMP), 15, 'MINUTE')" - }, - { - "dialect": "snowflake", - "sql": "SELECT TIME_SLICE(CAST('2024-05-09' AS DATE), 1, 'DAY')", - "formatted": "SELECT TIME_SLICE(CAST('2024-05-09' AS DATE), 1, 'DAY')" - }, - { - "dialect": "snowflake", - "sql": "SELECT TIME_SLICE(CAST('2024-05-09 08:50:57.891' AS TIMESTAMP), 1, 'HOUR', 'start')", - "formatted": "SELECT\n TIME_SLICE(\n CAST('2024-05-09 08:50:57.891' AS TIMESTAMP),\n 1,\n 'HOUR',\n 'start'\n )" - }, - { - "dialect": "snowflake", - "sql": "SELECT * EXCLUDE foo RENAME bar AS baz FROM tbl", - "formatted": "SELECT * EXCLUDE foo RENAME bar AS baz\nFROM tbl" - }, - { - "dialect": "snowflake", - "sql": "SELECT LIKE(col, 'pattern')", - "formatted": "SELECT LIKE(col, 'pattern')" - }, - { - "dialect": "snowflake", - "sql": "SELECT ILIKE(col, 'pattern')", - "formatted": "SELECT ILIKE(col, 'pattern')" - }, - { - "dialect": "snowflake", - "sql": "SELECT LIKE(col, 'pattern', '\\\\')", - "formatted": "SELECT LIKE(col, 'pattern', '\\\\')" - }, - { - "dialect": "snowflake", - "sql": "SELECT ILIKE(col, 'pattern', '\\\\')", - "formatted": "SELECT ILIKE(col, 'pattern', '\\\\')" - }, - { - "dialect": "snowflake", - "sql": "SELECT LIKE(col, 'pattern', '!')", - "formatted": "SELECT LIKE(col, 'pattern', '!')" - }, - { - "dialect": "snowflake", - "sql": "SELECT ILIKE(col, 'pattern', '!')", - "formatted": "SELECT ILIKE(col, 'pattern', '!')" - }, - { - "dialect": "snowflake", - "sql": "SELECT BASE64_ENCODE('Hello World')", - "formatted": "SELECT BASE64_ENCODE('Hello World')" - }, - { - "dialect": "snowflake", - "sql": "SELECT TRY_HEX_DECODE_BINARY('48656C6C6F')", - "formatted": "SELECT TRY_HEX_DECODE_BINARY('48656C6C6F')" - }, - { - "dialect": "snowflake", - "sql": "SELECT TRY_HEX_DECODE_STRING('48656C6C6F')", - "formatted": "SELECT TRY_HEX_DECODE_STRING('48656C6C6F')" - }, - { - "dialect": "snowflake", - "sql": "SELECT ARRAY_CONTAINS(1, [1])", - "formatted": "SELECT ARRAY_CONTAINS(1, [1])" - }, - { - "dialect": "snowflake", - "sql": "SELECT ARRAY_DISTINCT(['A', 'B', 'A'])", - "formatted": "SELECT ARRAY_DISTINCT(['A', 'B', 'A'])" - }, - { - "dialect": "snowflake", - "sql": "SELECT CURRENT_DATABASE()", - "formatted": "SELECT CURRENT_DATABASE()" - }, - { - "dialect": "snowflake", - "sql": "SELECT CURRENT_SCHEMA()", - "formatted": "SELECT CURRENT_SCHEMA()" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM @\"mystage\"", - "formatted": "SELECT *\nFROM @\"mystage\"" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM @\"myschema\".\"mystage\"/file.gz", - "formatted": "SELECT *\nFROM @\"myschema\".\"mystage\"/file.gz" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM @\"my_DB\".\"schEMA1\".mystage/file.gz", - "formatted": "SELECT *\nFROM @\"my_DB\".\"schEMA1\".mystage/file.gz" - }, - { - "dialect": "snowflake", - "sql": "SELECT metadata$filename FROM @s1/", - "formatted": "SELECT metadata$filename\nFROM @s1/" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM @~", - "formatted": "SELECT *\nFROM @~" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM @~/some/path/to/file.csv", - "formatted": "SELECT *\nFROM @~/some/path/to/file.csv" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM @mystage", - "formatted": "SELECT *\nFROM @mystage" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM '@mystage'", - "formatted": "SELECT *\nFROM '@mystage'" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM @namespace.mystage/path/to/file.json.gz", - "formatted": "SELECT *\nFROM @namespace.mystage/path/to/file.json.gz" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM @namespace.%table_name/path/to/file.json.gz", - "formatted": "SELECT *\nFROM @namespace.%table_name/path/to/file.json.gz" - }, - { - "dialect": "snowflake", - "sql": "SELECT $1, $2, metadata$filename FROM @mystage (PATTERN => '.*data-100.*')", - "formatted": "SELECT\n $1,\n $2,\n metadata$filename\nFROM @mystage (PATTERN => '.*data-100.*')" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM '@external/location' (FILE_FORMAT => 'path.to.csv')", - "formatted": null - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM (SELECT a FROM @foo)", - "formatted": "SELECT *\nFROM (\n SELECT a\n FROM @foo\n)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM (SELECT * FROM '@external/location' (FILE_FORMAT => 'path.to.csv'))", - "formatted": null - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM @foo/bar (FILE_FORMAT => ds_sandbox.test.my_csv_format, PATTERN => 'test') AS bla", - "formatted": "SELECT *\nFROM @foo/bar (FILE_FORMAT => ds_sandbox.test.my_csv_format, PATTERN => 'test') AS bla" - }, - { - "dialect": "snowflake", - "sql": "SELECT t.$1, t.$2 FROM @mystage1 (FILE_FORMAT => 'myformat', PATTERN => '.*data.*[.]csv.gz') AS t", - "formatted": "SELECT\n t.$1,\n t.$2\nFROM @mystage1 (FILE_FORMAT => 'myformat', PATTERN => '.*data.*[.]csv.gz') AS t" - }, - { - "dialect": "snowflake", - "sql": "SELECT parse_json($1):a.b FROM @mystage2/data1.json.gz", - "formatted": "SELECT parse_json($1):a.b\nFROM @mystage2/data1.json.gz" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM @mystage t (c1)", - "formatted": "SELECT *\nFROM @mystage t(c1)", - "previously_formatted": "SELECT\n *\nFROM @mystage AS t(c1)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM @foo/bar (PATTERN => 'test', FILE_FORMAT => ds_sandbox.test.my_csv_format) AS bla", - "formatted": "SELECT *\nFROM @foo/bar (PATTERN => 'test', FILE_FORMAT => ds_sandbox.test.my_csv_format) AS bla" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM @test.public.thing/location/somefile.csv( FILE_FORMAT => 'fmt' )", - "formatted": "SELECT *\nFROM @test.public.thing/location/somefile.csv (FILE_FORMAT => 'fmt')" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM testtable TABLESAMPLE BERNOULLI (20.3)", - "formatted": "SELECT *\nFROM testtable TABLESAMPLE BERNOULLI (20.3)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM testtable TABLESAMPLE SYSTEM (3) SEED (82)", - "formatted": "SELECT *\nFROM testtable TABLESAMPLE SYSTEM (3) SEED (82)" - }, - { - "dialect": "snowflake", - "sql": "SELECT a FROM test PIVOT(SUM(x) FOR y IN ('z', 'q')) AS x TABLESAMPLE BERNOULLI (0.1)", - "formatted": "SELECT a\nFROM test PIVOT(SUM(x) FOR y IN ('z', 'q')) AS x TABLESAMPLE BERNOULLI (0.1)" - }, - { - "dialect": "snowflake", - "sql": "SELECT i, j FROM table1 AS t1 INNER JOIN table2 AS t2 TABLESAMPLE BERNOULLI (50) WHERE t2.j = t1.i", - "formatted": "SELECT\n i,\n j\nFROM table1 AS t1\nINNER JOIN table2 AS t2 TABLESAMPLE BERNOULLI (50)\nWHERE t2.j = t1.i" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM (SELECT * FROM t1 JOIN t2 ON t1.a = t2.c) TABLESAMPLE BERNOULLI (1)", - "formatted": "SELECT *\nFROM (\n SELECT *\n FROM t1\n JOIN t2\n ON t1.a = t2.c\n) TABLESAMPLE BERNOULLI (1)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM testtable TABLESAMPLE (10 ROWS)", - "formatted": "SELECT *\nFROM testtable TABLESAMPLE (10 ROWS)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM testtable TABLESAMPLE (100)", - "formatted": "SELECT *\nFROM testtable TABLESAMPLE (100)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM testtable SAMPLE (10)", - "formatted": "SELECT *\nFROM testtable SAMPLE (10)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM testtable SAMPLE ROW (0)", - "formatted": "SELECT *\nFROM testtable SAMPLE ROW (0)" - }, - { - "dialect": "snowflake", - "sql": "SELECT a FROM test SAMPLE BLOCK (0.5) SEED (42)", - "formatted": "SELECT a\nFROM test SAMPLE BLOCK (0.5) SEED (42)" - }, - { - "dialect": "snowflake", - "sql": "SELECT user_id, value FROM table_name SAMPLE BERNOULLI ($s) SEED (0)", - "formatted": "SELECT\n user_id,\n value\nFROM table_name SAMPLE BERNOULLI ($s) SEED (0)" - }, - { - "dialect": "snowflake", - "sql": "SELECT CAST('12:00:00' AS TIME)", - "formatted": "SELECT CAST('12:00:00' AS TIME)" - }, - { - "dialect": "snowflake", - "sql": "SELECT DATE_PART(month, a)", - "formatted": "SELECT DATE_PART(month, a)" - }, - { - "dialect": "snowflake", - "sql": "SELECT TO_TIME(x) FROM t", - "formatted": "SELECT TO_TIME(x)\nFROM t" - }, - { - "dialect": "snowflake", - "sql": "SELECT TO_DATE('2019-02-28') + INTERVAL '1 day, 1 year'", - "formatted": "SELECT TO_DATE('2019-02-28') + INTERVAL '1 day, 1 year'" - }, - { - "dialect": "snowflake", - "sql": "SELECT CAST(a AS VARIANT)", - "formatted": "SELECT CAST(a AS VARIANT)" - }, - { - "dialect": "snowflake", - "sql": "SELECT CAST(a AS ARRAY)", - "formatted": "SELECT CAST(a AS ARRAY)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM my_table AT (STATEMENT => $query_id_var)", - "formatted": "SELECT *\nFROM my_table AT (STATEMENT => $query_id_var)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM my_table AT (OFFSET => -60 * 5)", - "formatted": "SELECT *\nFROM my_table AT (OFFSET => -60 * 5)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM my_table BEFORE (STATEMENT => $query_id_var)", - "formatted": "SELECT *\nFROM my_table BEFORE (STATEMENT => $query_id_var)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM my_table BEFORE (OFFSET => -60 * 5)", - "formatted": "SELECT *\nFROM my_table BEFORE (OFFSET => -60 * 5)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM my_table AT (TIMESTAMP => TO_TIMESTAMP(1432669154242, 3))", - "formatted": "SELECT *\nFROM my_table AT (TIMESTAMP => TO_TIMESTAMP(1432669154242, 3))" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM my_table AT (OFFSET => -60 * 5) AS T WHERE T.flag = 'valid'", - "formatted": "SELECT *\nFROM my_table AT (OFFSET => -60 * 5) AS T\nWHERE T.flag = 'valid'" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM my_table AT (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726')", - "formatted": "SELECT *\nFROM my_table AT (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726')" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM my_table BEFORE (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726')", - "formatted": "SELECT *\nFROM my_table BEFORE (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726')" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM my_table AT (TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp)", - "formatted": "SELECT *\nFROM my_table AT (TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM my_table AT(TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp_tz)", - "formatted": "SELECT *\nFROM my_table AT (TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp_tz)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM my_table BEFORE (TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp_tz);", - "formatted": "SELECT *\nFROM my_table BEFORE (TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp_tz);", - "previously_formatted": "SELECT\n *\nFROM my_table BEFORE (TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp_tz)" - }, - { - "dialect": "snowflake", - "sql": "\n SELECT oldt.* , newt.*\n FROM my_table BEFORE(STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726') AS oldt\n FULL OUTER JOIN my_table AT(STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726') AS newt\n ON oldt.id = newt.id\n WHERE oldt.id IS NULL OR newt.id IS NULL;\n ", - "formatted": "SELECT\n oldt.*,\n newt.*\nFROM my_table BEFORE (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726') AS oldt\nFULL OUTER JOIN my_table AT (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726') AS newt\n ON oldt.id = newt.id\nWHERE\n oldt.id IS NULL\n OR newt.id IS NULL;", - "previously_formatted": "SELECT\n oldt.*,\n newt.*\nFROM my_table BEFORE (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726') AS oldt\nFULL OUTER JOIN my_table AT (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726') AS newt\n ON oldt.id = newt.id\nWHERE\n oldt.id IS NULL\n OR newt.id IS NULL" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM TABLE('MYTABLE')", - "formatted": "SELECT *\nFROM TABLE('MYTABLE')" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM TABLE($MYVAR)", - "formatted": "SELECT *\nFROM TABLE($MYVAR)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM TABLE(?)", - "formatted": "SELECT *\nFROM TABLE(?)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM TABLE(:BINDING)", - "formatted": null - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM TABLE($MYVAR) WHERE COL1 = 10", - "formatted": "SELECT *\nFROM TABLE($MYVAR)\nWHERE COL1 = 10" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM TABLE('t1') AS f", - "formatted": "SELECT *\nFROM TABLE('t1') AS f" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM (TABLE('t1') CROSS JOIN TABLE('t2'))", - "formatted": "SELECT *\nFROM (TABLE('t1')\nCROSS JOIN TABLE('t2'))" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM TABLE('t1'), LATERAL (SELECT * FROM t2)", - "formatted": "SELECT *\nFROM TABLE('t1'), LATERAL (\n SELECT *\n FROM t2\n)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM TABLE('t1') UNION ALL SELECT * FROM TABLE('t2')", - "formatted": "SELECT *\nFROM TABLE('t1')\nUNION ALL\nSELECT *\nFROM TABLE('t2')" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM TABLE('t1') TABLESAMPLE BERNOULLI (20.3)", - "formatted": "SELECT *\nFROM TABLE('t1') TABLESAMPLE BERNOULLI (20.3)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM TABLE('MYDB.\"MYSCHEMA\".\"MYTABLE\"')", - "formatted": "SELECT *\nFROM TABLE('MYDB.\"MYSCHEMA\".\"MYTABLE\"')" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM TABLE($$MYDB. \"MYSCHEMA\".\"MYTABLE\"$$)", - "formatted": "SELECT *\nFROM TABLE($$MYDB. \"MYSCHEMA\".\"MYTABLE\"$$)" - }, - { - "dialect": "snowflake", - "sql": "SELECT value FROM TABLE(FLATTEN(input => SELECT PARSE_JSON('[1, 2]')))", - "formatted": "SELECT value\nFROM TABLE(FLATTEN(input => SELECT\n PARSE_JSON('[1, 2]')))" - }, - { - "dialect": "snowflake", - "sql": "SELECT SEARCH((play, line), 'dream')", - "formatted": "SELECT SEARCH((play, line), 'dream')" - }, - { - "dialect": "snowflake", - "sql": "SELECT SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER')", - "formatted": "SELECT SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER')" - }, - { - "dialect": "snowflake", - "sql": "SELECT SEARCH(character, 'king queen', SEARCH_MODE => 'AND')", - "formatted": "SELECT SEARCH(character, 'king queen', SEARCH_MODE => 'AND')" - }, - { - "dialect": "snowflake", - "sql": "SELECT SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER', SEARCH_MODE => 'OR')", - "formatted": "SELECT\n SEARCH(\n line,\n 'king',\n ANALYZER => 'UNICODE_ANALYZER',\n SEARCH_MODE => 'OR'\n )" - }, - { - "dialect": "snowflake", - "sql": "SELECT SEARCH(line, 'king')", - "formatted": "SELECT SEARCH(line, 'king')" - }, - { - "dialect": "snowflake", - "sql": "SELECT SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER')", - "formatted": "SELECT SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER')" - }, - { - "dialect": "snowflake", - "sql": "SELECT SEARCH(character, 'king queen', SEARCH_MODE => 'AND')", - "formatted": "SELECT SEARCH(character, 'king queen', SEARCH_MODE => 'AND')" - }, - { - "dialect": "snowflake", - "sql": "SELECT SEARCH(line, 'king', SEARCH_MODE => 'AND', ANALYZER => 'PATTERN_ANALYZER')", - "formatted": "SELECT\n SEARCH(\n line,\n 'king',\n SEARCH_MODE => 'AND',\n ANALYZER => 'PATTERN_ANALYZER'\n )" - }, - { - "dialect": "snowflake", - "sql": "SELECT REGEXP_COUNT('hello world', 'l ')", - "formatted": "SELECT REGEXP_COUNT('hello world', 'l ')" - }, - { - "dialect": "snowflake", - "sql": "SELECT REGEXP_COUNT('hello world', 'l', 1)", - "formatted": "SELECT REGEXP_COUNT('hello world', 'l', 1)" - }, - { - "dialect": "snowflake", - "sql": "SELECT REGEXP_COUNT('hello world', 'l', 1, 'i')", - "formatted": "SELECT REGEXP_COUNT('hello world', 'l', 1, 'i')" - }, - { - "dialect": "snowflake", - "sql": "SELECT REGEXP_INSTR('abc', 'a')", - "formatted": "SELECT REGEXP_INSTR('abc', 'a')" - }, - { - "dialect": "snowflake", - "sql": "SELECT REGEXP_INSTR('abc', 'a', 1, 1, 0, 'i')", - "formatted": "SELECT REGEXP_INSTR('abc', 'a', 1, 1, 0, 'i')" - }, - { - "dialect": "snowflake", - "sql": "SELECT TRY_CAST(x AS DOUBLE)", - "formatted": "SELECT TRY_CAST(x AS DOUBLE)" - }, - { - "dialect": "snowflake", - "sql": "SELECT TRY_CAST(FOO() AS TEXT)", - "formatted": "SELECT TRY_CAST(FOO() AS TEXT)" - }, - { - "dialect": "snowflake", - "sql": "SELECT $1", - "formatted": "SELECT $1" - }, - { - "dialect": "snowflake", - "sql": "SELECT $1.elem", - "formatted": "SELECT $1.elem" - }, - { - "dialect": "snowflake", - "sql": "SELECT $1:a.b", - "formatted": "SELECT $1:a.b" - }, - { - "dialect": "snowflake", - "sql": "SELECT t.$23:a.b", - "formatted": "SELECT t.$23:a.b" - }, - { - "dialect": "snowflake", - "sql": "SELECT t.$17:a[0].b[0].c", - "formatted": "SELECT t.$17:a[0].b[0].c" - }, - { - "dialect": "snowflake", - "sql": "SELECT C1 FROM t1 CHANGES (INFORMATION => APPEND_ONLY) AT (STREAM => 's1') END (TIMESTAMP => $ts2)", - "formatted": "SELECT C1\nFROM t1 CHANGES (INFORMATION => APPEND_ONLY) AT (STREAM => 's1') END (TIMESTAMP => $ts2)" - }, - { - "dialect": "snowflake", - "sql": "SELECT C1 FROM t1 CHANGES (INFORMATION => APPEND_ONLY) BEFORE (STATEMENT => 'STMT_ID') END (TIMESTAMP => $ts2)", - "formatted": "SELECT C1\nFROM t1 CHANGES (INFORMATION => APPEND_ONLY) BEFORE (STATEMENT => 'STMT_ID') END (TIMESTAMP => $ts2)" - }, - { - "dialect": "snowflake", - "sql": "SELECT 1 FROM some_table CHANGES (INFORMATION => APPEND_ONLY) AT (TIMESTAMP => TO_TIMESTAMP_TZ('2024-07-01 00:00:00+00:00')) END (TIMESTAMP => TO_TIMESTAMP_TZ('2024-07-01 14:28:59.999999+00:00'))", - "formatted": "SELECT 1\nFROM some_table CHANGES (INFORMATION => APPEND_ONLY) AT (TIMESTAMP => TO_TIMESTAMP_TZ('2024-07-01 00:00:00+00:00')) END (TIMESTAMP => TO_TIMESTAMP_TZ('2024-07-01 14:28:59.999999+00:00'))" - }, - { - "dialect": "snowflake", - "sql": "SELECT :1", - "formatted": null - }, - { - "dialect": "snowflake", - "sql": "SELECT :1, :2", - "formatted": null - }, - { - "dialect": "snowflake", - "sql": "SELECT :1 + :2", - "formatted": null - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM SEMANTIC_VIEW(foo METRICS a.b, a.c DIMENSIONS a.b, a.c WHERE a.b > '1995-01-01')", - "formatted": "SELECT *\nFROM SEMANTIC_VIEW(\n foo\n METRICS a.b, a.c\n DIMENSIONS a.b, a.c\n WHERE a.b > '1995-01-01'\n)" - }, - { - "dialect": "snowflake", - "sql": "SELECT col1, col2, metric1 FROM SEMANTIC_VIEW(mydb.myschema.my_semantic_view METRICS metric1 DIMENSIONS col1, DATE_TRUNC('MONTH', timestamp_col) AS col2) ORDER BY col1, col2 DESC", - "formatted": "SELECT\n col1,\n col2,\n metric1\nFROM SEMANTIC_VIEW(\n mydb.myschema.my_semantic_view\n METRICS metric1\n DIMENSIONS col1, DATE_TRUNC('MONTH', timestamp_col) AS col2\n)\nORDER BY\n col1,\n col2 DESC" - }, - { - "dialect": "snowflake", - "sql": "SELECT model!mladmin", - "formatted": "SELECT model!mladmin" - }, - { - "dialect": "snowflake", - "sql": "SELECT model!PREDICT(1)", - "formatted": "SELECT model!PREDICT(1)" - }, - { - "dialect": "snowflake", - "sql": "SELECT m!PREDICT(INPUT_DATA => {*}) AS p FROM tbl", - "formatted": "SELECT m!PREDICT(INPUT_DATA => {*}) AS p\nFROM tbl" - }, - { - "dialect": "snowflake", - "sql": "SELECT m!PREDICT(INPUT_DATA => {tbl.*}) AS p FROM tbl", - "formatted": "SELECT m!PREDICT(INPUT_DATA => {tbl.*}) AS p\nFROM tbl" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM TABLE(model_trained_with_labeled_data!DETECT_ANOMALIES(INPUT_DATA => TABLE(view_with_data_to_analyze), TIMESTAMP_COLNAME => 'date', TARGET_COLNAME => 'sales', CONFIG_OBJECT => OBJECT_CONSTRUCT('prediction_interval', 0.99)))", - "formatted": "SELECT *\nFROM TABLE(model_trained_with_labeled_data!DETECT_ANOMALIES(\n INPUT_DATA => TABLE(view_with_data_to_analyze),\n TIMESTAMP_COLNAME => 'date',\n TARGET_COLNAME => 'sales',\n CONFIG_OBJECT => OBJECT_CONSTRUCT('prediction_interval', 0.99)\n))" - }, - { - "dialect": "snowflake", - "sql": "SELECT 1 FROM TABLE(GENERATOR(ROWCOUNT => 10))", - "formatted": "SELECT 1\nFROM TABLE(GENERATOR(ROWCOUNT => 10))" - }, - { - "dialect": "snowflake", - "sql": "SELECT 1 FROM TABLE(GENERATOR(TIMELIMIT => 5))", - "formatted": "SELECT 1\nFROM TABLE(GENERATOR(TIMELIMIT => 5))" - }, - { - "dialect": "snowflake", - "sql": "SELECT 1 FROM TABLE(GENERATOR(ROWCOUNT => 10, TIMELIMIT => 5))", - "formatted": "SELECT 1\nFROM TABLE(GENERATOR(ROWCOUNT => 10, TIMELIMIT => 5))" - }, - { - "dialect": "snowflake", - "sql": "SELECT 1 FROM TABLE(GENERATOR(10))", - "formatted": "SELECT 1\nFROM TABLE(GENERATOR(10))" - }, - { - "dialect": "snowflake", - "sql": "SELECT 1 FROM TABLE(GENERATOR(10, 5))", - "formatted": "SELECT 1\nFROM TABLE(GENERATOR(10, 5))" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM a CROSS DIRECTED JOIN b USING (id)", - "formatted": "SELECT *\nFROM a\nCROSS DIRECTED JOIN b\n USING (id)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM a INNER DIRECTED JOIN b USING (id)", - "formatted": "SELECT *\nFROM a\nINNER DIRECTED JOIN b\n USING (id)" - }, - { - "dialect": "snowflake", - "sql": "SELECT * FROM a NATURAL INNER DIRECTED JOIN b USING (id)", - "formatted": "SELECT *\nFROM a\nNATURAL INNER DIRECTED JOIN b\n USING (id)" - }, - { - "dialect": "snowflake", - "sql": "SELECT TRY_PARSE_JSON('{\"x: 1}')", - "formatted": "SELECT TRY_PARSE_JSON('{\"x: 1}')" - }, - { - "dialect": "snowflake", - "sql": "SELECT number", - "formatted": "SELECT number" - }, - { - "dialect": "snowflake", - "sql": "SELECT TO_TIMESTAMP(123.4)", - "formatted": "SELECT TO_TIMESTAMP(123.4)" - }, - { - "dialect": "snowflake", - "sql": "SELECT DATE_PART(year FROM CAST('2024-04-08' AS DATE))", - "formatted": "SELECT DATE_PART(year FROM CAST('2024-04-08' AS DATE))" - }, - { - "dialect": "snowflake", - "sql": "SELECT DATE_PART('month' FROM CAST('2024-04-08' AS DATE))", - "formatted": "SELECT DATE_PART('month' FROM CAST('2024-04-08' AS DATE))" - }, - { - "dialect": "snowflake", - "sql": "SELECT DATE_PART(day FROM a)", - "formatted": "SELECT DATE_PART(day FROM a)" - }, - { - "dialect": "snowflake", - "sql": "SELECT SEARCH_IP(col, '192.168.0.0')", - "formatted": "SELECT SEARCH_IP(col, '192.168.0.0')" - }, - { - "dialect": "snowflake", - "sql": "SELECT DATEADD(DAY, -7, DATEADD(t.m, 1, CAST('2023-01-03' AS DATE))) FROM (SELECT 'month' AS m) AS t", - "formatted": "SELECT DATEADD(DAY, -7, DATEADD(t.m, 1, CAST('2023-01-03' AS DATE)))\nFROM (\n SELECT 'month' AS m\n) AS t" - }, - { - "dialect": "snowflake", - "sql": "SELECT FILE_URL FROM DIRECTORY(@mystage) WHERE SIZE > 100000", - "formatted": "SELECT FILE_URL\nFROM DIRECTORY(@mystage)\nWHERE SIZE > 100000" - }, - { - "dialect": "tsql", - "sql": "SELECT * FROM a..b", - "formatted": "SELECT *\nFROM a..b" - }, - { - "dialect": "tsql", - "sql": "SELECT TOP (SELECT 1) * FROM t", - "formatted": "SELECT TOP (\n SELECT 1\n)\n *\nFROM t" - }, - { - "dialect": "tsql", - "sql": "SELECT ATN2(x, y)", - "formatted": "SELECT ATN2(x, y)" - }, - { - "dialect": "tsql", - "sql": "SELECT EXP(1)", - "formatted": "SELECT EXP(1)" - }, - { - "dialect": "tsql", - "sql": "SELECT SYSDATETIMEOFFSET()", - "formatted": "SELECT SYSDATETIMEOFFSET()" - }, - { - "dialect": "tsql", - "sql": "SELECT COMPRESS('Hello World')", - "formatted": "SELECT COMPRESS('Hello World')" - }, - { - "dialect": "tsql", - "sql": "SELECT TRIM(' test ') AS Result", - "formatted": "SELECT TRIM(' test ') AS Result" - }, - { - "dialect": "tsql", - "sql": "SELECT TRIM('.,! ' FROM ' # test .') AS Result", - "formatted": "SELECT TRIM('.,! ' FROM ' # test .') AS Result" - }, - { - "dialect": "tsql", - "sql": "SELECT * FROM t TABLESAMPLE (10 PERCENT)", - "formatted": "SELECT *\nFROM t TABLESAMPLE (10 PERCENT)" - }, - { - "dialect": "tsql", - "sql": "SELECT * FROM t TABLESAMPLE (20 ROWS)", - "formatted": "SELECT *\nFROM t TABLESAMPLE (20 ROWS)" - }, - { - "dialect": "tsql", - "sql": "SELECT * FROM t TABLESAMPLE (10 PERCENT) REPEATABLE (123)", - "formatted": "SELECT *\nFROM t TABLESAMPLE (10 PERCENT) REPEATABLE (123)" - }, - { - "dialect": "tsql", - "sql": "SELECT CONCAT(column1, column2)", - "formatted": "SELECT CONCAT(column1, column2)" - }, - { - "dialect": "tsql", - "sql": "SELECT TestSpecialChar.Test# FROM TestSpecialChar", - "formatted": "SELECT TestSpecialChar.Test#\nFROM TestSpecialChar" - }, - { - "dialect": "tsql", - "sql": "SELECT TestSpecialChar.Test@ FROM TestSpecialChar", - "formatted": "SELECT TestSpecialChar.Test@\nFROM TestSpecialChar" - }, - { - "dialect": "tsql", - "sql": "SELECT TestSpecialChar.Test$ FROM TestSpecialChar", - "formatted": "SELECT TestSpecialChar.Test$\nFROM TestSpecialChar" - }, - { - "dialect": "tsql", - "sql": "SELECT TestSpecialChar.Test_ FROM TestSpecialChar", - "formatted": "SELECT TestSpecialChar.Test_\nFROM TestSpecialChar" - }, - { - "dialect": "tsql", - "sql": "SELECT TOP (2 + 1) 1", - "formatted": "SELECT TOP (2 + 1) 1" - }, - { - "dialect": "tsql", - "sql": "SELECT * FROM t WHERE NOT c", - "formatted": "SELECT *\nFROM t\nWHERE NOT c" - }, - { - "dialect": "tsql", - "sql": "WITH t1 AS (SELECT 1 AS a), t2 AS (SELECT 1 AS a) SELECT TOP 10 a FROM t1 UNION ALL SELECT TOP 10 a FROM t2", - "formatted": "WITH t1 AS (\n SELECT 1 AS a\n),\n\nt2 AS (\n SELECT 1 AS a\n)\n\nSELECT TOP 10 a\nFROM t1\nUNION ALL\nSELECT TOP 10 a\nFROM t2" - }, - { - "dialect": "tsql", - "sql": "SELECT TOP 10 s.RECORDID, n.c.VALUE('(/*:FORM_ROOT/*:SOME_TAG)[1]', 'float') AS SOME_TAG_VALUE FROM source_table.dbo.source_data AS s(nolock) CROSS APPLY FormContent.nodes('/*:FORM_ROOT') AS N(C)", - "formatted": "SELECT TOP 10\n s.RECORDID,\n n.c.VALUE('(/*:FORM_ROOT/*:SOME_TAG)[1]', 'float') AS SOME_TAG_VALUE\nFROM source_table.dbo.source_data AS s(nolock)\nCROSS APPLY FormContent.nodes('/*:FORM_ROOT') AS N(C)" - }, - { - "dialect": "tsql", - "sql": "WITH t1 AS (SELECT 1 AS a), t2 AS (SELECT 1 AS a) SELECT TOP 10 a FROM t1 UNION ALL SELECT TOP 10 a FROM t2 ORDER BY a DESC", - "formatted": "WITH t1 AS (\n SELECT 1 AS a\n),\n\nt2 AS (\n SELECT 1 AS a\n)\n\nSELECT TOP 10 a\nFROM t1\nUNION ALL\nSELECT TOP 10 a\nFROM t2\nORDER BY a DESC" - }, - { - "dialect": "tsql", - "sql": "WITH t1 AS (SELECT 1 AS a), t2 AS (SELECT 1 AS a) SELECT COUNT(*) FROM (SELECT TOP 10 a FROM t1 UNION ALL SELECT TOP 10 a FROM t2 ORDER BY a DESC) AS t", - "formatted": "WITH t1 AS (\n SELECT 1 AS a\n),\n\nt2 AS (\n SELECT 1 AS a\n)\n\nSELECT COUNT(*)\nFROM (\n SELECT TOP 10 a\n FROM t1\n UNION ALL\n SELECT TOP 10 a\n FROM t2\nORDER BY a DESC\n) AS t" - }, - { - "dialect": "tsql", - "sql": "SELECT 1 AS \"[x]\"", - "formatted": "SELECT 1 AS \"[x]\"" - }, - { - "dialect": "tsql", - "sql": "SELECT * FROM t WITH (TABLOCK, INDEX(myindex))", - "formatted": "SELECT *\nFROM t WITH (TABLOCK, INDEX(myindex))" - }, - { - "dialect": "tsql", - "sql": "SELECT * FROM t WITH (NOWAIT)", - "formatted": "SELECT *\nFROM t WITH (NOWAIT)" - }, - { - "dialect": "tsql", - "sql": "SELECT CASE WHEN a > 1 THEN b END", - "formatted": "SELECT CASE WHEN a > 1 THEN b END" - }, - { - "dialect": "tsql", - "sql": "SELECT * FROM taxi ORDER BY 1 OFFSET 0 ROWS FETCH NEXT 3 ROWS ONLY", - "formatted": "SELECT *\nFROM taxi\nORDER BY\n 1\nOFFSET 0 ROWS\nFETCH NEXT 3 ROWS ONLY" - }, - { - "dialect": "tsql", - "sql": "SELECT Employee_ID, Department_ID FROM @MyTableVar", - "formatted": "SELECT\n Employee_ID,\n Department_ID\nFROM @MyTableVar" - }, - { - "dialect": "tsql", - "sql": "SELECT * FROM #foo", - "formatted": "SELECT *\nFROM #foo" - }, - { - "dialect": "tsql", - "sql": "SELECT * FROM ##foo", - "formatted": "SELECT *\nFROM ##foo" - }, - { - "dialect": "tsql", - "sql": "SELECT a = 1", - "formatted": "SELECT a = 1" - }, - { - "dialect": "tsql", - "sql": "SELECT a = 1 UNION ALL SELECT a = b", - "formatted": "SELECT a = 1\nUNION ALL\nSELECT a = b" - }, - { - "dialect": "tsql", - "sql": "SELECT x FROM @MyTableVar AS m JOIN Employee ON m.EmployeeID = Employee.EmployeeID", - "formatted": "SELECT x\nFROM @MyTableVar AS m\nJOIN Employee\n ON m.EmployeeID = Employee.EmployeeID" - }, - { - "dialect": "tsql", - "sql": "SELECT DISTINCT DepartmentName, PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY BaseRate) OVER (PARTITION BY DepartmentName) AS MedianCont FROM dbo.DimEmployee", - "formatted": "SELECT DISTINCT\n DepartmentName,\n PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY BaseRate) OVER (PARTITION BY DepartmentName) AS MedianCont\nFROM dbo.DimEmployee" - }, - { - "dialect": "tsql", - "sql": "SELECT \"x\".\"y\" FROM foo", - "formatted": "SELECT \"x\".\"y\"\nFROM foo" - }, - { - "dialect": "tsql", - "sql": "SELECT val FROM (VALUES ((TRUE), (FALSE), (NULL))) AS t(val)", - "formatted": "SELECT val\nFROM (\n VALUES\n ((TRUE\n), (FALSE\n), (\n NULL\n))\n) AS t(val)" - }, - { - "dialect": "tsql", - "sql": "SELECT * FROM t FOR XML PATH, BINARY BASE64, ELEMENTS XSINIL", - "formatted": "SELECT *\nFROM t\nFOR XML\n PATH,\n BINARY BASE64,\n ELEMENTS XSINIL" - }, - { - "dialect": "tsql", - "sql": "SELECT (SELECT TOP 5 a, b FROM t ORDER BY b DESC FOR JSON PATH) AS j FROM x", - "formatted": "SELECT\n (\n SELECT TOP 5\n a,\n b\n FROM t\n ORDER BY\n b DESC\n FOR JSON\n PATH\n ) AS j\nFROM x" - }, - { - "dialect": "tsql", - "sql": "SELECT j FROM (SELECT a AS j FROM t FOR JSON PATH) AS x", - "formatted": "SELECT j\nFROM (\n SELECT a AS j\n FROM t\n FOR JSON\n PATH\n) AS x" - }, - { - "dialect": "tsql", - "sql": "SELECT * FROM t FOR JSON PATH, ROOT('Root'), INCLUDE_NULL_VALUES", - "formatted": "SELECT *\nFROM t\nFOR JSON\n PATH,\n ROOT('Root'),\n INCLUDE_NULL_VALUES" - }, - { - "dialect": "tsql", - "sql": "SELECT * FROM t FOR BROWSE", - "formatted": "SELECT *\nFROM t FOR BROWSE" - }, - { - "dialect": "tsql", - "sql": "SELECT CAST(SUBSTRING('ABCD~1234', CHARINDEX('~', 'ABCD~1234') + 1, LEN('ABCD~1234')) AS BIGINT)", - "formatted": "SELECT CAST(SUBSTRING('ABCD~1234', CHARINDEX('~', 'ABCD~1234') + 1, LEN('ABCD~1234')) AS BIGINT)" - }, - { - "dialect": "tsql", - "sql": "SELECT DATEPART(\"dd\", x)", - "formatted": "SELECT DATEPART(\"dd\", x)" - }, - { - "dialect": "tsql", - "sql": "SELECT DATEADD(YEAR, 1, '2017/08/25')", - "formatted": "SELECT DATEADD(YEAR, 1, '2017/08/25')" - }, - { - "dialect": "tsql", - "sql": "SELECT DATEDIFF(HOUR, 1.5, '2021-01-01')", - "formatted": "SELECT DATEDIFF(HOUR, 1.5, '2021-01-01')" - }, - { - "dialect": "tsql", - "sql": "SELECT DATEDIFF_BIG(HOUR, 1.5, '2021-01-01')", - "formatted": "SELECT DATEDIFF_BIG(HOUR, 1.5, '2021-01-01')" - }, - { - "dialect": "tsql", - "sql": "SELECT TOP 10 PERCENT", - "formatted": "SELECT TOP 10 PERCENT" - }, - { - "dialect": "tsql", - "sql": "SELECT TOP 10 PERCENT WITH TIES", - "formatted": "SELECT TOP 10 PERCENT WITH TIES" - }, - { - "dialect": "tsql", - "sql": "SELECT FORMAT(foo, 'dddd', 'de-CH')", - "formatted": "SELECT FORMAT(foo, 'dddd', 'de-CH')" - }, - { - "dialect": "tsql", - "sql": "SELECT FORMAT(EndOfDayRate, 'N', 'en-us')", - "formatted": "SELECT FORMAT(EndOfDayRate, 'N', 'en-us')" - }, - { - "dialect": "tsql", - "sql": "SELECT FORMAT('01-01-1991', 'd.mm.yyyy')", - "formatted": "SELECT FORMAT('01-01-1991', 'd.mm.yyyy')" - }, - { - "dialect": "tsql", - "sql": "SELECT FORMAT(12345, '###.###.###')", - "formatted": "SELECT FORMAT(12345, '###.###.###')" - }, - { - "dialect": "tsql", - "sql": "SELECT FORMAT(1234567, 'f')", - "formatted": "SELECT FORMAT(1234567, 'f')" - }, - { - "dialect": "tsql", - "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME AS OF 'foo'", - "formatted": "SELECT [x]\nFROM [a].[b] FOR SYSTEM_TIME AS OF 'foo'" - }, - { - "dialect": "tsql", - "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME AS OF 'foo' AS alias", - "formatted": "SELECT [x]\nFROM [a].[b] FOR SYSTEM_TIME AS OF 'foo' AS alias" - }, - { - "dialect": "tsql", - "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME FROM c TO d", - "formatted": "SELECT [x]\nFROM [a].[b] FOR SYSTEM_TIME FROM c TO d" - }, - { - "dialect": "tsql", - "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME BETWEEN c AND d", - "formatted": "SELECT [x]\nFROM [a].[b] FOR SYSTEM_TIME BETWEEN c AND d" - }, - { - "dialect": "tsql", - "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME CONTAINED IN (c, d)", - "formatted": "SELECT [x]\nFROM [a].[b] FOR SYSTEM_TIME CONTAINED IN (c, d)" - }, - { - "dialect": "tsql", - "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME ALL AS alias", - "formatted": "SELECT [x]\nFROM [a].[b] FOR SYSTEM_TIME ALL AS alias" - }, - { - "dialect": "tsql", - "sql": "SELECT x FROM a INNER LOOP JOIN b ON b.id = a.id", - "formatted": "SELECT x\nFROM a\nINNER LOOP JOIN b\n ON b.id = a.id" - }, - { - "dialect": "tsql", - "sql": "SELECT * FROM OPENJSON(@json)", - "formatted": "SELECT *\nFROM OPENJSON(@json)" - }, - { - "dialect": "tsql", - "sql": "WITH t AS (SELECT 1) SELECT * FROM t", - "formatted": "WITH t AS (\n SELECT 1\n)\n\nSELECT *\nFROM t" - }, - { - "dialect": "tsql", - "sql": "WITH t AS (SELECT \"c\") SELECT * FROM t", - "formatted": "WITH t AS (\n SELECT \"c\"\n)\n\nSELECT *\nFROM t" - }, - { - "dialect": "tsql", - "sql": "SELECT * FROM (SELECT 1) AS subq", - "formatted": "SELECT *\nFROM (\n SELECT 1\n) AS subq" - }, - { - "dialect": "tsql", - "sql": "SELECT * FROM (SELECT \"c\") AS subq", - "formatted": "SELECT *\nFROM (\n SELECT \"c\"\n) AS subq" - }, - { - "dialect": "tsql", - "sql": "SELECT NEXT VALUE FOR db.schema.sequence_name OVER (ORDER BY foo), col", - "formatted": "SELECT\n NEXT VALUE FOR db.schema.sequence_name OVER (ORDER BY foo),\n col" - }, - { - "dialect": "tsql", - "sql": "SELECT COUNT(1) FROM x", - "formatted": "SELECT COUNT(1)\nFROM x" - }, - { - "dialect": "tsql", - "sql": "SELECT COUNT_BIG(1) FROM x", - "formatted": "SELECT COUNT_BIG(1)\nFROM x" - }, - { - "dialect": "tsql", - "sql": "SELECT 1; SELECT 2", - "formatted": "SELECT 1;\nSELECT 2" - }, - { - "dialect": "tsql", - "sql": "SELECT go", - "formatted": "SELECT go" - }, - { - "dialect": "tsql", - "sql": "SELECT 1 WHERE EXISTS(SELECT 1)", - "formatted": "SELECT 1\nWHERE\n EXISTS(\n SELECT 1\n )" - }, - { - "dialect": "tsql", - "sql": "SELECT a = 1", - "formatted": "SELECT a = 1" - }, - { - "dialect": "tsql", - "sql": "SELECT * FROM [#temp_table]", - "formatted": "SELECT *\nFROM [#temp_table]" - }, - { - "dialect": "tsql", - "sql": "SELECT * FROM [##temp_table]", - "formatted": "SELECT *\nFROM [##temp_table]" - }, - { - "dialect": "tsql", - "sql": "SELECT * FROM @x", - "formatted": "SELECT *\nFROM @x" - } -] diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/helpers.rs b/crates/sqlbuild-rules/src/sql_lint/tests/helpers.rs deleted file mode 100644 index 7201b18958..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/tests/helpers.rs +++ /dev/null @@ -1,304 +0,0 @@ -use std::str::FromStr; - -use polyglot_sql::{Dialect, DialectType}; -use serde_json::{Map, Value, json}; - -use crate::sql_lint::_helpers::formatter::format_requests; -use crate::sql_lint::main::engine::lint_json; -use crate::sql_lint::main::formatter::format_json; -use crate::sql_lint::models::FormatRequest; -use crate::sql_lint::tests::test_types::CrossJoinTestCase; -use sqlbuild_analysis::sql_tokens::constants::CALL_SYNTAX_FUNCTIONS; -use sqlbuild_analysis::sql_tokens::main::builtin_function_names::builtin_function_names; -use sqlbuild_analysis::sql_tokens::main::canonical_tokens::canonical_tokens; -use sqlbuild_analysis::sql_tokens::main::query_fingerprint::query_fingerprint; - -/// The dependency sentinel the cross join cases name `order_policy`. -pub(crate) const POLICY_SENTINEL: &str = "__sqb_lint_7__"; - -/// Format a JSON array of format requests in parallel, as a JSON array of responses. -pub(crate) fn format_batch_json(request_json: &str) -> Result { - let requests: Vec = - serde_json::from_str(request_json).map_err(|error| error.to_string())?; - serde_json::to_string(&format_requests(requests)).map_err(|error| error.to_string()) -} - -pub(crate) fn nested_function_sql(depth: usize) -> String { - format!("SELECT {}1{}", "F(".repeat(depth), ")".repeat(depth)) -} - -pub(crate) fn diagnostics(sql: &str) -> Result, String> { - let response = lint_json( - &json!({ - "version": 1, - "sql": sql, - "dialect": "snowflake" - }) - .to_string(), - )?; - diagnostic_values(&response) -} - -pub(crate) fn diagnostics_for_rules(sql: &str, rules: &[&str]) -> Result, String> { - diagnostics_for_dialect(sql, "snowflake", rules) -} - -pub(crate) fn diagnostics_for_dialect( - sql: &str, - dialect: &str, - rules: &[&str], -) -> Result, String> { - let request = json!({ - "version": 1, - "sql": sql, - "dialect": dialect, - "enabled_rules": rules, - }); - let response = lint_json(&request.to_string())?; - diagnostic_values(&response) -} - -fn diagnostic_values(response: &str) -> Result, String> { - let payload: Value = serde_json::from_str(response).map_err(|error| error.to_string())?; - payload["diagnostics"] - .as_array() - .cloned() - .ok_or_else(|| "diagnostics should be an array".to_string()) -} - -/// A narrow width so that wrapping is exercised on most corpus queries. -const CORPUS_LINE_WIDTH: usize = 40; - -/// Format every corpus query of one dialect and return the formatted and refused counts. -pub(crate) fn check_dialect_corpus( - corpus: &[Value], - dialect: &str, -) -> Result<(usize, usize), String> { - let mut outcomes: Vec = Vec::new(); - for entry in corpus.iter().filter(|entry| entry["dialect"] == dialect) { - let sql = entry["sql"].as_str().unwrap_or_default(); - let response = format_once(sql, dialect, None)?; - let formatted = response["formatted"] == true; - let actual = response - .get("sql") - .filter(|_| formatted) - .cloned() - .unwrap_or(Value::Null); - let expected = entry["formatted"].as_str().map_or(Value::Null, Value::from); - assert_eq!(actual, expected, "{dialect}: {sql}"); - let printed = actual.as_str().unwrap_or(sql); - assert!( - only_keyword_case_differs(sql, printed, dialect)?, - "{dialect} fingerprint changed beyond keyword case: {sql}" - ); - let previous_layout: Option = entry["previously_formatted"] - .as_str() - .map(|previous| relaid_previous_layout(previous, dialect)) - .transpose()?; - let layouts = [ - entry["formatted"].as_str().map(str::to_string), - previous_layout, - ]; - for stable in layouts.iter().flatten() { - let stable = stable.as_str(); - let again = format_once(stable, dialect, None)?; - assert_eq!(again["sql"], stable, "{dialect} layout drifted: {stable}"); - assert_eq!( - again["changed"], false, - "{dialect} layout drifted: {stable}" - ); - assert_eq!( - query_fingerprint(again["sql"].as_str().unwrap_or_default(), dialect)?, - query_fingerprint(stable, dialect)?, - "{dialect} formatting changed the fingerprint of formatted SQL: {stable}" - ); - } - let wrapped = format_once(sql, dialect, Some(CORPUS_LINE_WIDTH))?; - assert_eq!(wrapped["formatted"], formatted, "{dialect} wrapped: {sql}"); - let wrapped_sql = wrapped["sql"].as_str().unwrap_or_default(); - let rewrapped = format_once(wrapped_sql, dialect, Some(CORPUS_LINE_WIDTH))?; - assert_eq!( - rewrapped["sql"], wrapped_sql, - "{dialect} wrapping drifted: {sql}" - ); - outcomes.push(formatted); - } - let formatted = outcomes.iter().filter(|outcome| **outcome).count(); - Ok((formatted, outcomes.len() - formatted)) -} - -/// Whether `printed` has the fingerprint tokens of `sql` except for words upper-cased by format. -fn only_keyword_case_differs(sql: &str, printed: &str, dialect: &str) -> Result { - let parser = Dialect::get(DialectType::from_str(dialect).map_err(|error| error.to_string())?); - let left_tokens = parser.tokenize(sql).map_err(|error| error.to_string())?; - let right_tokens = parser - .tokenize(printed) - .map_err(|error| error.to_string())?; - let left = canonical_tokens(sql, &left_tokens, &parser)?; - let right = canonical_tokens(printed, &right_tokens, &parser)?; - Ok(left.len() == right.len() - && left.iter().zip(&right).all(|(before, after)| { - before == after - || (before.text.eq_ignore_ascii_case(&after.text) - && after.text == after.text.to_ascii_uppercase()) - })) -} - -/// Re-format a layout of an earlier formatter release, which must keep its fingerprint. -fn relaid_previous_layout(previous: &str, dialect: &str) -> Result { - let relaid = format_once(previous, dialect, None)?; - let sql = relaid["sql"].as_str().unwrap_or_default().to_string(); - assert_eq!( - query_fingerprint(&sql, dialect)?, - query_fingerprint(previous, dialect)?, - "{dialect} formatting changed the fingerprint of a previous layout: {previous}" - ); - Ok(sql) -} - -pub(crate) fn format_at_width( - sql: &str, - dialect: &str, - line_width: usize, -) -> Result { - format_once(sql, dialect, Some(line_width)) -} - -fn format_once(sql: &str, dialect: &str, line_width: Option) -> Result { - let request = json!({"version": 1, "sql": sql, "dialect": dialect, "line_width": line_width}); - let response = format_json(&request.to_string())?; - serde_json::from_str(&response).map_err(|error| error.to_string()) -} - -/// Call names the formatter treats as built-ins in `dialect`, plus keyword-like call names. -const KEYWORD_LIKE_CALLS: [&str; 11] = [ - "if", - "iff", - "left", - "right", - "replace", - "insert", - "format", - "filter", - "date", - "time", - "timestamp", -]; - -/// Return names no 2/1/0-argument call could format, and calls printed with misplaced spaces. -pub(crate) fn function_call_spacing( - dialect: &str, - line_width: usize, - argument: &str, -) -> Result<(Vec, Vec), String> { - let dialect_type = DialectType::from_str(dialect).map_err(|error| error.to_string())?; - let mut names: Vec = builtin_function_names(dialect_type) - .into_iter() - .chain(KEYWORD_LIKE_CALLS.iter().map(|name| (*name).to_string())) - .filter(|name| name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')) - .filter(|name| !CALL_SYNTAX_FUNCTIONS.contains(&name.to_ascii_uppercase().as_str())) - .collect(); - names.sort_unstable(); - names.dedup(); - let mut misplaced: Vec = Vec::new(); - for arguments in [ - format!("{argument}, 1"), - argument.to_string(), - String::new(), - ] { - let responses = format_calls(&names, &arguments, dialect, line_width)?; - misplaced.extend( - names - .iter() - .zip(&responses) - .filter(|(_, response)| response["formatted"] == true) - .filter(|(name, response)| { - !is_tight_call( - name, - response["sql"].as_str().unwrap_or_default(), - &arguments, - ) - }) - .map(|(name, response)| format!("{name}: {}", response["sql"])), - ); - names = names - .into_iter() - .zip(&responses) - .filter(|(_, response)| response["formatted"] != true) - .map(|(name, _)| name) - .collect(); - } - Ok((names, misplaced)) -} - -fn format_calls( - names: &[String], - arguments: &str, - dialect: &str, - line_width: usize, -) -> Result, String> { - let requests: Vec = names - .iter() - .map(|name| { - json!({ - "version": 1, - "sql": format!("SELECT {name}({arguments}) AS x FROM t"), - "dialect": dialect, - "line_width": line_width, - }) - }) - .collect(); - serde_json::from_str(&format_batch_json(&Value::from(requests).to_string())?) - .map_err(|error| error.to_string()) -} - -/// Whether `sql` prints `name(` in authored or upper case with no space inside the call. -fn is_tight_call(name: &str, sql: &str, arguments: &str) -> bool { - [name.to_string(), name.to_ascii_uppercase()] - .iter() - .any(|printed| { - sql.contains(&format!("{printed}({arguments})")) - || sql.contains(&format!("{printed}(\n")) - }) - && !sql.contains("( ") - && !sql - .to_ascii_uppercase() - .contains(&format!("{} (", name.to_ascii_uppercase())) -} - -/// The `SQBRSQL038` messages for one cross join case. -pub(crate) fn cross_join_messages(test_case: &CrossJoinTestCase) -> Result, String> { - cross_join_texts(test_case, "message") -} - -/// The `SQBRSQL038` remediations for one cross join case. -pub(crate) fn cross_join_remediations( - test_case: &CrossJoinTestCase, -) -> Result, String> { - cross_join_texts(test_case, "remediation") -} - -fn cross_join_texts(test_case: &CrossJoinTestCase, field: &str) -> Result, String> { - let keys: Map = test_case - .one_row_relations - .iter() - .map(|name| ((*name).to_owned(), json!([[]]))) - .collect(); - let request = json!({ - "version": 1, - "sql": test_case.sql, - "dialect": test_case.dialect, - "enabled_rules": ["SQBRSQL038"], - "relation_keys": keys, - "relation_names": { POLICY_SENTINEL: "order_policy" }, - }); - let response: Value = serde_json::from_str(&lint_json(&request.to_string())?) - .map_err(|error| error.to_string())?; - Ok(response["diagnostics"] - .as_array() - .cloned() - .unwrap_or_default() - .iter() - .map(|diagnostic| diagnostic[field].as_str().unwrap_or_default().to_owned()) - .collect()) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_authored_tokens.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_authored_tokens.rs deleted file mode 100644 index cb29e9c288..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_authored_tokens.rs +++ /dev/null @@ -1,416 +0,0 @@ -use serde_json::{Value, json}; - -use std::collections::BTreeMap; - -use crate::sql_lint::_helpers::token_layout::{ - KeywordRecase, TokenTexts, tokenize, verify_printed_tokens, -}; -use crate::sql_lint::main::formatter::format_json; -use crate::sql_lint::tests::test_types; - -#[test] -fn given_generator_respellings_when_formatting_then_authored_spellings_are_kept() --> Result<(), String> { - let test_cases = [ - test_types::FormatTestCase { - description: "Snowflake STARTSWITH keeps its authored name", - sql: "select startswith(order_code, 'EU') as is_eu_order from orders", - expected_sql: "SELECT startswith(order_code, 'EU') AS is_eu_order\nFROM orders", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "function synonyms keep their authored names", - sql: "select SUBSTR(order_code, 1, 2) as region, TIMESTAMPDIFF(day, ordered_at, shipped_at) as days_to_ship, TRY_TO_DECIMAL(amount_text, 10, 2) as amount from orders", - expected_sql: "SELECT\n SUBSTR(order_code, 1, 2) AS region,\n TIMESTAMPDIFF(day, ordered_at, shipped_at) AS days_to_ship,\n TRY_TO_DECIMAL(amount_text, 10, 2) AS amount\nFROM orders", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "operator and function spellings stay authored", - sql: "select IFNULL(status, 'open') as status from orders where quantity != 0", - expected_sql: "SELECT IFNULL(status, 'open') AS status\nFROM orders\nWHERE quantity != 0", - expected_changed: true, - }, - ]; - for test_case in &test_cases { - let response = format_json( - &json!({"version": 1, "sql": test_case.sql, "dialect": "snowflake"}).to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - assert_eq!( - payload["sql"], test_case.expected_sql, - "{}", - test_case.description - ); - assert_eq!( - payload["changed"], test_case.expected_changed, - "{}", - test_case.description - ); - } - Ok(()) -} - -#[test] -fn given_generator_restructuring_when_formatting_then_authored_tokens_are_laid_out() --> Result<(), String> { - let test_cases = [ - test_types::DialectFormatTestCase { - description: "PostgreSQL comma SUBSTRING arguments are not rewritten to FROM/FOR", - dialect: "postgres", - sql: "select substring(code, 1, 2) as region from orders", - expected_sql: "SELECT SUBSTRING(code, 1, 2) AS region\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "DuckDB adjacent string literals after a quoted-in-oracle alias keep comma spacing", - dialect: "duckdb", - sql: "select sku as replace, 'north' 'east' as region, sku from products", - expected_sql: "SELECT\n sku AS replace,\n 'north' 'east' AS region,\n sku\nFROM products", - }, - test_types::DialectFormatTestCase { - description: "DuckDB adjacent string literals after a plain column keep comma spacing", - dialect: "duckdb", - sql: "select sku, 'north' 'east' as region from products", - expected_sql: "SELECT\n sku,\n 'north' 'east' AS region\nFROM products", - }, - test_types::DialectFormatTestCase { - description: "DuckDB SELECT ALL keeps ALL", - dialect: "duckdb", - sql: "select all order_id from orders", - expected_sql: "SELECT ALL order_id\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "Snowflake SELECT ALL keeps ALL", - dialect: "snowflake", - sql: "select all order_id from orders", - expected_sql: "SELECT ALL order_id\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "PostgreSQL SELECT ALL keeps ALL", - dialect: "postgres", - sql: "select all order_id from orders", - expected_sql: "SELECT ALL order_id\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "BigQuery SELECT ALL keeps ALL", - dialect: "bigquery", - sql: "select all order_id from orders", - expected_sql: "SELECT ALL order_id\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "Databricks SELECT ALL keeps ALL", - dialect: "databricks", - sql: "select all order_id from orders", - expected_sql: "SELECT all order_id\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "T-SQL SELECT ALL keeps ALL", - dialect: "tsql", - sql: "select all order_id from orders", - expected_sql: "SELECT ALL order_id\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "implicit aliases stay implicit", - dialect: "duckdb", - sql: "select order_id id, amount from orders o", - expected_sql: "SELECT\n order_id id,\n amount\nFROM orders o", - }, - test_types::DialectFormatTestCase { - description: "a trailing select comma and statement terminator are kept", - dialect: "duckdb", - sql: "select order_id, amount, from orders;", - expected_sql: "SELECT\n order_id,\n amount,\nFROM orders;", - }, - test_types::DialectFormatTestCase { - description: "adjacent authored tokens are never split", - dialect: "duckdb", - sql: "select $name as v from orders where status != 1", - expected_sql: "SELECT $name AS v\nFROM orders\nWHERE status != 1", - }, - test_types::DialectFormatTestCase { - description: "BigQuery numeric-prefixed path parts stay joined", - dialect: "bigquery", - sql: "select * from orders.archive.25_", - expected_sql: "SELECT *\nFROM orders.archive.25_", - }, - ]; - for test_case in &test_cases { - let response = format_json( - &json!({"version": 1, "sql": test_case.sql, "dialect": test_case.dialect}).to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - assert_eq!( - payload["sql"], test_case.expected_sql, - "{}", - test_case.description - ); - let again = format_json( - &json!({"version": 1, "sql": test_case.expected_sql, "dialect": test_case.dialect}) - .to_string(), - )?; - let repeated: Value = serde_json::from_str(&again).map_err(|error| error.to_string())?; - assert_eq!(repeated["changed"], false, "{}", test_case.description); - } - Ok(()) -} - -#[test] -fn given_case_or_line_sensitive_tokens_when_formatting_then_they_are_kept_exactly() --> Result<(), String> { - let test_cases = [ - test_types::DialectFormatTestCase { - description: "DuckDB keyword-named aliases and qualified names keep their case", - dialect: "duckdb", - sql: "select pos as index, sku as replace, t.view from orders t", - expected_sql: "SELECT\n pos AS index,\n sku AS replace,\n t.view\nFROM orders t", - }, - test_types::DialectFormatTestCase { - description: "BigQuery keyword-named table names keep their case", - dialect: "bigquery", - sql: "select * from inventory.view", - expected_sql: "SELECT *\nFROM inventory.view", - }, - test_types::DialectFormatTestCase { - description: "Snowflake path keys keep their case", - dialect: "snowflake", - sql: "select payload:Customer.Name as name, payload:Date::string as day from orders", - expected_sql: "SELECT\n payload:Customer.Name AS name,\n payload:Date::string AS day\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "user-defined functions keep their case, built-ins are uppercased", - dialect: "bigquery", - sql: "select MyFunc(a), ds.MyUdf(b), coalesce(a, b) from orders", - expected_sql: "SELECT\n MyFunc(a),\n ds.MyUdf(b),\n COALESCE(a, b)\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "DuckDB non-reserved keyword operands keep their case before any operator", - dialect: "duckdb", - sql: "select a, Rows << 1, Rows == 1, Rows ^ 2 from t", - expected_sql: "SELECT\n a,\n Rows << 1,\n Rows == 1,\n Rows ^ 2\nFROM t", - }, - test_types::DialectFormatTestCase { - description: "BigQuery comma-joined keyword-named tables keep their case", - dialect: "bigquery", - sql: "select * from ds.a, View v", - expected_sql: "SELECT *\nFROM ds.a, View v", - }, - test_types::DialectFormatTestCase { - description: "DuckDB adjacent string literals keep their line break", - dialect: "duckdb", - sql: "select 'north'\n'east' as region from orders", - expected_sql: "SELECT\n 'north'\n 'east' AS region\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "PostgreSQL adjacent string literals keep their line break", - dialect: "postgres", - sql: "select 'north'\n 'east' as region from orders", - expected_sql: "SELECT\n 'north'\n 'east' AS region\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "Snowflake double-slash comments are kept", - dialect: "snowflake", - sql: "select order_id, // keep this note\n customer_id from orders", - expected_sql: "SELECT\n order_id, // keep this note\n customer_id\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "BigQuery hash comments are kept", - dialect: "bigquery", - sql: "select order_id, # keep this note\n customer_id from orders", - expected_sql: "SELECT\n order_id, # keep this note\n customer_id\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "T-SQL temporary table hashes are not comments", - dialect: "tsql", - sql: "select #orders.order_id from #orders", - expected_sql: "SELECT #orders.order_id\nFROM #orders", - }, - test_types::DialectFormatTestCase { - description: "DuckDB integer division and slashes inside strings are not comments", - dialect: "duckdb", - sql: "select 7 // 2 as half, 'https://example.com/a' as url from orders", - expected_sql: "SELECT\n 7 // 2 AS half,\n 'https://example.com/a' AS url\nFROM orders", - }, - test_types::DialectFormatTestCase { - description: "Snowflake double slashes inside strings are not comments", - dialect: "snowflake", - sql: "select 'https://example.com//a' as url from orders", - expected_sql: "SELECT 'https://example.com//a' AS url\nFROM orders", - }, - ]; - for test_case in &test_cases { - let response = format_json( - &json!({"version": 1, "sql": test_case.sql, "dialect": test_case.dialect}).to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - assert_eq!( - payload["sql"], test_case.expected_sql, - "{}", - test_case.description - ); - let again = format_json( - &json!({"version": 1, "sql": test_case.expected_sql, "dialect": test_case.dialect}) - .to_string(), - )?; - let repeated: Value = serde_json::from_str(&again).map_err(|error| error.to_string())?; - assert_eq!(repeated["changed"], false, "{}", test_case.description); - } - Ok(()) -} - -#[test] -fn given_formatted_candidates_when_checking_token_invariant_then_only_layout_may_differ() { - let test_cases = [ - test_types::TokenInvariantTestCase { - description: "whitespace and word case may change", - before: "select a,b from orders -- note", - after: "SELECT\n a,\n b\nFROM orders -- note", - recases: &[], - expected_error: None, - }, - test_types::TokenInvariantTestCase { - description: "blank lines between CTEs keep comments on their tokens", - before: "WITH orders AS (SELECT 1), -- note\n-- paid only\npaid AS (SELECT 2) SELECT * FROM paid", - after: "WITH orders AS (\n SELECT 1\n), -- note\n\n-- paid only\npaid AS (\n SELECT 2\n)\n\nSELECT *\nFROM paid", - recases: &[], - expected_error: None, - }, - test_types::TokenInvariantTestCase { - description: "a comment moved across a CTE blank line is refused", - before: "WITH orders AS (SELECT 1), -- note\npaid AS (SELECT 2) SELECT * FROM paid", - after: "WITH orders AS (\n SELECT 1\n)\n-- note\n,\n\npaid AS (\n SELECT 2\n)\n\nSELECT *\nFROM paid", - recases: &[], - expected_error: Some("native formatter could not preserve comment token attachments"), - }, - test_types::TokenInvariantTestCase { - description: "a renamed function is refused", - before: "select startswith(code, 'EU') from orders", - after: "SELECT STARTS_WITH(code, 'EU') FROM orders", - recases: &[], - expected_error: Some( - "native formatter would change authored SQL tokens, not only layout", - ), - }, - test_types::TokenInvariantTestCase { - description: "an added alias keyword is refused", - before: "select a b from orders", - after: "SELECT a AS b FROM orders", - recases: &[], - expected_error: Some( - "native formatter would change authored SQL tokens, not only layout", - ), - }, - test_types::TokenInvariantTestCase { - description: "string literal whitespace is refused", - before: "select 'a b' from orders", - after: "SELECT 'a b' FROM orders", - recases: &[], - expected_error: Some( - "native formatter would change authored SQL tokens, not only layout", - ), - }, - test_types::TokenInvariantTestCase { - description: "a recorded keyword recase from the layout guide is allowed", - before: "select a from orders order by a nulls last", - after: "SELECT a FROM orders ORDER BY a NULLS LAST", - recases: &[(7, "NULLS", true), (8, "LAST", true)], - expected_error: None, - }, - test_types::TokenInvariantTestCase { - description: "an unrecorded non-reserved keyword recase is refused", - before: "select a from orders order by a nulls last", - after: "SELECT a FROM orders ORDER BY a NULLS LAST", - recases: &[], - expected_error: Some( - "native formatter would change authored SQL tokens, not only layout", - ), - }, - test_types::TokenInvariantTestCase { - description: "a recase whose guide token is not a keyword is refused", - before: "select a from orders order by a nulls last", - after: "SELECT a FROM orders ORDER BY a NULLS LAST", - recases: &[(7, "NULLS", false), (8, "LAST", false)], - expected_error: Some( - "native formatter would change authored SQL tokens, not only layout", - ), - }, - test_types::TokenInvariantTestCase { - description: "an identifier case change is refused", - before: "select a from orders", - after: "SELECT a FROM Orders", - recases: &[], - expected_error: Some( - "native formatter would change authored SQL tokens, not only layout", - ), - }, - test_types::TokenInvariantTestCase { - description: "a path key case change is refused", - before: "select payload:customerId from orders", - after: "SELECT payload:CUSTOMERID FROM orders", - recases: &[], - expected_error: Some( - "native formatter would change authored SQL tokens, not only layout", - ), - }, - test_types::TokenInvariantTestCase { - description: "a built-in function may be recased", - before: "select coalesce(a, 0) from orders", - after: "SELECT COALESCE(a, 0) FROM orders", - recases: &[], - expected_error: None, - }, - test_types::TokenInvariantTestCase { - description: "joining adjacent string literals across a line break is refused", - before: "select 'a'\n'b' from orders", - after: "SELECT 'a' 'b' FROM orders", - recases: &[], - expected_error: Some( - "native formatter would join adjacent string literals across a line break", - ), - }, - test_types::TokenInvariantTestCase { - description: "a dropped double-slash comment is refused", - before: "select a // note\nfrom orders", - after: "SELECT a FROM orders", - recases: &[], - expected_error: Some("native formatter could not preserve comment token attachments"), - }, - test_types::TokenInvariantTestCase { - description: "a comment moved to another token is refused", - before: "select a, -- note\n b from orders", - after: "SELECT a, b -- note\nFROM orders", - recases: &[], - expected_error: Some("native formatter could not preserve comment token attachments"), - }, - ]; - let dialect = polyglot_sql::Dialect::get(polyglot_sql::DialectType::Snowflake); - for test_case in test_cases { - let recases: BTreeMap = test_case - .recases - .iter() - .map(|(index, generated, keyword)| { - ( - *index, - KeywordRecase { - generated: (*generated).to_string(), - keyword: *keyword, - }, - ) - }) - .collect(); - let before_tokens = tokenize(test_case.before, &dialect).expect("before tokenizes"); - let after_tokens = tokenize(test_case.after, &dialect).expect("after tokenizes"); - let before_texts = - TokenTexts::read(test_case.before, &before_tokens, &dialect).expect("before reads"); - let actual = verify_printed_tokens( - (test_case.before, &before_tokens, &before_texts), - (test_case.after, &after_tokens), - &dialect, - &recases, - ); - assert_eq!( - actual.err().as_deref(), - test_case.expected_error, - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_batch_lint.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_batch_lint.rs deleted file mode 100644 index 5e4bf9ced7..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_batch_lint.rs +++ /dev/null @@ -1,43 +0,0 @@ -use crate::sql_lint::main::batch_engine::lint_batch_json; -use crate::sql_lint::main::engine::lint_json; -use crate::sql_lint::tests::test_types::BatchLintTestCase; -use serde_json::{Value, json}; - -#[test] -fn given_mixed_lint_inputs_when_batching_then_preserves_order_errors_and_findings() { - let test_cases = [BatchLintTestCase { - description: "mixed valid and invalid SQL retains standalone evidence", - queries: &[ - "SELECT 1", - "SELECT FROM", - "SELECT order_id FROM orders WHERE order_id = NULL", - ], - expected_responses: 3, - }]; - for test_case in test_cases { - let requests: Vec = test_case - .queries - .iter() - .map(|sql| json!({"version": 1, "sql": sql, "dialect": "duckdb"})) - .collect(); - let expected: Vec = requests - .iter() - .map(|request| { - serde_json::to_value(lint_json(&request.to_string()).map(|response| { - serde_json::from_str::(&response).expect("valid response") - })) - .expect("serializable result") - }) - .collect(); - let response = lint_batch_json(&serde_json::to_string(&requests).expect("valid requests")) - .expect("valid batch"); - let actual: Vec = serde_json::from_str(&response).expect("valid response"); - assert_eq!( - actual.len(), - test_case.expected_responses, - "{}", - test_case.description - ); - assert_eq!(actual, expected, "{}", test_case.description); - } -} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_comment_formatter.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_comment_formatter.rs deleted file mode 100644 index adc81ce1ed..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_comment_formatter.rs +++ /dev/null @@ -1,95 +0,0 @@ -use serde_json::{Value, json}; - -use crate::sql_lint::main::formatter::format_json; -use crate::sql_lint::tests::test_types; - -#[test] -fn given_leading_cte_comments_when_formatting_then_comments_stay_above_their_cte() --> Result<(), String> { - let test_cases = [ - test_types::FormatTestCase { - description: "leading comment before a middle CTE stays above it", - sql: "WITH\nupstream AS (\n SELECT * FROM orders\n),\n\n-- Explains the next CTE: line one,\n-- line two.\ndistinct_rows AS (\n SELECT order_id FROM upstream GROUP BY ALL\n)\n\nSELECT order_id FROM distinct_rows", - expected_sql: "WITH upstream AS (\n SELECT *\n FROM orders\n),\n\n-- Explains the next CTE: line one,\n-- line two.\ndistinct_rows AS (\n SELECT order_id\n FROM upstream\n GROUP BY ALL\n)\n\nSELECT order_id\nFROM distinct_rows", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "formatted leading CTE comment is idempotent", - sql: "WITH upstream AS (\n SELECT *\n FROM orders\n),\n\n-- Explains the next CTE: line one,\n-- line two.\ndistinct_rows AS (\n SELECT order_id\n FROM upstream\n GROUP BY ALL\n)\n\nSELECT order_id\nFROM distinct_rows", - expected_sql: "WITH upstream AS (\n SELECT *\n FROM orders\n),\n\n-- Explains the next CTE: line one,\n-- line two.\ndistinct_rows AS (\n SELECT order_id\n FROM upstream\n GROUP BY ALL\n)\n\nSELECT order_id\nFROM distinct_rows", - expected_changed: false, - }, - test_types::FormatTestCase { - description: "leading comment before the first CTE stays above it", - sql: "WITH\n-- Explains the first CTE.\nupstream AS (SELECT * FROM orders)\nSELECT order_id FROM upstream", - expected_sql: "WITH\n-- Explains the first CTE.\nupstream AS (\n SELECT *\n FROM orders\n)\n\nSELECT order_id\nFROM upstream", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "leading comment before the last of three CTEs stays above it", - sql: "WITH a AS (SELECT 1 AS x),\nb AS (SELECT x FROM a),\n /* Explains the last CTE. */\nc AS (SELECT x FROM b)\nSELECT x FROM c", - expected_sql: "WITH a AS (\n SELECT 1 AS x\n),\n\nb AS (\n SELECT x\n FROM a\n),\n\n/* Explains the last CTE. */\nc AS (\n SELECT x\n FROM b\n)\n\nSELECT x\nFROM c", - expected_changed: true, - }, - ]; - - for test_case in &test_cases { - let response = format_json( - &json!({"version": 1, "sql": test_case.sql, "dialect": "snowflake"}).to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - assert_eq!( - payload["sql"], test_case.expected_sql, - "{}", - test_case.description - ); - assert_eq!( - payload["changed"], test_case.expected_changed, - "{}", - test_case.description - ); - } - Ok(()) -} - -#[test] -fn given_trailing_comments_after_the_last_token_when_formatting_then_no_newline_is_added() --> Result<(), String> { - let test_cases = [ - test_types::FormatTestCase { - description: "a trailing line comment ends the output without a newline", - sql: "SELECT 1 AS order_id\nFROM orders -- latest orders", - expected_sql: "SELECT 1 AS order_id\nFROM orders -- latest orders", - expected_changed: false, - }, - test_types::FormatTestCase { - description: "line comments after the last token keep no trailing newline", - sql: "select 1 as order_id from orders\n\n-- first note\n\n-- second note", - expected_sql: "SELECT 1 AS order_id\nFROM orders -- first note\n -- second note", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "a trailing block comment ends the output", - sql: "SELECT 1 AS order_id\nFROM orders /* latest orders */", - expected_sql: "SELECT 1 AS order_id\nFROM orders /* latest orders */", - expected_changed: false, - }, - ]; - for test_case in &test_cases { - let response = format_json( - &json!({"version": 1, "sql": test_case.sql, "dialect": "duckdb"}).to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - assert_eq!( - payload["sql"], test_case.expected_sql, - "{}", - test_case.description - ); - assert_eq!( - payload["changed"], test_case.expected_changed, - "{}", - test_case.description - ); - } - Ok(()) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_cross_joins.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_cross_joins.rs deleted file mode 100644 index 2817d87427..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_cross_joins.rs +++ /dev/null @@ -1,286 +0,0 @@ -use crate::sql_lint::main::cross_join_subject::cross_join_subject; -use crate::sql_lint::tests::helpers::{ - POLICY_SENTINEL, cross_join_messages, cross_join_remediations, -}; -use crate::sql_lint::tests::test_types::{CrossJoinSubjectTestCase, CrossJoinTestCase}; - -const GENERIC_MESSAGE: &str = "CROSS JOIN creates an explicit cartesian product"; -const UNNAMED_MESSAGE: &str = "CROSS JOIN creates an explicit cartesian product with an unnamed relation; only a file-wide exception (rule and path) can cover it"; - -#[test] -fn given_cross_joins_when_one_side_is_provably_one_row_then_only_multiplying_joins_are_reported() --> Result<(), String> { - let test_cases = [ - CrossJoinTestCase { - description: "two unknown relations multiply rows", - sql: "SELECT o.order_id FROM orders AS o CROSS JOIN customers AS c", - dialect: "duckdb", - one_row_relations: &[], - expected_messages: &[ - "CROSS JOIN with `customers` creates an explicit cartesian product", - ], - }, - CrossJoinTestCase { - description: "a one-row seed on the right", - sql: "SELECT o.order_id FROM orders AS o CROSS JOIN __sqb_lint_7__ AS p", - dialect: "duckdb", - one_row_relations: &[POLICY_SENTINEL], - expected_messages: &[], - }, - CrossJoinTestCase { - description: "a seed with several rows names its relation", - sql: "SELECT o.order_id FROM orders AS o CROSS JOIN __sqb_lint_7__ AS p", - dialect: "duckdb", - one_row_relations: &[], - expected_messages: &[ - "CROSS JOIN with `order_policy` creates an explicit cartesian product", - ], - }, - CrossJoinTestCase { - description: "a one-row seed on the left of the first join", - sql: "SELECT o.order_id FROM __sqb_lint_7__ AS p CROSS JOIN orders AS o", - dialect: "duckdb", - one_row_relations: &[POLICY_SENTINEL], - expected_messages: &[], - }, - CrossJoinTestCase { - description: "an aggregate without GROUP BY carried through a filtering CTE", - sql: "WITH totals AS (SELECT COUNT(*) AS order_count FROM orders), \ - kept AS (SELECT order_count FROM totals WHERE order_count > 0) \ - SELECT o.order_id, k.order_count FROM orders AS o CROSS JOIN kept AS k", - dialect: "duckdb", - one_row_relations: &[], - expected_messages: &[], - }, - CrossJoinTestCase { - description: "an aggregate with GROUP BY can return many rows", - sql: "WITH totals AS (SELECT customer_id, COUNT(*) AS order_count FROM orders GROUP BY customer_id) \ - SELECT o.order_id FROM orders AS o CROSS JOIN totals AS t", - dialect: "duckdb", - one_row_relations: &[], - expected_messages: &["CROSS JOIN with `totals` creates an explicit cartesian product"], - }, - CrossJoinTestCase { - description: "a windowed count is not an aggregate", - sql: "WITH totals AS (SELECT COUNT(*) OVER () AS order_count FROM orders) \ - SELECT o.order_id FROM orders AS o CROSS JOIN totals AS t", - dialect: "duckdb", - one_row_relations: &[], - expected_messages: &["CROSS JOIN with `totals` creates an explicit cartesian product"], - }, - CrossJoinTestCase { - description: "LIMIT 1", - sql: "WITH latest AS (SELECT order_id FROM orders ORDER BY order_id DESC LIMIT 1) \ - SELECT c.customer_id FROM customers AS c CROSS JOIN latest AS l", - dialect: "duckdb", - one_row_relations: &[], - expected_messages: &[], - }, - CrossJoinTestCase { - description: "LIMIT 2 can return two rows", - sql: "WITH latest AS (SELECT order_id FROM orders ORDER BY order_id DESC LIMIT 2) \ - SELECT c.customer_id FROM customers AS c CROSS JOIN latest AS l", - dialect: "duckdb", - one_row_relations: &[], - expected_messages: &["CROSS JOIN with `latest` creates an explicit cartesian product"], - }, - CrossJoinTestCase { - description: "one-row VALUES", - sql: "WITH settings AS (SELECT * FROM (VALUES (1, 'EUR')) AS v(version, currency)) \ - SELECT o.order_id FROM orders AS o CROSS JOIN settings AS s", - dialect: "duckdb", - one_row_relations: &[], - expected_messages: &[], - }, - CrossJoinTestCase { - description: "two-row VALUES", - sql: "WITH settings AS (SELECT * FROM (VALUES (1), (2)) AS v(version)) \ - SELECT o.order_id FROM orders AS o CROSS JOIN settings AS s", - dialect: "duckdb", - one_row_relations: &[], - expected_messages: &[ - "CROSS JOIN with `settings` creates an explicit cartesian product", - ], - }, - CrossJoinTestCase { - description: "QUALIFY ROW_NUMBER() = 1 without a partition", - sql: "WITH newest AS (SELECT order_id FROM orders QUALIFY ROW_NUMBER() OVER (ORDER BY order_id DESC) = 1) \ - SELECT c.customer_id FROM customers AS c CROSS JOIN newest AS n", - dialect: "snowflake", - one_row_relations: &[], - expected_messages: &[], - }, - CrossJoinTestCase { - description: "QUALIFY ROW_NUMBER() = 1 with a partition keeps one row per partition", - sql: "WITH newest AS (SELECT order_id FROM orders QUALIFY ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY order_id DESC) = 1) \ - SELECT c.customer_id FROM customers AS c CROSS JOIN newest AS n", - dialect: "snowflake", - one_row_relations: &[], - expected_messages: &["CROSS JOIN with `newest` creates an explicit cartesian product"], - }, - CrossJoinTestCase { - description: "a join inside the carrying CTE stops the proof", - sql: "WITH totals AS (SELECT COUNT(*) AS n FROM orders), \ - joined AS (SELECT t.n FROM totals AS t INNER JOIN customers AS c ON c.n = t.n) \ - SELECT o.order_id FROM orders AS o CROSS JOIN joined AS j", - dialect: "duckdb", - one_row_relations: &[], - expected_messages: &["CROSS JOIN with `joined` creates an explicit cartesian product"], - }, - CrossJoinTestCase { - description: "UNNEST in a carrying CTE stops the proof", - sql: "WITH totals AS (SELECT LIST(order_id) AS ids FROM orders), \ - expanded AS (SELECT UNNEST(ids) AS order_id FROM totals) \ - SELECT c.customer_id FROM customers AS c CROSS JOIN expanded AS e", - dialect: "duckdb", - one_row_relations: &[], - expected_messages: &[ - "CROSS JOIN with `expanded` creates an explicit cartesian product", - ], - }, - CrossJoinTestCase { - description: "a set operation stops the proof", - sql: "WITH both_totals AS (SELECT COUNT(*) AS n FROM orders UNION ALL SELECT COUNT(*) AS n FROM customers) \ - SELECT o.order_id FROM orders AS o CROSS JOIN both_totals AS b", - dialect: "duckdb", - one_row_relations: &[], - expected_messages: &[ - "CROSS JOIN with `both_totals` creates an explicit cartesian product", - ], - }, - CrossJoinTestCase { - description: "the finding names the CTE that contains the join", - sql: "WITH checks AS (SELECT o.order_id FROM orders AS o CROSS JOIN customers AS c) \ - SELECT order_id FROM checks", - dialect: "duckdb", - one_row_relations: &[], - expected_messages: &[ - "CROSS JOIN with `customers` in CTE `checks` creates an explicit cartesian product", - ], - }, - CrossJoinTestCase { - description: "a later join still multiplies the rows before it", - sql: "SELECT o.order_id FROM __sqb_lint_7__ AS p CROSS JOIN orders AS o CROSS JOIN customers AS c", - dialect: "duckdb", - one_row_relations: &[POLICY_SENTINEL], - expected_messages: &[ - "CROSS JOIN with `customers` creates an explicit cartesian product", - ], - }, - CrossJoinTestCase { - description: "an unnamed relation says only a file-wide exception covers it", - sql: "SELECT o.order_id FROM orders AS o CROSS JOIN UNNEST([1, 2]) AS u(n)", - dialect: "duckdb", - one_row_relations: &[], - expected_messages: &[UNNAMED_MESSAGE], - }, - CrossJoinTestCase { - description: "a named window is never treated as unpartitioned", - sql: "WITH newest AS (SELECT order_id FROM orders WINDOW w AS (PARTITION BY customer_id ORDER BY order_id DESC) QUALIFY ROW_NUMBER() OVER w = 1) \ - SELECT c.customer_id FROM customers AS c CROSS JOIN newest AS n", - dialect: "duckdb", - one_row_relations: &[], - expected_messages: &["CROSS JOIN with `newest` creates an explicit cartesian product"], - }, - CrossJoinTestCase { - description: "a join under a nested WITH names the top-level CTE", - sql: "WITH outer_checks AS (WITH inner_checks AS (SELECT o.order_id FROM orders AS o CROSS JOIN customers AS c) SELECT order_id FROM inner_checks) \ - SELECT order_id FROM outer_checks", - dialect: "duckdb", - one_row_relations: &[], - expected_messages: &[ - "CROSS JOIN with `customers` in CTE `outer_checks` creates an explicit cartesian product", - ], - }, - CrossJoinTestCase { - description: "a WITH inside a subquery of the final select names no CTE", - sql: "SELECT s.order_id FROM (WITH inner_checks AS (SELECT o.order_id FROM orders AS o CROSS JOIN customers AS c) SELECT order_id FROM inner_checks) AS s", - dialect: "duckdb", - one_row_relations: &[], - expected_messages: &[ - "CROSS JOIN with `customers` creates an explicit cartesian product", - ], - }, - CrossJoinTestCase { - description: "a WITH owned by a set operation names its CTE", - sql: "WITH checks AS (SELECT o.order_id FROM orders AS o CROSS JOIN customers AS c) \ - SELECT order_id FROM checks UNION ALL SELECT order_id FROM orders", - dialect: "duckdb", - one_row_relations: &[], - expected_messages: &[ - "CROSS JOIN with `customers` in CTE `checks` creates an explicit cartesian product", - ], - }, - ]; - - for test_case in test_cases { - let messages: Vec = cross_join_messages(&test_case)?; - - assert_eq!( - messages, test_case.expected_messages, - "{}", - test_case.description - ); - } - Ok(()) -} - -#[test] -fn given_unnamed_cross_join_when_linting_then_help_prints_a_file_wide_exception() --> Result<(), String> { - let test_cases = [CrossJoinTestCase { - description: "CROSS JOIN UNNEST", - sql: "SELECT o.order_id FROM orders AS o CROSS JOIN UNNEST([1, 2]) AS u(n)", - dialect: "duckdb", - one_row_relations: &[], - expected_messages: &[ - "[[rules.rule_exceptions]]\n rule = \"SQBRSQL038\"\n path = \"\"\n reason = \"\"", - ], - }]; - - for test_case in test_cases { - let remediations: Vec = cross_join_remediations(&test_case)?; - - assert_eq!(remediations.len(), 1, "{}", test_case.description); - assert!( - remediations[0].contains(test_case.expected_messages[0]), - "{}: {remediations:?}", - test_case.description - ); - } - Ok(()) -} - -#[test] -fn given_cross_join_messages_when_reading_their_subject_then_relation_and_cte_are_named() { - let test_cases = [ - CrossJoinSubjectTestCase { - description: "a top-level join", - message: "CROSS JOIN with `order_policy` creates an explicit cartesian product", - expected_subject: Some(("order_policy", None)), - }, - CrossJoinSubjectTestCase { - description: "a join inside a CTE", - message: "CROSS JOIN with `order_policy` in CTE `order_checks` creates an explicit cartesian product", - expected_subject: Some(("order_policy", Some("order_checks"))), - }, - CrossJoinSubjectTestCase { - description: "the generic message names nothing", - message: GENERIC_MESSAGE, - expected_subject: None, - }, - ]; - - for test_case in test_cases { - let subject = cross_join_subject(test_case.message); - - assert_eq!( - subject - .as_ref() - .map(|(relation, cte)| (relation.as_str(), cte.as_deref())), - test_case.expected_subject, - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_empty_fixture_lint.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_empty_fixture_lint.rs deleted file mode 100644 index b5500742b2..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_empty_fixture_lint.rs +++ /dev/null @@ -1,61 +0,0 @@ -use serde_json::{Value, json}; - -use crate::sql_lint::main::engine::lint_json; -use crate::sql_lint::tests::test_types; - -#[test] -fn given_empty_fixture_context_when_linting_projected_star_then_only_canonical_inputs_pass() --> Result<(), String> { - let test_cases = [ - test_types::EmptyFixtureLintTestCase { - description: "canonical ref fixture", - sql: "WITH __ref__orders AS (SELECT * FROM __empty_fixture()) SELECT 1", - allows_empty_fixture_star: true, - expected_diagnostic_count: 0, - }, - test_types::EmptyFixtureLintTestCase { - description: "canonical source fixture", - sql: "WITH __source__web__orders AS (SELECT * FROM __empty_fixture()) SELECT 1", - allows_empty_fixture_star: true, - expected_diagnostic_count: 0, - }, - test_types::EmptyFixtureLintTestCase { - description: "ordinary SQL has no intrinsic exemption", - sql: "SELECT * FROM __empty_fixture()", - allows_empty_fixture_star: false, - expected_diagnostic_count: 1, - }, - test_types::EmptyFixtureLintTestCase { - description: "expected fixture remains controlled", - sql: "WITH __expected__orders AS (SELECT * FROM __empty_fixture()) SELECT 1", - allows_empty_fixture_star: true, - expected_diagnostic_count: 1, - }, - test_types::EmptyFixtureLintTestCase { - description: "additional projection is not canonical", - sql: "WITH __ref__orders AS (SELECT *, 1 AS extra FROM __empty_fixture()) SELECT 1", - allows_empty_fixture_star: true, - expected_diagnostic_count: 1, - }, - ]; - for test_case in test_cases { - let response = lint_json( - &json!({ - "version": 1, - "sql": test_case.sql, - "dialect": "snowflake", - "enabled_rules": ["SQBRSQL021"], - "allows_empty_fixture_star": test_case.allows_empty_fixture_star - }) - .to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - assert_eq!( - payload["diagnostics"].as_array().map_or(0, Vec::len), - test_case.expected_diagnostic_count, - "{}", - test_case.description - ); - } - Ok(()) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_format_corpus.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_format_corpus.rs deleted file mode 100644 index c40d2ff006..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_format_corpus.rs +++ /dev/null @@ -1,61 +0,0 @@ -use serde_json::Value; - -use crate::sql_lint::tests::helpers::check_dialect_corpus; -use crate::sql_lint::tests::test_types; - -const FORMAT_CORPUS: &str = include_str!("fixtures/format_corpus.json"); - -#[test] -fn given_dialect_corpus_when_formatting_then_output_layout_and_refusals_are_stable() --> Result<(), String> { - let test_cases = [ - test_types::FormatCorpusTestCase { - description: "BigQuery identity corpus", - dialect: "bigquery", - expected_formatted: 168, - expected_refused: 0, - }, - test_types::FormatCorpusTestCase { - description: "Databricks identity corpus", - dialect: "databricks", - expected_formatted: 62, - expected_refused: 0, - }, - test_types::FormatCorpusTestCase { - description: "DuckDB identity corpus", - dialect: "duckdb", - expected_formatted: 178, - expected_refused: 11, - }, - test_types::FormatCorpusTestCase { - description: "PostgreSQL identity corpus", - dialect: "postgres", - expected_formatted: 132, - expected_refused: 5, - }, - test_types::FormatCorpusTestCase { - description: "Snowflake identity corpus", - dialect: "snowflake", - expected_formatted: 429, - expected_refused: 6, - }, - test_types::FormatCorpusTestCase { - description: "T-SQL identity corpus", - dialect: "tsql", - expected_formatted: 75, - expected_refused: 0, - }, - ]; - let corpus: Vec = - serde_json::from_str(FORMAT_CORPUS).map_err(|error| error.to_string())?; - for test_case in test_cases { - let (formatted, refused) = check_dialect_corpus(&corpus, test_case.dialect)?; - assert_eq!( - (formatted, refused), - (test_case.expected_formatted, test_case.expected_refused), - "{}", - test_case.description - ); - } - Ok(()) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_function_call_spacing.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_function_call_spacing.rs deleted file mode 100644 index 4acd30cdbf..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_function_call_spacing.rs +++ /dev/null @@ -1,117 +0,0 @@ -use crate::sql_lint::tests::helpers::function_call_spacing; -use crate::sql_lint::tests::test_types; - -#[test] -fn given_builtin_call_names_when_formatting_then_no_space_enters_the_call() -> Result<(), String> { - let test_cases = [ - test_types::FunctionCallSpacingTestCase { - description: "duckdb compact calls", - dialect: "duckdb", - line_width: 100, - argument: "a", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "duckdb breaking calls", - dialect: "duckdb", - line_width: 40, - argument: "order_amount_with_a_long_descriptive_name", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "snowflake compact calls", - dialect: "snowflake", - line_width: 100, - argument: "a", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "snowflake breaking calls", - dialect: "snowflake", - line_width: 40, - argument: "order_amount_with_a_long_descriptive_name", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "bigquery compact calls", - dialect: "bigquery", - line_width: 100, - argument: "a", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "bigquery breaking calls", - dialect: "bigquery", - line_width: 40, - argument: "order_amount_with_a_long_descriptive_name", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "postgres compact calls", - dialect: "postgres", - line_width: 100, - argument: "a", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "postgres breaking calls", - dialect: "postgres", - line_width: 40, - argument: "order_amount_with_a_long_descriptive_name", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "tsql compact calls", - dialect: "tsql", - line_width: 100, - argument: "a", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "tsql breaking calls", - dialect: "tsql", - line_width: 40, - argument: "order_amount_with_a_long_descriptive_name", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "databricks compact calls", - dialect: "databricks", - line_width: 100, - argument: "a", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "databricks breaking calls", - dialect: "databricks", - line_width: 40, - argument: "order_amount_with_a_long_descriptive_name", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "generic compact calls", - dialect: "generic", - line_width: 100, - argument: "a", - expected_refused: &[], - }, - test_types::FunctionCallSpacingTestCase { - description: "generic breaking calls", - dialect: "generic", - line_width: 40, - argument: "order_amount_with_a_long_descriptive_name", - expected_refused: &[], - }, - ]; - for test_case in &test_cases { - let (refused, misplaced) = - function_call_spacing(test_case.dialect, test_case.line_width, test_case.argument)?; - assert_eq!( - refused, test_case.expected_refused, - "{}", - test_case.description - ); - assert_eq!(misplaced, Vec::::new(), "{}", test_case.description); - } - Ok(()) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_idempotency_reprint.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_idempotency_reprint.rs deleted file mode 100644 index edf70b5826..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_idempotency_reprint.rs +++ /dev/null @@ -1,48 +0,0 @@ -use serde_json::{Value, json}; - -use crate::sql_lint::_helpers::formatter::RELAYOUTS; -use crate::sql_lint::main::formatter::format_json; -use crate::sql_lint::tests::test_types; - -#[test] -fn given_formatted_and_unformatted_bodies_when_formatting_then_only_changed_bodies_are_reprinted() --> Result<(), String> { - let test_cases = [ - test_types::ReprintTestCase { - description: "an already formatted body is not re-printed", - sql: "SELECT\n order_id,\n amount\nFROM orders\nWHERE amount > 0", - expected_changed: false, - expected_reprints: 0, - }, - test_types::ReprintTestCase { - description: "a body the formatter changes is re-printed once", - sql: "select order_id, amount from orders where amount > 0", - expected_changed: true, - expected_reprints: 1, - }, - ]; - for test_case in &test_cases { - RELAYOUTS.with(|relayouts| relayouts.set(0)); - let response = format_json( - &json!({"version": 1, "sql": test_case.sql, "dialect": "duckdb"}).to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - assert_eq!( - payload["formatted"], true, - "{}: {payload}", - test_case.description - ); - assert_eq!( - payload["changed"], test_case.expected_changed, - "{}: {payload}", - test_case.description - ); - assert_eq!( - RELAYOUTS.with(std::cell::Cell::get), - test_case.expected_reprints, - "{}", - test_case.description - ); - } - Ok(()) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_layout_rules.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_layout_rules.rs deleted file mode 100644 index b748af51a2..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_layout_rules.rs +++ /dev/null @@ -1,287 +0,0 @@ -use serde_json::Value; - -use crate::sql_lint::tests::helpers::format_at_width; -use crate::sql_lint::tests::test_types; -use sqlbuild_analysis::sql_tokens::main::query_fingerprint::query_fingerprint; - -#[test] -fn given_clause_layout_rules_when_formatting_then_each_rule_applies_idempotently() --> Result<(), String> { - let test_cases = [ - test_types::LayoutRuleTestCase { - description: "a long Snowflake FROM VALUES puts each unsplit row on its own line", - dialect: "snowflake", - sql: "SELECT * FROM VALUES (1, '2024-01-15', 'PAID', 'north warehouse', 'priority shipping'), (2, '2024-01-16', 'OPEN', 'south warehouse', 'standard shipping')", - line_width: 60, - expected_sql: "SELECT *\nFROM VALUES\n (1, '2024-01-15', 'PAID', 'north warehouse', 'priority shipping'),\n (2, '2024-01-16', 'OPEN', 'south warehouse', 'standard shipping')", - }, - test_types::LayoutRuleTestCase { - description: "a short VALUES list stays on one line", - dialect: "snowflake", - sql: "SELECT * FROM VALUES ('a', 'A'), ('b', 'B')", - line_width: 100, - expected_sql: "SELECT *\nFROM VALUES ('a', 'A'), ('b', 'B')", - }, - test_types::LayoutRuleTestCase { - description: "a long parenthesized PostgreSQL VALUES moves under its parenthesis", - dialect: "postgres", - sql: "SELECT order_id, status FROM (VALUES (1, 'paid and shipped to customer'), (2, 'cancelled by customer')) AS orders(order_id, status)", - line_width: 60, - expected_sql: "SELECT\n order_id,\n status\nFROM (\n VALUES\n (1, 'paid and shipped to customer'),\n (2, 'cancelled by customer')\n) AS orders(order_id, status)", - }, - test_types::LayoutRuleTestCase { - description: "a long T-SQL VALUES relation keeps rows whole", - dialect: "tsql", - sql: "SELECT order_id FROM (VALUES (1, 'paid and shipped to customer'), (2, 'cancelled by customer')) AS orders(order_id, status)", - line_width: 60, - expected_sql: "SELECT order_id\nFROM (\n VALUES\n (1, 'paid and shipped to customer'),\n (2, 'cancelled by customer')\n) AS orders(order_id, status)", - }, - test_types::LayoutRuleTestCase { - description: "DuckDB INSERT VALUES rows stay whole when a row is over the width", - dialect: "duckdb", - sql: "INSERT INTO orders VALUES (1, 'paid and shipped to the customer by priority courier'), (2, 'open')", - line_width: 40, - expected_sql: "INSERT INTO orders\nVALUES\n (1, 'paid and shipped to the customer by priority courier'),\n (2, 'open')", - }, - test_types::LayoutRuleTestCase { - description: "Snowflake CAST breaks around IFF, not inside NUMBER(38, 2)", - dialect: "snowflake", - sql: "SELECT CAST(IFF(amount > 100000000, amount * 1000000000000, amount / 100000000000000) AS NUMBER(38, 2)) AS total FROM orders", - line_width: 80, - expected_sql: "SELECT\n CAST(\n IFF(\n amount > 100000000,\n amount * 1000000000000,\n amount / 100000000000000\n ) AS NUMBER(38, 2)\n ) AS total\nFROM orders", - }, - test_types::LayoutRuleTestCase { - description: "a DuckDB :: cast keeps DECIMAL(18, 4) whole", - dialect: "duckdb", - sql: "SELECT COALESCE(first_order_amount, second_order_amount)::DECIMAL(18, 4) AS amount FROM orders", - line_width: 50, - expected_sql: "SELECT\n COALESCE(\n first_order_amount,\n second_order_amount\n )::DECIMAL(18, 4) AS amount\nFROM orders", - }, - test_types::LayoutRuleTestCase { - description: "a BigQuery NUMERIC(38, 9) cast keeps its parameters", - dialect: "bigquery", - sql: "SELECT CAST(COALESCE(first_order_amount, second_order_amount) AS NUMERIC(38, 9)) AS amount FROM orders", - line_width: 50, - expected_sql: "SELECT\n CAST(\n COALESCE(\n first_order_amount,\n second_order_amount\n ) AS NUMERIC(38, 9)\n ) AS amount\nFROM orders", - }, - test_types::LayoutRuleTestCase { - description: "a T-SQL VARCHAR(MAX) cast keeps its parameter", - dialect: "tsql", - sql: "SELECT CAST(COALESCE(first_order_note, second_order_note) AS VARCHAR(MAX)) AS note FROM orders", - line_width: 50, - expected_sql: "SELECT\n CAST(\n COALESCE(\n first_order_note,\n second_order_note\n ) AS VARCHAR(MAX)\n ) AS note\nFROM orders", - }, - test_types::LayoutRuleTestCase { - description: "a Databricks DECIMAL(10, 2) cast keeps its parameters", - dialect: "databricks", - sql: "SELECT CAST(COALESCE(first_order_amount, second_order_amount) AS DECIMAL(10, 2)) AS amount FROM orders", - line_width: 50, - expected_sql: "SELECT\n CAST(\n COALESCE(\n first_order_amount,\n second_order_amount\n ) AS DECIMAL(10, 2)\n ) AS amount\nFROM orders", - }, - test_types::LayoutRuleTestCase { - description: "a long Snowflake DECODE puts each pair and the default on its own line", - dialect: "snowflake", - sql: "SELECT DECODE(status, 1, 'new', 2, 'open', 3, 'paid', 4, 'shipped', 5, 'cancelled', 'unknown') AS status_name FROM orders", - line_width: 60, - expected_sql: "SELECT\n DECODE(\n status,\n 1, 'new',\n 2, 'open',\n 3, 'paid',\n 4, 'shipped',\n 5, 'cancelled',\n 'unknown'\n ) AS status_name\nFROM orders", - }, - test_types::LayoutRuleTestCase { - description: "a DECODE without a default ends with a pair", - dialect: "snowflake", - sql: "SELECT DECODE(status, 1, 'new', 2, 'open', 3, 'paid', 4, 'shipped', 5, 'cancelled') AS status_name FROM orders", - line_width: 60, - expected_sql: "SELECT\n DECODE(\n status,\n 1, 'new',\n 2, 'open',\n 3, 'paid',\n 4, 'shipped',\n 5, 'cancelled'\n ) AS status_name\nFROM orders", - }, - test_types::LayoutRuleTestCase { - description: "a DECODE the layout guide breaks is regrouped into pairs", - dialect: "snowflake", - sql: "SELECT DECODE(status, 1, 'new', 2, 'open', 3, 'paid', 4, 'shipped', 5, 'cancelled', 6, 'returned', 'unknown') AS status_name FROM orders", - line_width: 100, - expected_sql: "SELECT\n DECODE(\n status,\n 1, 'new',\n 2, 'open',\n 3, 'paid',\n 4, 'shipped',\n 5, 'cancelled',\n 6, 'returned',\n 'unknown'\n ) AS status_name\nFROM orders", - }, - test_types::LayoutRuleTestCase { - description: "a short DECODE stays on one line", - dialect: "snowflake", - sql: "SELECT DECODE(status, 1, 'new', 'other') AS status_name FROM orders", - line_width: 100, - expected_sql: "SELECT DECODE(status, 1, 'new', 'other') AS status_name\nFROM orders", - }, - test_types::LayoutRuleTestCase { - description: "an operator after a multi-line CASE starts the next line", - dialect: "duckdb", - sql: "SELECT CASE WHEN status = 'paid' THEN 'customer paid in full' WHEN status = 'open' THEN 'customer has not paid yet' ELSE 'other' END || CASE WHEN rush THEN '!' ELSE '' END AS label FROM orders", - line_width: 100, - expected_sql: "SELECT\n CASE\n WHEN status = 'paid'\n THEN 'customer paid in full'\n WHEN status = 'open'\n THEN 'customer has not paid yet'\n ELSE 'other'\n END\n || CASE WHEN rush THEN '!' ELSE '' END AS label\nFROM orders", - }, - test_types::LayoutRuleTestCase { - description: "AND after a multi-line subquery starts the next line", - dialect: "postgres", - sql: "SELECT order_id FROM orders WHERE customer_id IN (SELECT customer_id FROM customers WHERE region = 'north') AND EXISTS (SELECT 1 FROM shipments)", - line_width: 100, - expected_sql: "SELECT order_id\nFROM orders\nWHERE\n customer_id IN (\n SELECT customer_id\n FROM customers\n WHERE region = 'north'\n )\n AND EXISTS(\n SELECT 1\n FROM shipments\n )", - }, - test_types::LayoutRuleTestCase { - description: "BETWEEN bounds keep their AND", - dialect: "snowflake", - sql: "SELECT order_id FROM orders WHERE amount BETWEEN 1 AND 9", - line_width: 100, - expected_sql: "SELECT order_id\nFROM orders\nWHERE amount BETWEEN 1 AND 9", - }, - test_types::LayoutRuleTestCase { - description: "CTEs and the final statement are separated by one blank line", - dialect: "duckdb", - sql: "with orders as (select * from raw_orders),\n\n\n-- paid only\npaid as (select * from orders where amount > 0)\n\n\nselect * from paid", - line_width: 100, - expected_sql: "WITH orders AS (\n SELECT *\n FROM raw_orders\n),\n\n-- paid only\npaid AS (\n SELECT *\n FROM orders\n WHERE amount > 0\n)\n\nSELECT *\nFROM paid", - }, - test_types::LayoutRuleTestCase { - description: "a trailing comment after a CTE comma keeps its line", - dialect: "snowflake", - sql: "WITH orders AS (SELECT * FROM raw_orders), -- raw\npaid AS (SELECT * FROM orders) SELECT * FROM paid", - line_width: 100, - expected_sql: "WITH orders AS (\n SELECT *\n FROM raw_orders\n), -- raw\n\npaid AS (\n SELECT *\n FROM orders\n)\n\nSELECT *\nFROM paid", - }, - test_types::LayoutRuleTestCase { - description: "a nested WITH is separated at its own indentation", - dialect: "postgres", - sql: "SELECT * FROM (WITH a AS MATERIALIZED (SELECT 1 AS x), b AS (SELECT x FROM a) SELECT x FROM b) AS nested", - line_width: 100, - expected_sql: "SELECT *\nFROM (\n WITH a AS MATERIALIZED (\n SELECT 1 AS x\n ),\n\n b AS (\n SELECT x\n FROM a\n )\n\n SELECT x\n FROM b\n) AS nested", - }, - test_types::LayoutRuleTestCase { - description: "BigQuery recursive CTEs with column lists are separated", - dialect: "bigquery", - sql: "WITH RECURSIVE numbers (n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM numbers WHERE n < 3), doubled AS (SELECT n * 2 AS n FROM numbers) SELECT n FROM doubled", - line_width: 100, - expected_sql: "WITH RECURSIVE numbers (n) AS (\n SELECT 1\n UNION ALL\n SELECT n + 1\n FROM numbers\n WHERE n < 3\n),\n\ndoubled AS (\n SELECT n * 2 AS n\n FROM numbers\n)\n\nSELECT n\nFROM doubled", - }, - test_types::LayoutRuleTestCase { - description: "a T-SQL WITH is separated before INSERT", - dialect: "tsql", - sql: "WITH paid AS (SELECT order_id FROM orders) INSERT INTO paid_orders SELECT order_id FROM paid", - line_width: 100, - expected_sql: "WITH paid AS (\n SELECT order_id\n FROM orders\n)\n\nINSERT INTO paid_orders\nSELECT order_id\nFROM paid", - }, - test_types::LayoutRuleTestCase { - description: "a CTE macro placeholder is separated like a CTE", - dialect: "duckdb", - sql: "WITH __sqb_lint_0__, paid AS (SELECT * FROM orders) SELECT * FROM paid", - line_width: 100, - expected_sql: "WITH\n__sqb_lint_0__,\n\npaid AS (\n SELECT *\n FROM orders\n)\n\nSELECT *\nFROM paid", - }, - test_types::LayoutRuleTestCase { - description: "comma-less CTE macro placeholders in a row are separated like CTEs", - dialect: "duckdb", - sql: "WITH a AS (SELECT 1 AS x), __sqb_lint_0__ __sqb_lint_1__ paid AS (SELECT * FROM a) SELECT * FROM paid", - line_width: 100, - expected_sql: "WITH a AS (\n SELECT 1 AS x\n),\n\n__sqb_lint_0__\n\n__sqb_lint_1__\npaid AS (\n SELECT *\n FROM a\n)\nSELECT *\nFROM paid", - }, - test_types::LayoutRuleTestCase { - description: "an alias and comma that push a call over the width break the call", - dialect: "snowflake", - sql: "SELECT ARRAY_CONTAINS('web'::VARIANT, COALESCE(o.order_channels, ARRAY_CONSTRUCT())) AS has_web_channel, o.order_id FROM orders o", - line_width: 80, - expected_sql: "SELECT\n ARRAY_CONTAINS(\n 'web'::VARIANT,\n COALESCE(o.order_channels, ARRAY_CONSTRUCT())\n ) AS has_web_channel,\n o.order_id\nFROM orders o", - }, - test_types::LayoutRuleTestCase { - description: "an AND chain inside a call argument joins its line, then wraps", - dialect: "snowflake", - sql: "SELECT CAST(MAX(IFF(line_number = 1 AND shipped_after_days = 960, order_amount, NULL)) AS NUMBER(38, 10)) AS first_line_amount_shipped_late FROM orders", - line_width: 100, - expected_sql: "SELECT\n CAST(\n MAX(IFF(line_number = 1 AND shipped_after_days = 960, order_amount, NULL)) AS NUMBER(38, 10)\n ) AS first_line_amount_shipped_late\nFROM orders", - }, - test_types::LayoutRuleTestCase { - description: "a one-line CASE over the width breaks at its branches", - dialect: "duckdb", - sql: "SELECT CASE WHEN o.shipped_at IS NULL THEN NULL WHEN o.refund_amount IS NULL THEN 0 ELSE 1 END AS has_refund FROM orders o", - line_width: 80, - expected_sql: "SELECT\n CASE\n WHEN o.shipped_at IS NULL\n THEN NULL\n WHEN o.refund_amount IS NULL\n THEN 0\n ELSE 1\n END AS has_refund\nFROM orders o", - }, - test_types::LayoutRuleTestCase { - description: "a WITHIN GROUP line breaks the aggregate call", - dialect: "snowflake", - sql: "SELECT customer_id, LISTAGG(DISTINCT LOWER(product_category_name), ', ') WITHIN GROUP (ORDER BY product_category_name) AS categories FROM orders GROUP BY customer_id", - line_width: 80, - expected_sql: "SELECT\n customer_id,\n LISTAGG(\n DISTINCT LOWER(product_category_name),\n ', '\n ) WITHIN GROUP (ORDER BY product_category_name) AS categories\nFROM orders\nGROUP BY\n customer_id", - }, - test_types::LayoutRuleTestCase { - description: "a window that continues on later lines breaks at its clauses", - dialect: "snowflake", - sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY CASE o.status WHEN 'paid' THEN 0 WHEN 'open' THEN 1 ELSE 2 END, o.ordered_at DESC) AS rn FROM orders o", - line_width: 60, - expected_sql: "SELECT\n ROW_NUMBER() OVER (\n PARTITION BY o.customer_id\n ORDER BY CASE o.status\n WHEN 'paid'\n THEN 0\n WHEN 'open'\n THEN 1\n ELSE 2\n END, o.ordered_at DESC\n ) AS rn\nFROM orders o", - }, - test_types::LayoutRuleTestCase { - description: "SELECT * and a single WHERE condition share their clause lines", - dialect: "duckdb", - sql: "select * from orders where amount > 0", - line_width: 100, - expected_sql: "SELECT *\nFROM orders\nWHERE amount > 0", - }, - test_types::LayoutRuleTestCase { - description: "SELECT DISTINCT keeps its single item, several items break", - dialect: "snowflake", - sql: "SELECT DISTINCT order_id FROM orders WHERE amount > 0 AND status = 'paid' UNION ALL SELECT order_id, customer_id FROM returns", - line_width: 100, - expected_sql: "SELECT DISTINCT order_id\nFROM orders\nWHERE\n amount > 0\n AND status = 'paid'\nUNION ALL\nSELECT\n order_id,\n customer_id\nFROM returns", - }, - test_types::LayoutRuleTestCase { - description: "T-SQL TOP, HAVING and Snowflake-style single items share lines", - dialect: "tsql", - sql: "SELECT TOP 10 COUNT(*) AS n FROM orders GROUP BY customer_id HAVING COUNT(*) > 1", - line_width: 100, - expected_sql: "SELECT TOP 10 COUNT(*) AS n\nFROM orders\nGROUP BY\n customer_id\nHAVING COUNT(*) > 1", - }, - test_types::LayoutRuleTestCase { - description: "QUALIFY with one condition shares its line", - dialect: "databricks", - sql: "SELECT order_id, ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at) AS rn FROM orders QUALIFY rn = 1", - line_width: 100, - expected_sql: "SELECT\n order_id,\n ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at) AS rn\nFROM orders\nQUALIFY rn = 1", - }, - test_types::LayoutRuleTestCase { - description: "a single item that does not fit stays under SELECT", - dialect: "bigquery", - sql: "SELECT COALESCE(first_order_amount, second_order_amount, third_order_amount) AS amount FROM orders WHERE first_order_amount + second_order_amount + third_order_amount > 100", - line_width: 60, - expected_sql: "SELECT\n COALESCE(\n first_order_amount,\n second_order_amount,\n third_order_amount\n ) AS amount\nFROM orders\nWHERE\n first_order_amount\n + second_order_amount\n + third_order_amount > 100", - }, - test_types::LayoutRuleTestCase { - description: "a short CASE item shares the SELECT line", - dialect: "duckdb", - sql: "SELECT CASE WHEN amount > 0 THEN 'paid' ELSE 'open' END AS status FROM orders", - line_width: 100, - expected_sql: "SELECT CASE WHEN amount > 0 THEN 'paid' ELSE 'open' END AS status\nFROM orders", - }, - test_types::LayoutRuleTestCase { - description: "a multi-line single item stays under SELECT", - dialect: "duckdb", - sql: "SELECT CASE WHEN amount > 1000 THEN 'large paid order' WHEN amount > 0 THEN 'paid order' ELSE 'open order' END AS status FROM orders", - line_width: 100, - expected_sql: "SELECT\n CASE\n WHEN amount > 1000\n THEN 'large paid order'\n WHEN amount > 0\n THEN 'paid order'\n ELSE 'open order'\n END AS status\nFROM orders", - }, - ]; - for test_case in &test_cases { - let response: Value = - format_at_width(test_case.sql, test_case.dialect, test_case.line_width)?; - assert_eq!(response["formatted"], true, "{}", test_case.description); - assert_eq!( - response["sql"], test_case.expected_sql, - "{}", - test_case.description - ); - assert_eq!( - query_fingerprint(test_case.expected_sql, test_case.dialect)?, - query_fingerprint(test_case.sql, test_case.dialect)?, - "{}: layout changed the fingerprint", - test_case.description - ); - let again: Value = format_at_width( - test_case.expected_sql, - test_case.dialect, - test_case.line_width, - )?; - assert_eq!(again["changed"], false, "{}", test_case.description); - } - Ok(()) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_line_wrap.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_line_wrap.rs deleted file mode 100644 index 3ae97ec83d..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_line_wrap.rs +++ /dev/null @@ -1,107 +0,0 @@ -use std::collections::HashMap; - -use serde_json::{Value, json}; - -use crate::sql_lint::main::formatter::format_json; -use crate::sql_lint::tests::test_types; - -#[test] -fn given_lines_over_the_width_when_formatting_then_each_wrap_rule_applies_idempotently() --> Result<(), String> { - let test_cases = [ - test_types::LineWrapTestCase { - description: "function arguments go one per line", - sql: "select coalesce(first_order_amount, second_order_amount, third_order_amount, fallback_amount) as amount from orders", - line_width: 60, - token_widths: &[], - expected_sql: "SELECT\n COALESCE(\n first_order_amount,\n second_order_amount,\n third_order_amount,\n fallback_amount\n ) AS amount\nFROM orders", - }, - test_types::LineWrapTestCase { - description: "IN lists go one element per line", - sql: "select order_id from orders where region in ('north', 'south', 'east', 'west', 'central', 'remote')", - line_width: 50, - token_widths: &[], - expected_sql: "SELECT order_id\nFROM orders\nWHERE\n region IN (\n 'north',\n 'south',\n 'east',\n 'west',\n 'central',\n 'remote'\n )", - }, - test_types::LineWrapTestCase { - description: "window PARTITION BY and ORDER BY go on separate lines", - sql: "select row_number() over (partition by customer_id, region_code order by ordered_at desc, order_id desc) as rank_in_customer from orders", - line_width: 60, - token_widths: &[], - expected_sql: "SELECT\n ROW_NUMBER() OVER (\n PARTITION BY customer_id, region_code\n ORDER BY ordered_at DESC, order_id DESC\n ) AS rank_in_customer\nFROM orders", - }, - test_types::LineWrapTestCase { - description: "join conditions break before AND but not inside BETWEEN", - sql: "select o.order_id from orders o join customers c on o.customer_id = c.customer_id and o.amount between 1 and 9 and o.store = c.store", - line_width: 60, - token_widths: &[], - expected_sql: "SELECT o.order_id\nFROM orders o\nJOIN customers c\n ON o.customer_id = c.customer_id\n AND o.amount BETWEEN 1 AND 9\n AND o.store = c.store", - }, - test_types::LineWrapTestCase { - description: "arithmetic chains break before operators", - sql: "select first_amount + second_amount + third_amount + fourth_amount as total from orders", - line_width: 50, - token_widths: &[], - expected_sql: "SELECT\n first_amount\n + second_amount\n + third_amount\n + fourth_amount AS total\nFROM orders", - }, - test_types::LineWrapTestCase { - description: "a top-level AND chain breaks before a nested group", - sql: "select order_id from orders where cast(ordered_at as DATE) >= cast(@start as DATE) and cast(ordered_at as DATE) < cast(@end as DATE)", - line_width: 60, - token_widths: &[], - expected_sql: "SELECT order_id\nFROM orders\nWHERE\n CAST(ordered_at AS DATE) >= CAST(@start AS DATE)\n AND CAST(ordered_at AS DATE) < CAST(@end AS DATE)", - }, - test_types::LineWrapTestCase { - description: "a long string literal is never broken", - sql: "select 'a long literal that is wider than the configured line width' as note from orders", - line_width: 40, - token_widths: &[], - expected_sql: "SELECT\n 'a long literal that is wider than the configured line width' AS note\nFROM orders", - }, - test_types::LineWrapTestCase { - description: "a line with a comment is never broken", - sql: "select coalesce(first_amount, second_amount, third_amount) as amount -- keep\nfrom orders", - line_width: 40, - token_widths: &[], - expected_sql: "SELECT\n COALESCE(first_amount, second_amount, third_amount) AS amount -- keep\nFROM orders", - }, - test_types::LineWrapTestCase { - description: "a sentinel counts with the width of the text it stands for", - sql: "select coalesce(__sqb_lint_0__, b) as amount from orders", - line_width: 40, - token_widths: &[("__SQB_LINT_0__", 30)], - expected_sql: "SELECT\n COALESCE(\n __sqb_lint_0__,\n b\n ) AS amount\nFROM orders", - }, - test_types::LineWrapTestCase { - description: "lines within the width are unchanged", - sql: "select coalesce(first_amount, second_amount) as amount from orders", - line_width: 100, - token_widths: &[], - expected_sql: "SELECT COALESCE(first_amount, second_amount) AS amount\nFROM orders", - }, - ]; - for test_case in &test_cases { - let token_widths: HashMap<&str, usize> = test_case.token_widths.iter().copied().collect(); - let request = |sql: &str| { - json!({ - "version": 1, - "sql": sql, - "dialect": "duckdb", - "line_width": test_case.line_width, - "token_widths": token_widths, - }) - .to_string() - }; - let response: Value = serde_json::from_str(&format_json(&request(test_case.sql))?) - .map_err(|error| error.to_string())?; - assert_eq!( - response["sql"], test_case.expected_sql, - "{}", - test_case.description - ); - let again: Value = serde_json::from_str(&format_json(&request(test_case.expected_sql))?) - .map_err(|error| error.to_string())?; - assert_eq!(again["changed"], false, "{}", test_case.description); - } - Ok(()) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_lint_exemptions.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_lint_exemptions.rs deleted file mode 100644 index 46f0ca78e0..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_lint_exemptions.rs +++ /dev/null @@ -1,130 +0,0 @@ -use serde_json::{Value, json}; - -use crate::sql_lint::main::engine::lint_json; -use crate::sql_lint::tests::test_types::{ - DynamicOutputStarLintTestCase, SqlMacroOutputLintTestCase, -}; - -#[test] -fn given_sql_macro_output_when_linting_qualification_then_each_column_counts_as_written() --> Result<(), String> { - let test_cases = [ - SqlMacroOutputLintTestCase { - description: "a bare column from an expression macro beside a qualified column fails", - sql: "SELECT o.order_id, status AS s FROM orders AS o", - output: "status", - relations: &[], - expected_count: 1, - }, - SqlMacroOutputLintTestCase { - description: "reads through a passed relation pass beside a bare caller column", - sql: "SELECT order_id, o.amount - o.discount AS net FROM orders AS o", - output: "o.amount - o.discount", - relations: &["o"], - expected_count: 0, - }, - SqlMacroOutputLintTestCase { - description: "reads through a passed relation pass beside a hand-qualified column", - sql: "SELECT o.order_id, o.amount - o.discount AS net FROM orders AS o", - output: "o.amount - o.discount", - relations: &["o"], - expected_count: 0, - }, - SqlMacroOutputLintTestCase { - description: "the caller's own mix beside reads through a passed relation fails", - sql: "SELECT o.order_id, status, o.amount AS a FROM orders AS o", - output: "o.amount", - relations: &["o"], - expected_count: 1, - }, - SqlMacroOutputLintTestCase { - description: "an aliased read counts as qualified beside a bare caller column", - sql: "SELECT order_id, s.amount AS a FROM orders AS s", - output: "s.amount", - relations: &["orders"], - expected_count: 1, - }, - SqlMacroOutputLintTestCase { - description: "a query scope the macro generated whole is judged in the macro file", - sql: "WITH c AS (SELECT a, w.b FROM w) SELECT * FROM c", - output: "SELECT a, w.b FROM w", - relations: &[], - expected_count: 0, - }, - ]; - for test_case in test_cases { - let start: usize = test_case - .sql - .find(test_case.output) - .ok_or("the output should be in the SQL")?; - let response = lint_json( - &json!({ - "version": 1, - "sql": test_case.sql, - "dialect": "duckdb", - "enabled_rules": ["SQBRSQL028"], - "sql_macro_outputs": [{ - "start": start, - "end": start + test_case.output.len(), - "relations": test_case.relations, - }], - }) - .to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - let diagnostics = payload["diagnostics"] - .as_array() - .ok_or("diagnostics should be an array")?; - - assert_eq!( - diagnostics.len(), - test_case.expected_count, - "{}", - test_case.description - ); - } - Ok(()) -} - -#[test] -fn given_compiler_proven_dynamic_output_when_linting_then_only_terminal_star_is_exempt() --> Result<(), String> { - let test_cases = [DynamicOutputStarLintTestCase { - description: "unrelated CTE star remains controlled", - sql: "WITH pivot_input AS (SELECT customer_id, category, amount FROM orders), \ -final AS (SELECT * FROM pivot_input PIVOT(MAX(amount) FOR category IN (ANY))), \ -unrelated AS (SELECT * FROM pivot_input) SELECT * FROM final", - expected_count: 1, - expected_prefix: "unrelated AS (SELECT ", - }]; - for test_case in test_cases { - let response = lint_json( - &json!({ - "version": 1, - "sql": test_case.sql, - "dialect": "snowflake", - "enabled_rules": ["SQBRSQL021"], - "allows_dynamic_output_star": true, - }) - .to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - let diagnostics = payload["diagnostics"] - .as_array() - .ok_or("diagnostics should be an array")?; - - assert_eq!( - diagnostics.len(), - test_case.expected_count, - "{}", - test_case.description - ); - let start = diagnostics[0]["start"].as_u64().unwrap_or_default() as usize; - assert!( - test_case.sql[..start].ends_with(test_case.expected_prefix), - "{}", - test_case.description - ); - } - Ok(()) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_set_operation_formatter.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_set_operation_formatter.rs deleted file mode 100644 index 9f8e7fbe85..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_set_operation_formatter.rs +++ /dev/null @@ -1,59 +0,0 @@ -use serde_json::{Value, json}; - -use crate::sql_lint::main::formatter::format_json; -use crate::sql_lint::tests::test_types; - -const DIALECTS: [&str; 6] = [ - "bigquery", - "databricks", - "duckdb", - "postgres", - "snowflake", - "tsql", -]; - -#[test] -fn given_set_operation_after_scalar_subquery_operand_when_formatting_then_keyword_stays_outer() --> Result<(), String> { - let test_cases = [ - test_types::FormatTestCase { - description: "UNION ALL after WHERE scalar subquery", - sql: "SELECT o.id, o.total FROM orders AS o WHERE o.total < (SELECT MAX(l.amount) FROM limits AS l) UNION ALL SELECT o.id, o.total FROM orders AS o", - expected_sql: "SELECT\n o.id,\n o.total\nFROM orders AS o\nWHERE\n o.total < (\n SELECT MAX(l.amount)\n FROM limits AS l\n )\nUNION ALL\nSELECT\n o.id,\n o.total\nFROM orders AS o", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "EXCEPT after WHERE scalar subquery", - sql: "SELECT o.id FROM orders AS o WHERE o.total > (SELECT MAX(l.amount) FROM limits AS l) EXCEPT SELECT o.id FROM orders AS o", - expected_sql: "SELECT o.id\nFROM orders AS o\nWHERE\n o.total > (\n SELECT MAX(l.amount)\n FROM limits AS l\n )\nEXCEPT\nSELECT o.id\nFROM orders AS o", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "INTERSECT after WHERE scalar subquery", - sql: "SELECT o.id FROM orders AS o WHERE o.total > (SELECT MAX(l.amount) FROM limits AS l) INTERSECT SELECT o.id FROM orders AS o", - expected_sql: "SELECT o.id\nFROM orders AS o\nWHERE\n o.total > (\n SELECT MAX(l.amount)\n FROM limits AS l\n )\nINTERSECT\nSELECT o.id\nFROM orders AS o", - expected_changed: true, - }, - ]; - - for dialect in DIALECTS { - for test_case in &test_cases { - let response = format_json( - &json!({"version": 1, "sql": test_case.sql, "dialect": dialect}).to_string(), - )?; - let payload: Value = - serde_json::from_str(&response).map_err(|error| error.to_string())?; - assert_eq!( - payload["sql"], test_case.expected_sql, - "{dialect}: {}", - test_case.description - ); - assert_eq!( - payload["changed"], test_case.expected_changed, - "{dialect}: {}", - test_case.description - ); - } - } - Ok(()) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_sql_lint.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_sql_lint.rs deleted file mode 100644 index a71418df82..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_sql_lint.rs +++ /dev/null @@ -1,1969 +0,0 @@ -use serde_json::{Value, json}; - -use crate::sql_lint::main::engine::lint_json; -use crate::sql_lint::main::formatter::format_json; -use crate::sql_lint::tests::helpers::format_batch_json; -use crate::sql_lint::tests::{helpers, test_types}; - -#[test] -fn given_production_function_depth_when_linting_then_bounded_parser_accepts_it() { - let test_cases = [test_types::FunctionDepthTestCase { - description: "production depth remains supported", - depth: 65, - expected_diagnostic_count: 0, - }]; - for test_case in test_cases { - let sql = helpers::nested_function_sql(test_case.depth); - let diagnostics = helpers::diagnostics(&sql).unwrap_or_else(|error| { - panic!("{}: {error}", test_case.description); - }); - assert_eq!( - diagnostics.len(), - test_case.expected_diagnostic_count, - "{}", - test_case.description - ); - } -} - -#[test] -fn given_excessive_function_depth_when_linting_then_complexity_guard_rejects_it() { - let test_cases = [test_types::FunctionDepthFailureTestCase { - description: "excessive depth remains bounded", - depth: 129, - expected_code: "E_GUARD_FUNCTION_NESTING_DEPTH_EXCEEDED", - expected_limit: "configured limit 128", - }]; - for test_case in test_cases { - let sql = helpers::nested_function_sql(test_case.depth); - let error = - helpers::diagnostics(&sql).expect_err("excessive nested calls should remain bounded"); - assert!( - error.contains(test_case.expected_code), - "{}", - test_case.description - ); - assert!( - error.contains(test_case.expected_limit), - "{}", - test_case.description - ); - } -} - -#[test] -fn given_sql_cases_when_linting_then_diagnostics_match() -> Result<(), String> { - let test_cases = [ - test_types::LintTestCase { - description: "NULL comparison", - sql: "SELECT value FROM a WHERE value = NULL", - expected_codes: &["SQBRSQL001"], - expected_anchors: &[("SQBRSQL001", "=")], - }, - test_types::LintTestCase { - description: "wrapped NULL comparison", - sql: "SELECT value FROM a WHERE (NULL) = value", - expected_codes: &["SQBRSQL001"], - expected_anchors: &[("SQBRSQL001", "=")], - }, - test_types::LintTestCase { - description: "implicit cartesian join", - sql: "SELECT a.id FROM a, b", - expected_codes: &["SQBRSQL002"], - expected_anchors: &[("SQBRSQL002", ",")], - }, - test_types::LintTestCase { - description: "unconditioned join", - sql: "SELECT a.id FROM a JOIN b", - expected_codes: &["SQBRSQL003"], - expected_anchors: &[("SQBRSQL003", "JOIN")], - }, - test_types::LintTestCase { - description: "unordered limit", - sql: "SELECT id FROM items LIMIT 1", - expected_codes: &["SQBRSQL004"], - expected_anchors: &[("SQBRSQL004", "LIMIT")], - }, - test_types::LintTestCase { - description: "limit zero selects no rows in any order", - sql: "SELECT id FROM items LIMIT 0", - expected_codes: &[], - expected_anchors: &[], - }, - test_types::LintTestCase { - description: "limit zero with an offset still selects no rows", - sql: "SELECT id FROM items LIMIT 0 OFFSET 5", - expected_codes: &[], - expected_anchors: &[], - }, - test_types::LintTestCase { - description: "semi-structured limit and offset keys are not row selection clauses", - sql: "SELECT payload:Limit::VARCHAR AS order_limit, payload:Offset::NUMBER AS order_offset FROM orders", - expected_codes: &[], - expected_anchors: &[], - }, - test_types::LintTestCase { - description: "semi-structured limit key does not hide a later row limit clause", - sql: "SELECT payload:Limit::VARCHAR AS order_limit FROM orders LIMIT 1", - expected_codes: &["SQBRSQL004"], - expected_anchors: &[("SQBRSQL004", "LIMIT")], - }, - test_types::LintTestCase { - description: "semi-structured limit key in a filter is not a row limit clause", - sql: "SELECT order_id FROM orders WHERE payload:Limit::NUMBER > 3", - expected_codes: &[], - expected_anchors: &[], - }, - test_types::LintTestCase { - description: "unused CTE", - sql: "WITH unused AS (SELECT 1) SELECT 1", - expected_codes: &["SQBRSQL005"], - expected_anchors: &[("SQBRSQL005", "unused")], - }, - test_types::LintTestCase { - description: "unreachable CTE chain", - sql: "WITH first AS (SELECT 1), second AS (SELECT * FROM first) SELECT 1", - expected_codes: &["SQBRSQL005", "SQBRSQL005"], - expected_anchors: &[], - }, - test_types::LintTestCase { - description: "unused CTE on set operation", - sql: "WITH unused AS (SELECT 1 AS id) SELECT id FROM left_table UNION ALL SELECT id FROM right_table", - expected_codes: &["SQBRSQL005"], - expected_anchors: &[("SQBRSQL005", "unused")], - }, - test_types::LintTestCase { - description: "redundant distinct", - sql: "SELECT DISTINCT id FROM items GROUP BY id", - expected_codes: &["SQBRSQL006"], - expected_anchors: &[("SQBRSQL006", "DISTINCT")], - }, - test_types::LintTestCase { - description: "aggregate distinct is not a select modifier", - sql: "SELECT category, COUNT(DISTINCT id) FROM items GROUP BY category", - expected_codes: &[], - expected_anchors: &[], - }, - test_types::LintTestCase { - description: "positional set star", - sql: "SELECT * FROM a UNION ALL SELECT * FROM b", - expected_codes: &["SQBRSQL007"], - expected_anchors: &[("SQBRSQL007", "UNION")], - }, - test_types::LintTestCase { - description: "parenthesized positional set star", - sql: "(SELECT * FROM a) UNION ALL (SELECT id FROM b)", - expected_codes: &["SQBRSQL007"], - expected_anchors: &[("SQBRSQL007", "UNION")], - }, - test_types::LintTestCase { - description: "nested set wildcard does not contaminate outer set operation", - sql: "SELECT 'actual' WHERE EXISTS (SELECT id FROM actual EXCEPT SELECT * FROM expected) UNION ALL SELECT 'expected'", - expected_codes: &["SQBRSQL007"], - expected_anchors: &[("SQBRSQL007", "EXCEPT")], - }, - test_types::LintTestCase { - description: "multiple statements", - sql: "SELECT value FROM first LIMIT 1; SELECT value FROM second OFFSET 1", - expected_codes: &["SQBRSQL004", "SQBRSQL004"], - expected_anchors: &[("SQBRSQL004", "LIMIT"), ("SQBRSQL004", "OFFSET")], - }, - test_types::LintTestCase { - description: "nested query anchors", - sql: "SELECT * FROM (SELECT id FROM inner_items ORDER BY id LIMIT 1) safe JOIN outer_items ON TRUE OFFSET 2", - expected_codes: &["SQBRSQL039", "SQBRSQL003", "SQBRSQL004"], - expected_anchors: &[ - ("SQBRSQL039", "SELECT"), - ("SQBRSQL003", "JOIN"), - ("SQBRSQL004", "OFFSET"), - ], - }, - test_types::LintTestCase { - description: "safe NULL predicate", - sql: "SELECT value FROM a WHERE value IS NULL", - expected_codes: &[], - expected_anchors: &[], - }, - test_types::LintTestCase { - description: "NULL inside function", - sql: "SELECT value FROM a WHERE COALESCE(value, NULL) = 1", - expected_codes: &[], - expected_anchors: &[], - }, - test_types::LintTestCase { - description: "NULL assignment", - sql: "UPDATE items SET value = NULL", - expected_codes: &[], - expected_anchors: &[], - }, - test_types::LintTestCase { - description: "cross join", - sql: "SELECT a.id FROM a CROSS JOIN b", - expected_codes: &[], - expected_anchors: &[], - }, - test_types::LintTestCase { - description: "keyed join", - sql: "SELECT a.id FROM a JOIN b ON a.id = b.id", - expected_codes: &[], - expected_anchors: &[], - }, - test_types::LintTestCase { - description: "ordered limit", - sql: "SELECT id FROM items ORDER BY id LIMIT 1", - expected_codes: &[], - expected_anchors: &[], - }, - test_types::LintTestCase { - description: "used CTE", - sql: "WITH used AS (SELECT 1 AS id) SELECT id FROM used", - expected_codes: &[], - expected_anchors: &[], - }, - test_types::LintTestCase { - description: "group without distinct", - sql: "SELECT id FROM items GROUP BY id", - expected_codes: &[], - expected_anchors: &[], - }, - test_types::LintTestCase { - description: "nonredundant distinct", - sql: "SELECT DISTINCT a FROM items GROUP BY a, b", - expected_codes: &[], - expected_anchors: &[], - }, - test_types::LintTestCase { - description: "within group", - sql: "SELECT DISTINCT LISTAGG(x, ',') WITHIN GROUP (ORDER BY x) FROM items", - expected_codes: &[], - expected_anchors: &[], - }, - test_types::LintTestCase { - description: "explicit positional set", - sql: "SELECT id FROM a UNION ALL SELECT id FROM b", - expected_codes: &[], - expected_anchors: &[], - }, - test_types::LintTestCase { - description: "nested star outside set arms", - sql: "SELECT id, name FROM (SELECT * FROM base) b UNION ALL SELECT id, name FROM other", - expected_codes: &["SQBRSQL039"], - expected_anchors: &[("SQBRSQL039", "SELECT")], - }, - test_types::LintTestCase { - description: "nested star inside wrapped set arm", - sql: "(SELECT id FROM (SELECT * FROM base) b) UNION ALL (SELECT id FROM other)", - expected_codes: &["SQBRSQL039"], - expected_anchors: &[("SQBRSQL039", "SELECT")], - }, - ]; - - for test_case in &test_cases { - let diagnostics = helpers::diagnostics(test_case.sql)?; - let codes: Vec<&str> = diagnostics - .iter() - .filter_map(|item| item["code"].as_str()) - .collect(); - let anchors: Vec<(&str, &str)> = diagnostics - .iter() - .filter_map(|item| { - let code = item["code"].as_str()?; - let start = item["start"].as_u64()? as usize; - let end = item["end"].as_u64()? as usize; - Some((code, &test_case.sql[start..end])) - }) - .collect(); - assert_eq!(codes, test_case.expected_codes, "{}", test_case.description); - for expected_anchor in test_case.expected_anchors { - assert!( - anchors.contains(expected_anchor), - "{}", - test_case.description - ); - } - } - Ok(()) -} - -#[test] -fn given_plain_sql_rules_when_linting_then_project_context_is_not_required() -> Result<(), String> { - let test_cases = [ - test_types::PlainSqlLintTestCase { - description: "trailing comment", - sql: "SELECT id -- why\nFROM items", - rule: "SQBRSQL033", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "detached comment", - sql: "SELECT id\n-- why\n\nFROM items", - rule: "SQBRSQL033", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "attached predicate comment", - sql: "SELECT id FROM items WHERE active\n-- retain settled rows\nAND settled", - rule: "SQBRSQL033", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "attached CTE comment", - sql: "WITH\n-- normalized rows\nnormalized AS (SELECT id FROM items)\nSELECT id FROM normalized", - rule: "SQBRSQL033", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "attached join comment", - sql: "SELECT left_rows.id FROM left_rows\n-- match stable identities\nJOIN right_rows ON left_rows.id = right_rows.id", - rule: "SQBRSQL033", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "attached filter comment", - sql: "SELECT id FROM items\n-- retain active rows\nWHERE active", - rule: "SQBRSQL033", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "attached projection comment", - sql: "SELECT\n-- stable identity\nid\nFROM items", - rule: "SQBRSQL033", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "attached CASE branch comment", - sql: "SELECT CASE\n-- settled outcome\nWHEN settled THEN 1 ELSE 0 END FROM items", - rule: "SQBRSQL033", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "attached ordering comment with Unicode and CRLF", - sql: "SELECT id FROM items\r\n-- deterministic café order\r\nORDER BY id", - rule: "SQBRSQL033", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "block comment", - sql: "SELECT id\n/* selected identity */\nFROM items", - rule: "SQBRSQL033", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "body without CTE", - sql: "SELECT id FROM items", - rule: "SQBRSQL034", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "logic in terminal select", - sql: "WITH final_rows AS (SELECT id FROM items) SELECT id FROM final_rows WHERE id > 0", - rule: "SQBRSQL034", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "terminal select reads wrong CTE", - sql: "WITH first_rows AS (SELECT 1 AS id), final_rows AS (SELECT id FROM first_rows) SELECT id FROM first_rows", - rule: "SQBRSQL035", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "plain terminal select", - sql: "WITH first_rows AS (SELECT 1 AS id), final_rows AS (SELECT id FROM first_rows) SELECT id FROM final_rows", - rule: "SQBRSQL035", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "plain terminal select with keyword-named columns", - sql: "WITH final_rows AS (SELECT 1 AS date, 2 AS comment, 3 AS key, 4 AS percent, 5 AS sequence, 6 AS time) SELECT date, comment, key, percent, sequence, time FROM final_rows", - rule: "SQBRSQL035", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "terminal date literal remains an expression", - sql: "WITH final_rows AS (SELECT 1 AS id) SELECT DATE '2026-01-01' FROM final_rows", - rule: "SQBRSQL035", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "terminal modulo calculation remains an expression", - sql: "WITH final_rows AS (SELECT 10 AS quantity, 3 AS pack_size) SELECT quantity % pack_size AS remainder FROM final_rows", - rule: "SQBRSQL035", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "plain terminal select from CTE with output column list", - sql: "WITH final_rows(id) AS (SELECT 1) SELECT id FROM final_rows", - rule: "SQBRSQL035", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "CTE output column list names its calculation", - sql: "WITH final_rows(id) AS (SELECT 1) SELECT id FROM final_rows", - rule: "SQBRSQL022", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "terminal union remains outside final CTE", - sql: "WITH final_rows AS (SELECT 1 AS id) SELECT id FROM final_rows UNION ALL SELECT id FROM final_rows", - rule: "SQBRSQL034", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "terminal union is not a plain final CTE read", - sql: "WITH final_rows AS (SELECT 1 AS id) SELECT id FROM final_rows UNION ALL SELECT id FROM final_rows", - rule: "SQBRSQL035", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "parenthesized terminal union remains outside final CTE", - sql: "WITH final_rows AS (SELECT 1 AS id) (SELECT id FROM final_rows) UNION ALL (SELECT id FROM final_rows)", - rule: "SQBRSQL034", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "parenthesized terminal union is not a plain final CTE read", - sql: "WITH final_rows AS (SELECT 1 AS id) (SELECT id FROM final_rows) UNION ALL (SELECT id FROM final_rows)", - rule: "SQBRSQL035", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "mixed parenthesized terminal union remains outside final CTE", - sql: "WITH final_rows AS (SELECT 1 AS id) (SELECT id FROM final_rows) UNION ALL SELECT id FROM final_rows", - rule: "SQBRSQL034", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "mixed parenthesized terminal union is not a plain final CTE read", - sql: "WITH final_rows AS (SELECT 1 AS id) (SELECT id FROM final_rows) UNION ALL SELECT id FROM final_rows", - rule: "SQBRSQL035", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "mixed parenthesized terminal intersect is not a plain final CTE read", - sql: "WITH final_rows AS (SELECT 1 AS id) (SELECT id FROM final_rows) INTERSECT SELECT id FROM final_rows", - rule: "SQBRSQL035", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "mixed parenthesized terminal except is not a plain final CTE read", - sql: "WITH final_rows AS (SELECT 1 AS id) (SELECT id FROM final_rows) EXCEPT SELECT id FROM final_rows", - rule: "SQBRSQL035", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "nested CTE", - sql: "SELECT id FROM (WITH nested_rows AS (SELECT 1 AS id) SELECT id FROM nested_rows)", - rule: "SQBRSQL036", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "Snowflake hierarchy START WITH is not a nested CTE", - sql: "WITH entities AS (SELECT id, parent_id FROM source_rows), hierarchy AS (SELECT id FROM entities START WITH parent_id IS NULL CONNECT BY parent_id = PRIOR id) SELECT id FROM hierarchy", - rule: "SQBRSQL036", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "recursive CTE", - sql: "WITH RECURSIVE numbers AS (SELECT 1 UNION ALL SELECT 2) SELECT * FROM numbers", - rule: "SQBRSQL037", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "cross join", - sql: "SELECT left_rows.id FROM left_rows CROSS JOIN right_rows", - rule: "SQBRSQL038", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "inline query after from", - sql: "SELECT derived.order_id FROM (SELECT order_id FROM orders) AS derived", - rule: "SQBRSQL039", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "inline query after join", - sql: "SELECT orders.order_id FROM orders INNER JOIN (SELECT order_id FROM shipments) AS shipped_orders ON orders.order_id = shipped_orders.order_id", - rule: "SQBRSQL039", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "inline values query after full outer join", - sql: "SELECT COALESCE(actual.order_id, expected.order_id) AS order_id FROM actual FULL OUTER JOIN (SELECT column1 AS order_id FROM VALUES (1), (2)) AS expected ON actual.order_id = expected.order_id", - rule: "SQBRSQL039", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "inline values relation after join", - sql: "SELECT orders.order_id FROM orders INNER JOIN (VALUES (1), (2)) AS expected(order_id) ON orders.order_id = expected.order_id", - rule: "SQBRSQL039", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "inline nested with query", - sql: "SELECT derived.order_id FROM (WITH imported_orders AS (SELECT order_id FROM orders) SELECT order_id FROM imported_orders) AS derived", - rule: "SQBRSQL039", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "redundantly parenthesized inline query", - sql: "SELECT derived.order_id FROM (((SELECT order_id FROM orders))) AS derived", - rule: "SQBRSQL039", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "comma-separated inline query", - sql: "SELECT orders.order_id FROM orders, (SELECT order_id FROM shipments) AS shipped_orders", - rule: "SQBRSQL039", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "scalar subquery", - sql: "SELECT (SELECT MAX(order_id) FROM orders) AS latest_order_id", - rule: "SQBRSQL039", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "scalar subquery containing a derived relation", - sql: "SELECT (SELECT derived.order_id FROM (SELECT nested.order_id FROM (SELECT 1 AS order_id) AS nested) AS derived) AS order_id", - rule: "SQBRSQL039", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "exists subquery containing a derived relation", - sql: "SELECT order_id FROM orders WHERE EXISTS (SELECT 1 FROM (SELECT nested.order_id FROM (SELECT order_id FROM shipments) AS nested) AS shipped_orders WHERE shipped_orders.order_id = orders.order_id)", - rule: "SQBRSQL039", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "exists set branch containing a derived relation", - sql: "SELECT EXISTS (SELECT 1 UNION ALL SELECT derived.order_id FROM (SELECT 1 AS order_id) AS derived)", - rule: "SQBRSQL039", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "scalar with query containing a derived relation in a CTE", - sql: "SELECT (WITH derived AS (SELECT nested.order_id FROM (SELECT 1 AS order_id) AS nested) SELECT order_id FROM derived) AS order_id", - rule: "SQBRSQL039", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "scalar with query containing a derived terminal relation", - sql: "SELECT (WITH orders AS (SELECT 1 AS order_id) SELECT derived.order_id FROM (SELECT order_id FROM orders) AS derived) AS order_id", - rule: "SQBRSQL039", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "nested derived relations outside an excluded subquery", - sql: "SELECT derived.order_id FROM (SELECT nested.order_id FROM (SELECT 1 AS order_id) AS nested) AS derived", - rule: "SQBRSQL039", - expected_count: 2, - }, - test_types::PlainSqlLintTestCase { - description: "nested derived relations inside a parenthesized ordinary relation", - sql: "SELECT derived.order_id FROM ((SELECT nested.order_id FROM (SELECT 1 AS order_id) AS nested)) AS derived", - rule: "SQBRSQL039", - expected_count: 2, - }, - test_types::PlainSqlLintTestCase { - description: "scalar subquery after projection comma", - sql: "SELECT 1 AS constant_value, (SELECT MAX(order_id) FROM orders) AS latest_order_id", - rule: "SQBRSQL039", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "scalar subquery after expression from", - sql: "SELECT EXTRACT(YEAR FROM (SELECT MAX(order_date) FROM orders)) AS latest_order_year", - rule: "SQBRSQL039", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "scalar subquery after sibling projection subquery", - sql: "SELECT (SELECT MAX(order_id) FROM orders), COALESCE(NULL, (SELECT MAX(order_id) FROM orders))", - rule: "SQBRSQL039", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "expression from after sibling projection subquery", - sql: "SELECT (SELECT MAX(order_id) FROM orders), EXTRACT(YEAR FROM (SELECT MAX(order_date) FROM orders))", - rule: "SQBRSQL039", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "scalar subquery after order by comma", - sql: "SELECT order_id FROM orders ORDER BY order_id, (SELECT MAX(status_id) FROM statuses)", - rule: "SQBRSQL039", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "scalar subquery with a derived relation after order by comma", - sql: "SELECT order_id FROM orders ORDER BY order_id, (SELECT derived.status_id FROM (SELECT 1 AS status_id) AS derived)", - rule: "SQBRSQL039", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "exists subquery", - sql: "SELECT order_id FROM orders WHERE EXISTS (SELECT 1 FROM shipments WHERE shipments.order_id = orders.order_id)", - rule: "SQBRSQL039", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "in subquery", - sql: "SELECT order_id FROM orders WHERE order_id IN (SELECT order_id FROM shipments)", - rule: "SQBRSQL039", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "named CTE relation", - sql: "WITH shipped_orders AS (SELECT order_id FROM shipments) SELECT orders.order_id FROM orders INNER JOIN shipped_orders ON orders.order_id = shipped_orders.order_id", - rule: "SQBRSQL039", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "materialized CTE containing a derived relation", - sql: "WITH orders AS MATERIALIZED (SELECT derived.order_id FROM (SELECT 1 AS order_id) AS derived) SELECT order_id FROM orders", - rule: "SQBRSQL039", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "not materialized CTE containing a derived relation", - sql: "WITH orders AS NOT MATERIALIZED (SELECT derived.order_id FROM (SELECT 1 AS order_id) AS derived) SELECT order_id FROM orders", - rule: "SQBRSQL039", - expected_count: 1, - }, - test_types::PlainSqlLintTestCase { - description: "lateral query relation", - sql: "SELECT orders.order_id FROM orders, LATERAL (SELECT orders.order_id) AS derived", - rule: "SQBRSQL039", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "apply query relation", - sql: "SELECT orders.order_id FROM orders CROSS APPLY (SELECT orders.order_id) AS derived", - rule: "SQBRSQL039", - expected_count: 0, - }, - test_types::PlainSqlLintTestCase { - description: "table function relation", - sql: "SELECT flattened.value FROM TABLE(FLATTEN(input => PARSE_JSON('[1]'))) AS flattened", - rule: "SQBRSQL039", - expected_count: 0, - }, - ]; - - for test_case in test_cases { - let diagnostics = helpers::diagnostics_for_rules(test_case.sql, &[test_case.rule])?; - assert_eq!( - diagnostics.len(), - test_case.expected_count, - "{}", - test_case.description - ); - assert!( - diagnostics - .iter() - .all(|diagnostic| diagnostic["code"] == test_case.rule), - "{}", - test_case.description - ); - } - Ok(()) -} - -#[test] -fn given_native_rules_when_linting_then_diagnosis_and_remediation_are_actionable() --> Result<(), String> { - let test_cases = [ - test_types::LintMetadataTestCase { - description: "NULL comparison guidance", - sql: "SELECT value FROM a WHERE value = NULL", - expected_code: "SQBRSQL001", - expected_message: "Comparison with NULL is never true", - expected_remediation: "Use IS NULL or IS NOT NULL when testing for NULL.", - }, - test_types::LintMetadataTestCase { - description: "implicit join guidance", - sql: "SELECT a.id FROM a, b", - expected_code: "SQBRSQL002", - expected_message: "Comma-separated sources create an implicit cartesian join", - expected_remediation: "Replace comma-separated sources with an explicit keyed join, or use CROSS JOIN when the cartesian product is intentional.", - }, - test_types::LintMetadataTestCase { - description: "unconditioned join guidance", - sql: "SELECT a.id FROM a JOIN b", - expected_code: "SQBRSQL003", - expected_message: "Non-cross join has no meaningful condition", - expected_remediation: "Add a meaningful ON or USING condition, or declare an intentional cartesian product with CROSS JOIN.", - }, - test_types::LintMetadataTestCase { - description: "unordered limit guidance", - sql: "SELECT id FROM items LIMIT 1", - expected_code: "SQBRSQL004", - expected_message: "Row selection is nondeterministic", - expected_remediation: "Add ORDER BY with a deterministic tie-breaker before LIMIT or OFFSET.", - }, - test_types::LintMetadataTestCase { - description: "unused CTE guidance", - sql: "WITH unused AS (SELECT 1) SELECT 1", - expected_code: "SQBRSQL005", - expected_message: "CTE is unreachable from the final query", - expected_remediation: "Reference the CTE from the final query or another reachable CTE, or remove it.", - }, - test_types::LintMetadataTestCase { - description: "redundant distinct guidance", - sql: "SELECT DISTINCT id FROM items GROUP BY id", - expected_code: "SQBRSQL006", - expected_message: "DISTINCT is redundant with the grouped output", - expected_remediation: "Remove DISTINCT; the equivalent GROUP BY already determines the output groups.", - }, - test_types::LintMetadataTestCase { - description: "positional set star guidance", - sql: "SELECT * FROM a UNION ALL SELECT * FROM b", - expected_code: "SQBRSQL007", - expected_message: "Positional set operation is vulnerable to column-order drift", - expected_remediation: "Enumerate columns in the same order in every set-operation branch.", - }, - ]; - - for test_case in test_cases { - let diagnostics = helpers::diagnostics(test_case.sql)?; - let diagnostic = diagnostics - .first() - .ok_or_else(|| format!("{} should report a diagnostic", test_case.description))?; - assert_eq!( - diagnostic["code"], test_case.expected_code, - "{}", - test_case.description - ); - assert_eq!( - diagnostic["message"], test_case.expected_message, - "{}", - test_case.description - ); - assert_eq!( - diagnostic["remediation"], test_case.expected_remediation, - "{}", - test_case.description - ); - let start = diagnostic["start"].as_u64().unwrap_or_default(); - let end = diagnostic["end"].as_u64().unwrap_or_default(); - assert!( - end > start, - "{} should report a non-empty range", - test_case.description - ); - } - Ok(()) -} - -#[test] -fn given_fixable_and_ambiguous_findings_when_linting_then_only_proven_edits_are_returned() --> Result<(), String> { - let test_cases = [ - test_types::LintFixTestCase { - description: "right-hand NULL equality", - sql: "SELECT value FROM a WHERE value = NULL", - expected_code: "SQBRSQL001", - expected_replacement: Some("IS"), - }, - test_types::LintFixTestCase { - description: "right-hand NULL inequality", - sql: "SELECT value FROM a WHERE value <> NULL", - expected_code: "SQBRSQL001", - expected_replacement: Some("IS NOT"), - }, - test_types::LintFixTestCase { - description: "left-hand NULL comparison requires authored intent", - sql: "SELECT value FROM a WHERE NULL = value", - expected_code: "SQBRSQL001", - expected_replacement: None, - }, - test_types::LintFixTestCase { - description: "plain conditionless join", - sql: "SELECT a.id FROM a JOIN b", - expected_code: "SQBRSQL003", - expected_replacement: Some("CROSS JOIN"), - }, - test_types::LintFixTestCase { - description: "qualified conditionless join requires intent", - sql: "SELECT a.id FROM a LEFT JOIN b", - expected_code: "SQBRSQL003", - expected_replacement: None, - }, - test_types::LintFixTestCase { - description: "conditionless semi join is never rewritten as cross join", - sql: "SELECT a.id FROM a SEMI JOIN b", - expected_code: "SQBRSQL003", - expected_replacement: None, - }, - test_types::LintFixTestCase { - description: "redundant distinct", - sql: "SELECT DISTINCT id FROM items GROUP BY id", - expected_code: "SQBRSQL006", - expected_replacement: Some(""), - }, - test_types::LintFixTestCase { - description: "single unused select CTE", - sql: "WITH unused AS (SELECT 1) SELECT 1", - expected_code: "SQBRSQL005", - expected_replacement: Some(""), - }, - test_types::LintFixTestCase { - description: "reserved framework CTE is never deleted", - sql: "WITH __expected__items AS (SELECT 1) SELECT 1", - expected_code: "SQBRSQL005", - expected_replacement: None, - }, - ]; - - for test_case in test_cases { - let diagnostics = helpers::diagnostics(test_case.sql)?; - let diagnostic = diagnostics - .iter() - .find(|item| item["code"] == test_case.expected_code) - .ok_or_else(|| format!("{} should report", test_case.description))?; - assert_eq!( - diagnostic["fix"]["replacement"].as_str(), - test_case.expected_replacement, - "{}", - test_case.description - ); - } - Ok(()) -} - -#[test] -fn given_additional_rule_cases_when_linting_then_findings_and_fixes_match() -> Result<(), String> { - let test_cases = [ - test_types::AdditionalLintRuleTestCase { - description: "bare union", - sql: "SELECT 1 UNION SELECT 2", - rule: "SQBRSQL008", - expected_anchor: Some("UNION"), - expected_replacement: Some("UNION DISTINCT"), - }, - test_types::AdditionalLintRuleTestCase { - description: "duplicate relation alias", - sql: "SELECT x.id FROM alpha AS x JOIN beta AS x ON x.id = x.id", - rule: "SQBRSQL009", - expected_anchor: Some("x"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "duplicate output alias", - sql: "SELECT a AS value, b AS value FROM items", - rule: "SQBRSQL010", - expected_anchor: Some("value"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "mixed grouping references", - sql: "SELECT a, b FROM items GROUP BY 1, b", - rule: "SQBRSQL011", - expected_anchor: Some("GROUP"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "redundant else null", - sql: "SELECT CASE WHEN a THEN b ELSE NULL END FROM items", - rule: "SQBRSQL012", - expected_anchor: Some("ELSE NULL"), - expected_replacement: Some(""), - }, - test_types::AdditionalLintRuleTestCase { - description: "function-like distinct", - sql: "SELECT DISTINCT(a) FROM items", - rule: "SQBRSQL013", - expected_anchor: Some("DISTINCT(a)"), - expected_replacement: Some("DISTINCT a"), - }, - test_types::AdditionalLintRuleTestCase { - description: "constant scaffold predicate", - sql: "SELECT a FROM items WHERE 1 = 1", - rule: "SQBRSQL014", - expected_anchor: Some("="), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "consecutive statement terminator", - sql: "SELECT 1;;", - rule: "SQBRSQL015", - expected_anchor: Some(";"), - expected_replacement: Some(""), - }, - test_types::AdditionalLintRuleTestCase { - description: "redundant self alias", - sql: "SELECT value AS value FROM items", - rule: "SQBRSQL016", - expected_anchor: Some(" AS value"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "qualified column self alias is redundant", - sql: "SELECT items.value AS value FROM items", - rule: "SQBRSQL016", - expected_anchor: Some(" AS value"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "qualified column self alias after an expression is redundant", - sql: "SELECT item_id + 1 AS next_item_id, items.value AS value FROM items", - rule: "SQBRSQL016", - expected_anchor: Some(" AS value"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "top-modified column self alias is redundant", - sql: "SELECT TOP 5 value AS value FROM items", - rule: "SQBRSQL016", - expected_anchor: Some(" AS value"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "terminal operand name does not make a calculation self-aliased", - sql: "SELECT price + value AS value FROM items", - rule: "SQBRSQL016", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "cast type syntax is not an output self alias", - sql: "SELECT CAST(order_id::INTEGER AS INTEGER) AS order_id FROM orders", - rule: "SQBRSQL016", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "relation self alias is not an output self alias", - sql: "SELECT orders.order_id FROM orders AS orders", - rule: "SQBRSQL016", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "semi-structured path alias is not redundant", - sql: "SELECT event.payload:data:order_id AS order_id FROM events AS event", - rule: "SQBRSQL016", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "constant row count", - sql: "SELECT COUNT(1) FROM items", - rule: "SQBRSQL017", - expected_anchor: Some("1"), - expected_replacement: Some("*"), - }, - test_types::AdditionalLintRuleTestCase { - description: "unordered row number", - sql: "SELECT ROW_NUMBER() OVER (PARTITION BY id) FROM items", - rule: "SQBRSQL018", - expected_anchor: Some("ROW_NUMBER"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "literal null in not-in list", - sql: "SELECT id FROM items WHERE id NOT IN (1, NULL)", - rule: "SQBRSQL019", - expected_anchor: Some("NOT"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "set branch arity mismatch", - sql: "SELECT a, b FROM first UNION ALL SELECT c FROM second", - rule: "SQBRSQL020", - expected_anchor: Some("UNION"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "uncontrolled projection star", - sql: "SELECT * FROM items", - rule: "SQBRSQL021", - expected_anchor: Some("*"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "projection star without from remains uncontrolled", - sql: "SELECT *", - rule: "SQBRSQL021", - expected_anchor: Some("*"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "mixed dynamic pivot projection remains uncontrolled", - sql: "SELECT *, customer_id FROM orders PIVOT(MAX(amount) FOR category IN (ANY))", - rule: "SQBRSQL021", - expected_anchor: Some("*"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "static pivot star remains uncontrolled", - sql: "SELECT * FROM orders PIVOT(MAX(amount) FOR category IN ('books'))", - rule: "SQBRSQL021", - expected_anchor: Some("*"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "static pivot any string remains uncontrolled", - sql: "SELECT * FROM orders PIVOT(MAX(amount) FOR category IN ('ANY'))", - rule: "SQBRSQL021", - expected_anchor: Some("*"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "nested dynamic pivot does not exempt outer projection star", - sql: "SELECT * FROM orders WHERE EXISTS (SELECT 1 FROM items PIVOT(MAX(amount) FOR category IN (ANY)))", - rule: "SQBRSQL021", - expected_anchor: Some("*"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "unrelated any does not exempt static pivot star", - sql: "SELECT * FROM orders PIVOT(MAX(amount) FOR category IN ('books')) WHERE customer_id = ANY (SELECT customer_id FROM customers)", - rule: "SQBRSQL021", - expected_anchor: Some("*"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "comma joined dynamic pivot star remains uncontrolled", - sql: "SELECT * FROM orders PIVOT(MAX(amount) FOR category IN (ANY)) AS p, customers AS c", - rule: "SQBRSQL021", - expected_anchor: Some("*"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "joined audit relation parameter star remains uncontrolled", - sql: "SELECT * FROM __sqlbuild_audit_parameter_0__ AS a CROSS JOIN customers AS c", - rule: "SQBRSQL021", - expected_anchor: Some("*"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "terminal framework dependency star remains uncontrolled", - sql: "SELECT * FROM __sqb_lint_0__", - rule: "SQBRSQL021", - expected_anchor: Some("*"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "transformed framework dependency import star remains uncontrolled", - sql: "WITH imported_orders AS (SELECT *, CURRENT_TIMESTAMP AS loaded_at FROM __sqb_lint_0__) SELECT order_id FROM imported_orders", - rule: "SQBRSQL021", - expected_anchor: Some("*"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "ordinary relation import star remains uncontrolled", - sql: "WITH imported_orders AS (SELECT * FROM orders) SELECT order_id FROM imported_orders", - rule: "SQBRSQL021", - expected_anchor: Some("*"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "nested framework dependency import star remains uncontrolled", - sql: "WITH outer_query AS (WITH imported_orders AS (SELECT * FROM __sqb_lint_0__) SELECT order_id FROM imported_orders) SELECT order_id FROM outer_query", - rule: "SQBRSQL021", - expected_anchor: Some("*"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "unaliased calculated projection", - sql: "SELECT price * quantity FROM items", - rule: "SQBRSQL022", - expected_anchor: Some("price"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "unused table alias", - sql: "SELECT id FROM items AS unused", - rule: "SQBRSQL023", - expected_anchor: Some("AS unused"), - expected_replacement: Some(""), - }, - test_types::AdditionalLintRuleTestCase { - description: "left join rejected in where", - sql: "SELECT a.id FROM a LEFT JOIN b AS right_side ON a.id = right_side.id WHERE right_side.active = TRUE", - rule: "SQBRSQL024", - expected_anchor: Some("LEFT"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "implicit inner join", - sql: "SELECT a.id FROM a JOIN b ON a.id = b.id", - rule: "SQBRSQL025", - expected_anchor: Some("JOIN"), - expected_replacement: Some("INNER JOIN"), - }, - test_types::AdditionalLintRuleTestCase { - description: "mixed order directions", - sql: "SELECT a, b FROM items ORDER BY a, b DESC", - rule: "SQBRSQL026", - expected_anchor: Some("ORDER"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "unqualified multi-source column", - sql: "SELECT id FROM a JOIN b ON a.id = b.id", - rule: "SQBRSQL027", - expected_anchor: Some("id"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "mixed single-source qualification", - sql: "SELECT items.id, name FROM items", - rule: "SQBRSQL028", - expected_anchor: Some("name"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "plain macro slot reads count as unqualified beside a plain group read", - sql: "SELECT __sqlbuild_macro_parameter_1__, __sqlbuild_macro_parameter_0__.__sqlbuild_macro_parameter_2__ AS orders FROM __sqlbuild_macro_parameter_0__", - rule: "SQBRSQL028", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "a plain macro slot read beside a hand-qualified column is mixed", - sql: "SELECT __sqlbuild_macro_parameter_0__.__sqlbuild_macro_parameter_1__ AS orders, s.refunds FROM __sqlbuild_macro_parameter_0__ AS s", - rule: "SQBRSQL028", - expected_anchor: Some("__sqlbuild_macro_parameter_0__"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "unknown relation qualifier", - sql: "SELECT missing.id FROM items AS present", - rule: "SQBRSQL029", - expected_anchor: Some("missing"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "Snowflake shorthand cast type is not an unqualified column", - sql: "SELECT src.value::STRING AS value FROM items AS src", - rule: "SQBRSQL028", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "Snowflake JSON path and shorthand cast type are not columns", - sql: "SELECT src.payload:item::TEXT AS item FROM items AS src", - rule: "SQBRSQL028", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "Snowflake shorthand cast type is not a multi-source column", - sql: "SELECT first.value::TEXT AS value FROM first JOIN second ON first.id = second.id", - rule: "SQBRSQL027", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "implicit table alias is known in query scope", - sql: "SELECT present.id FROM items present", - rule: "SQBRSQL029", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "implicit joined alias contributes a selected value", - sql: "SELECT left_side.id, right_side.name FROM alpha left_side LEFT JOIN beta right_side ON left_side.id = right_side.id", - rule: "SQBRSQL032", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "Snowflake lateral flatten comma is explicit lateral syntax", - sql: "SELECT flattened.value FROM items, LATERAL FLATTEN(input => items.payload) flattened", - rule: "SQBRSQL002", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "Snowflake VALUES row separator is not a relation separator", - sql: "SELECT column1 FROM VALUES (1), (2), (3)", - rule: "SQBRSQL002", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "arithmetic multiplication is not a projected wildcard", - sql: "SELECT price * quantity AS total FROM items", - rule: "SQBRSQL021", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "number-led multiplication is not a projected wildcard", - sql: "SELECT 100 * ratio AS percentage FROM items", - rule: "SQBRSQL021", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "TOP projection wildcard remains controlled by star rule", - sql: "SELECT TOP 10 * FROM items", - rule: "SQBRSQL021", - expected_anchor: Some("*"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "qualified wildcard modifier does not require an alias", - sql: "SELECT src.* EXCLUDE (internal_id) FROM items AS src", - rule: "SQBRSQL022", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "EXISTS literal projection does not require an alias", - sql: "SELECT parent.id FROM parent WHERE EXISTS (SELECT 1 FROM child WHERE child.id = parent.id)", - rule: "SQBRSQL022", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "correlated subquery can reference an outer relation", - sql: "SELECT parent.id FROM parent WHERE EXISTS (SELECT 1 FROM child WHERE child.id = parent.id)", - rule: "SQBRSQL029", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "Snowflake JSON path key is not a relation qualifier", - sql: "SELECT PARSE_JSON(src.payload):item.name::VARCHAR AS item_name FROM items AS src", - rule: "SQBRSQL029", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "Snowflake JSON path starts from a qualified relation value", - sql: "SELECT parsed:item::VARCHAR AS item FROM items AS src, LATERAL FLATTEN(input => src.payload) AS parsed", - rule: "SQBRSQL027", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "Snowflake ASOF JOIN alias is known in query scope", - sql: "SELECT matched.ts FROM requested request ASOF JOIN available matched MATCH_CONDITION (request.ts >= matched.ts) ON request.id = matched.id", - rule: "SQBRSQL029", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "later set branches inherit the first branch output name", - sql: "SELECT 0 AS offset_value UNION ALL SELECT 1 UNION ALL SELECT 2", - rule: "SQBRSQL022", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "scalar subquery output does not require an alias inside its scope", - sql: "SELECT (SELECT MAX(child.value) FROM child WHERE child.id = parent.id) AS maximum_value FROM parent", - rule: "SQBRSQL022", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "later scalar function argument does not require an inner alias", - sql: "SELECT COALESCE((SELECT MAX(change_id) FROM first_changes), (SELECT MAX(change_id) FROM second_changes), 0) AS latest_change_id FROM support_tickets", - rule: "SQBRSQL022", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "scalar aggregate nested in conditional coalesce does not require an inner alias", - sql: "SELECT CASE WHEN batch.layout = 'fallback' THEN COALESCE((SELECT MAX(change.line_id) FROM first_changes AS change WHERE change.batch_id = batch.batch_id AND change.line_id < batch.line_id), (SELECT MAX(archive.line_id) FROM archived_changes AS archive WHERE archive.batch_id = batch.batch_id AND archive.line_id < batch.line_id), header.line_id) + 1 ELSE batch.start_line END AS start_line FROM support_batches AS batch LEFT JOIN support_headers AS header ON header.batch_id = batch.batch_id", - rule: "SQBRSQL022", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "prior nested relation does not contaminate a later scalar function argument", - sql: "WITH prior_changes AS (SELECT (SELECT MAX(change_id) FROM support_changes) AS latest_change_id FROM support_tickets), current_changes AS (SELECT COALESCE((SELECT MAX(change_id) FROM first_changes), (SELECT MAX(change_id) FROM second_changes), 0) AS latest_change_id FROM current_tickets) SELECT * FROM current_changes", - rule: "SQBRSQL022", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "scalar subquery after a prior CTE relation remains in projection scope", - sql: "WITH products_used AS (SELECT product_id FROM products), ticket_totals AS (SELECT ticket.ticket_id, (SELECT MAX(change_id) FROM support_changes) AS latest_change_id FROM support_tickets AS ticket) SELECT * FROM ticket_totals", - rule: "SQBRSQL022", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "comma-derived table calculation still requires an alias", - sql: "SELECT derived.value FROM items, (SELECT amount + tax FROM totals) AS derived", - rule: "SQBRSQL022", - expected_anchor: Some("amount"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "scalar subquery scope does not cross a prior statement", - sql: "SELECT order_id FROM orders; SELECT invoice_id, (SELECT MAX(amount + tax) FROM totals) AS maximum_total FROM invoices", - rule: "SQBRSQL022", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "contextual keyword remains a simple qualified projection", - sql: "SELECT flattened.index FROM items AS src, LATERAL FLATTEN(input => src.payload) AS flattened", - rule: "SQBRSQL022", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "generated expression sentinel is not an authored column", - sql: "SELECT first.id FROM first JOIN second ON first.id = second.id WHERE __sqb_lint_0__", - rule: "SQBRSQL027", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "null-safe comparison FROM is not a relation clause", - sql: "SELECT CASE WHEN src.actual IS DISTINCT FROM src.expected THEN 1 ELSE 0 END AS differs FROM items AS src", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "earlier projection alias is not a relation column", - sql: "SELECT flattened.value AS item_index, item_index + 1 AS next_index FROM items AS src, LATERAL FLATTEN(input => src.payload) AS flattened", - rule: "SQBRSQL027", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "current date context value is not a multi-source column", - sql: "SELECT first_side.id FROM first_side JOIN second_side ON first_side.id = second_side.id WHERE first_side.created_at >= CURRENT_DATE", - rule: "SQBRSQL027", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "current date context value does not mix qualification styles", - sql: "SELECT first_side.id FROM first_side WHERE first_side.created_at >= CURRENT_DATE", - rule: "SQBRSQL028", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "simple boolean case", - sql: "SELECT CASE WHEN amount > 0 THEN TRUE ELSE FALSE END AS positive FROM items", - rule: "SQBRSQL030", - expected_anchor: Some("CASE WHEN amount > 0 THEN TRUE ELSE FALSE END"), - expected_replacement: Some("COALESCE(amount > 0, FALSE)"), - }, - test_types::AdditionalLintRuleTestCase { - description: "unicode before simple boolean case preserves replacement text", - sql: "SELECT 'é', CASE WHEN amount > 0 THEN TRUE ELSE FALSE END AS positive FROM items", - rule: "SQBRSQL030", - expected_anchor: Some("CASE WHEN amount > 0 THEN TRUE ELSE FALSE END"), - expected_replacement: Some("COALESCE(amount > 0, FALSE)"), - }, - test_types::AdditionalLintRuleTestCase { - description: "commented boolean case is diagnosed without a fix", - sql: "SELECT CASE WHEN amount /* reason */ > 0 THEN TRUE ELSE FALSE END FROM items", - rule: "SQBRSQL030", - expected_anchor: Some("CASE WHEN amount /* reason */ > 0 THEN TRUE ELSE FALSE END"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "multi-branch boolean case is diagnosed without an unsafe partial fix", - sql: "SELECT CASE WHEN status = 'open' THEN TRUE WHEN priority = 'high' THEN TRUE ELSE FALSE END AS actionable FROM support_tickets", - rule: "SQBRSQL030", - expected_anchor: Some( - "CASE WHEN status = 'open' THEN TRUE WHEN priority = 'high' THEN TRUE ELSE FALSE END", - ), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "case nested directly in else", - sql: "SELECT CASE WHEN a THEN 1 ELSE CASE WHEN b THEN 2 END END FROM items", - rule: "SQBRSQL031", - expected_anchor: Some("CASE"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "joined relation contributes no values", - sql: "SELECT a.id FROM a LEFT JOIN b ON a.id = b.id", - rule: "SQBRSQL032", - expected_anchor: Some("JOIN"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "joined relation contributes through an aggregate expression", - sql: "SELECT ARRAY_AGG(b.value) AS values FROM a LEFT JOIN b ON a.id = b.id", - rule: "SQBRSQL032", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "bare projected star consumes every joined relation", - sql: "SELECT * FROM orders AS o INNER JOIN customers AS c ON o.customer_id = c.customer_id", - rule: "SQBRSQL032", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "unqualified projection prevents an unsupported unused relation claim", - sql: "SELECT customer_name FROM orders AS o INNER JOIN customers AS c ON o.customer_id = c.customer_id", - rule: "SQBRSQL032", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "unqualified aggregate argument prevents an unsupported unused relation claim", - sql: "SELECT MAX(customer_name) FROM orders AS o INNER JOIN customers AS c ON o.customer_id = c.customer_id", - rule: "SQBRSQL032", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "left qualified star does not consume joined relation", - sql: "SELECT o.* FROM orders AS o INNER JOIN customers AS c ON o.customer_id = c.customer_id", - rule: "SQBRSQL032", - expected_anchor: Some("JOIN"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "cross joined relation contributes through a filter predicate", - sql: "SELECT orders.order_id FROM orders CROSS JOIN processing_cutoff AS cutoff WHERE orders.created_at < cutoff.created_at", - rule: "SQBRSQL032", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "lateral relation contributes through a downstream join condition", - sql: "SELECT orders.order_id, products.name FROM orders CROSS JOIN LATERAL UNNEST(orders.product_ids) AS item INNER JOIN products ON (products.product_id = item.product_id)", - rule: "SQBRSQL032", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "explicit union is clean", - sql: "SELECT 1 UNION ALL SELECT 2", - rule: "SQBRSQL008", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "ordered row number without a declared key is unproven", - sql: "SELECT ROW_NUMBER() OVER (PARTITION BY id ORDER BY created_at, id) FROM items", - rule: "SQBRSQL018", - expected_anchor: Some("ROW_NUMBER"), - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "equal set arity is clean", - sql: "SELECT a, b FROM first UNION ALL SELECT c, d FROM second", - rule: "SQBRSQL020", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "dynamic pivot star has data-dependent columns", - sql: "SELECT * FROM orders PIVOT(MAX(amount) FOR category IN (ANY ORDER BY category))", - rule: "SQBRSQL021", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "audit relation parameter star has invocation-dependent columns", - sql: "SELECT * FROM __sqlbuild_audit_parameter_0__", - rule: "SQBRSQL021", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "qualified alias is used", - sql: "SELECT kept.id FROM items AS kept", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "left join predicate retained in on is clean", - sql: "SELECT a.id FROM a LEFT JOIN b ON a.id = b.id AND b.active = TRUE", - rule: "SQBRSQL024", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "explicit inner join is clean", - sql: "SELECT a.id FROM a INNER JOIN b ON a.id = b.id", - rule: "SQBRSQL025", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "null literal inside not-in subquery expression is not a literal list", - sql: "SELECT id FROM items WHERE id NOT IN (SELECT COALESCE(id, NULL) FROM other)", - rule: "SQBRSQL019", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "left join null probe preserves unmatched rows", - sql: "SELECT a.id FROM a LEFT JOIN b ON a.id = b.id WHERE b.id IS NULL", - rule: "SQBRSQL024", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "left join predicate guarded by null probe preserves unmatched rows", - sql: "SELECT a.id FROM a LEFT JOIN b ON a.id = b.id WHERE b.active = TRUE OR b.id IS NULL", - rule: "SQBRSQL024", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "qualified catalog path is not treated as an unknown relation alias", - sql: "SELECT catalog.schema.items.id FROM catalog.schema.items", - rule: "SQBRSQL029", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "correlated subquery keeps outer table alias", - sql: "SELECT id FROM items AS kept WHERE EXISTS (SELECT 1 FROM other WHERE other.id = kept.id)", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "asof join is not rewritten as an inner join", - sql: "SELECT a.id FROM a ASOF JOIN b ON a.id = b.id", - rule: "SQBRSQL025", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "semi join is not rewritten as an inner join", - sql: "SELECT a.id FROM a SEMI JOIN b ON a.id = b.id", - rule: "SQBRSQL025", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "table alias carrying a column list is retained", - sql: "SELECT a, b FROM items AS t(a, b)", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "bracket list commas do not change set projection arity", - sql: "SELECT a, [1, 2, 3] AS arr FROM t UNION ALL SELECT a, [4] AS arr FROM t", - rule: "SQBRSQL020", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "quoted output alias is not redundant with an unquoted identifier", - sql: "SELECT value AS \"value\" FROM items", - rule: "SQBRSQL016", - expected_anchor: None, - expected_replacement: None, - }, - ]; - - for test_case in test_cases { - let diagnostics = helpers::diagnostics_for_rules(test_case.sql, &[test_case.rule])?; - assert_eq!( - diagnostics.len(), - usize::from(test_case.expected_anchor.is_some()), - "{}", - test_case.description - ); - let _ = test_case.expected_anchor.map(|expected_anchor| { - let diagnostic = &diagnostics[0]; - let start = diagnostic["start"].as_u64().unwrap_or_default() as usize; - let end = diagnostic["end"].as_u64().unwrap_or_default() as usize; - let source: String = test_case - .sql - .chars() - .skip(start) - .take(end - start) - .collect(); - assert_eq!(source, expected_anchor, "{}", test_case.description); - assert_eq!( - diagnostic["fix"]["replacement"].as_str(), - test_case.expected_replacement, - "{}", - test_case.description - ); - }); - } - Ok(()) -} - -#[test] -fn given_ceremonial_cte_contexts_when_linting_then_only_harness_dependencies_are_reachable() --> Result<(), String> { - let test_cases = [ - test_types::CeremonialCteLintTestCase { - description: "macro harness outputs keep their helper dependency reachable", - sql: "WITH input_values AS (SELECT 7 AS amount), __macro_actual__ AS (SELECT amount FROM input_values), __macro_expected__ AS (SELECT 7 AS amount) SELECT 1", - expected_count: 0, - expected_start: None, - }, - test_types::CeremonialCteLintTestCase { - description: "non-harness double-underscore CTE remains diagnosable", - sql: "WITH __scratch AS (SELECT 1), __macro_actual__ AS (SELECT 7 AS amount), __macro_expected__ AS (SELECT 7 AS amount) SELECT 1", - expected_count: 1, - expected_start: Some(5), - }, - ]; - for test_case in test_cases { - let response = lint_json( - &json!({ - "version": 1, - "sql": test_case.sql, - "dialect": "duckdb", - "enabled_rules": ["SQBRSQL005"], - "allows_ceremonial_select": true, - }) - .to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - let diagnostics = payload["diagnostics"] - .as_array() - .ok_or_else(|| "diagnostics should be an array".to_string())?; - - assert_eq!( - diagnostics.len(), - test_case.expected_count, - "{}", - test_case.description - ); - assert_eq!( - diagnostics - .first() - .and_then(|diagnostic| diagnostic["start"].as_u64()), - test_case.expected_start, - "{}", - test_case.description - ); - } - Ok(()) -} - -#[test] -fn given_dependency_identity_when_linting_import_stars_then_only_plain_top_level_import_is_accepted() --> Result<(), String> { - let test_cases = [ - test_types::DependencyImportLintTestCase { - description: "plain top-level dependency import", - sql: "WITH imported_orders AS (SELECT * FROM __sqb_lint_0__) SELECT order_id FROM imported_orders", - dependency_identifiers: &["__sqb_lint_0__"], - expected_count: 0, - }, - test_types::DependencyImportLintTestCase { - description: "star without from remains diagnosable", - sql: "WITH imported_orders AS (SELECT *) SELECT order_id FROM imported_orders", - dependency_identifiers: &["__sqb_lint_0__"], - expected_count: 1, - }, - test_types::DependencyImportLintTestCase { - description: "set operation import is transformed", - sql: "WITH imported_orders AS (SELECT * FROM __sqb_lint_0__ UNION ALL SELECT order_id FROM orders) SELECT order_id FROM imported_orders", - dependency_identifiers: &["__sqb_lint_0__"], - expected_count: 1, - }, - test_types::DependencyImportLintTestCase { - description: "dynamic relation is not a dependency", - sql: "WITH imported_orders AS (SELECT * FROM __sqb_lint_0__) SELECT order_id FROM imported_orders", - dependency_identifiers: &[], - expected_count: 1, - }, - test_types::DependencyImportLintTestCase { - description: "terminal dependency star remains uncontrolled", - sql: "SELECT * FROM __sqb_lint_0__", - dependency_identifiers: &["__sqb_lint_0__"], - expected_count: 1, - }, - test_types::DependencyImportLintTestCase { - description: "transformed dependency import remains uncontrolled", - sql: "WITH imported_orders AS (SELECT *, CURRENT_TIMESTAMP AS loaded_at FROM __sqb_lint_0__) SELECT order_id FROM imported_orders", - dependency_identifiers: &["__sqb_lint_0__"], - expected_count: 1, - }, - test_types::DependencyImportLintTestCase { - description: "nested dependency import remains uncontrolled", - sql: "WITH outer_query AS (WITH imported_orders AS (SELECT * FROM __sqb_lint_0__) SELECT order_id FROM imported_orders) SELECT order_id FROM outer_query", - dependency_identifiers: &["__sqb_lint_0__"], - expected_count: 1, - }, - ]; - for test_case in test_cases { - let response = lint_json( - &json!({ - "version": 1, - "sql": test_case.sql, - "dialect": "duckdb", - "enabled_rules": ["SQBRSQL021"], - "dependency_identifiers": test_case.dependency_identifiers, - }) - .to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - let diagnostics = payload["diagnostics"] - .as_array() - .ok_or_else(|| "diagnostics should be an array".to_owned())?; - assert_eq!( - diagnostics.len(), - test_case.expected_count, - "{}", - test_case.description - ); - } - Ok(()) -} - -#[test] -fn given_ceremonial_select_context_when_linting_terminal_shape_then_rule_035_allows_it() --> Result<(), String> { - let test_cases = [test_types::PlainSqlLintTestCase { - description: "direct ceremonial select is valid in harness context", - sql: "WITH __source__raw_orders AS (SELECT 1 AS order_id), __expected__orders AS (SELECT 1 AS order_id) SELECT 1", - rule: "SQBRSQL035", - expected_count: 0, - }]; - for test_case in test_cases { - let response = lint_json( - &json!({ - "version": 1, - "sql": test_case.sql, - "dialect": "duckdb", - "enabled_rules": [test_case.rule], - "allows_ceremonial_select": true, - }) - .to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - let diagnostics = payload["diagnostics"] - .as_array() - .ok_or_else(|| "diagnostics should be an array".to_owned())?; - assert_eq!( - diagnostics.len(), - test_case.expected_count, - "{}", - test_case.description - ); - } - Ok(()) -} - -#[test] -fn given_tsql_bare_union_when_linting_then_diagnostic_has_no_invalid_fix() -> Result<(), String> { - let test_cases = [test_types::DialectLintRuleTestCase { - description: "T-SQL bare UNION is diagnosed without an invalid fix", - sql: "SELECT 1 UNION SELECT 2", - dialect: "tsql", - rule: "SQBRSQL008", - expected_count: 1, - expected_fix: false, - expected_reason: "the active dialect does not accept explicit UNION DISTINCT", - }]; - - for test_case in test_cases { - let diagnostics = - helpers::diagnostics_for_dialect(test_case.sql, test_case.dialect, &[test_case.rule])?; - assert_eq!( - diagnostics.len(), - test_case.expected_count, - "{}", - test_case.description - ); - assert_eq!( - diagnostics[0].get("fix").is_some(), - test_case.expected_fix, - "{}", - test_case.description - ); - assert_eq!( - diagnostics[0]["fix_unavailable_reason"], test_case.expected_reason, - "{}", - test_case.description - ); - } - Ok(()) -} - -#[test] -fn given_format_cases_when_formatting_then_output_matches() -> Result<(), String> { - let test_cases = [ - test_types::FormatTestCase { - description: "comment-free canonical SQL", - sql: "select a,b from items where a=1", - expected_sql: "SELECT\n a,\n b\nFROM items\nWHERE a = 1", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "lowercase function canonicalization preserves structure", - sql: "select count(1) from items", - expected_sql: "SELECT COUNT(1)\nFROM items", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "block comment attachment", - sql: "SELECT a, /* preserve */ b FROM items", - expected_sql: "SELECT\n a, /* preserve */\n b\nFROM items", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "line and leading comments", - sql: "-- lead\nselect a,b from items -- tail\nwhere a=1", - expected_sql: "-- lead\nSELECT\n a,\n b\nFROM items -- tail\nWHERE a = 1", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "standalone projection comments remain attached to following expressions", - sql: "SELECT\n -- first value\n a,\n -- second value\n b\nFROM items", - expected_sql: "SELECT\n -- first value\n a,\n -- second value\n b\nFROM items", - expected_changed: false, - }, - test_types::FormatTestCase { - description: "comment marker string", - sql: "select '-- not a comment' as value", - expected_sql: "SELECT '-- not a comment' AS value", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "doubled apostrophe string remains compiler compatible", - sql: "select 'Customer''s order' as value", - expected_sql: "SELECT 'Customer''s order' AS value", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "explicit null ordering remains explicit", - sql: "select id from items order by category nulls last, created_at desc nulls first", - expected_sql: "SELECT id\nFROM items\nORDER BY\n category NULLS LAST,\n created_at DESC NULLS FIRST", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "explicit cast type spellings remain authored", - sql: "select cast(label as TEXT) as label, cast(id as INTEGER) as id, cast(observed_at as TIMESTAMP_NTZ) as observed_at, cast(amount as NUMERIC(10, 2)) as amount from items", - expected_sql: "SELECT\n CAST(label AS TEXT) AS label,\n CAST(id AS INTEGER) AS id,\n CAST(observed_at AS TIMESTAMP_NTZ) AS observed_at,\n CAST(amount AS NUMERIC(10, 2)) AS amount\nFROM items", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "nested and try cast type spellings remain authored", - sql: "select cast(coalesce(try_cast(raw_id as INTEGER), 0) as NUMERIC(10, 2)) as amount, cast(tags as ARRAY) as tags from items", - expected_sql: "SELECT\n CAST(COALESCE(TRY_CAST(raw_id AS INTEGER), 0) AS NUMERIC(10, 2)) AS amount,\n CAST(tags AS ARRAY) AS tags\nFROM items", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "commented cast type remains idempotent", - sql: "select cast(amount as NUMERIC(/* precision */ 10, 2)) as amount from items", - expected_sql: "SELECT\n CAST(amount AS NUMERIC( /* precision */10, 2)) AS amount\nFROM items", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "cast-free token expansion remains supported", - sql: "select id label from items", - expected_sql: "SELECT id label\nFROM items", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "cast-free multiple statements remain supported", - sql: "select 1; select 2", - expected_sql: "SELECT 1;\nSELECT 2", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "trailing comment after a kept statement terminator", - sql: "select a from items; -- retained", - expected_sql: "SELECT a\nFROM items; -- retained", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "unparenthesized values relation remains unparenthesized", - sql: "select column1::varchar as customer_key, column2::integer as order_count from values ('c1', 1), ('c2', 2)", - expected_sql: "SELECT\n column1::varchar AS customer_key,\n column2::integer AS order_count\nFROM VALUES ('c1', 1), ('c2', 2)", - expected_changed: true, - }, - ]; - - for test_case in &test_cases { - let response = format_json( - &json!({"version": 1, "sql": test_case.sql, "dialect": "snowflake"}).to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - assert_eq!( - payload["sql"], test_case.expected_sql, - "{}", - test_case.description - ); - assert_eq!( - payload["changed"], test_case.expected_changed, - "{}", - test_case.description - ); - } - Ok(()) -} - -#[test] -fn given_ordered_sql_batch_when_formatting_then_outputs_are_parallel_and_ordered() --> Result<(), String> { - let test_cases = [ - test_types::FormatTestCase { - description: "first order query", - sql: "select order_id from orders where order_id=1", - expected_sql: "SELECT order_id\nFROM orders\nWHERE order_id = 1", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "second product query", - sql: "select product_id from products where product_id=2", - expected_sql: "SELECT product_id\nFROM products\nWHERE product_id = 2", - expected_changed: true, - }, - ]; - let requests: Vec = test_cases - .iter() - .map(|test_case| json!({"version": 1, "sql": test_case.sql, "dialect": "snowflake"})) - .collect(); - - let response = - format_batch_json(&serde_json::to_string(&requests).map_err(|error| error.to_string())?)?; - let payloads: Vec = - serde_json::from_str(&response).map_err(|error| error.to_string())?; - - assert_eq!(payloads.len(), test_cases.len()); - let mut payload_iter = payloads.iter(); - for test_case in &test_cases { - let payload = payload_iter - .next() - .ok_or_else(|| format!("{}: batch response omitted an item", test_case.description))?; - assert_eq!( - payload["sql"], test_case.expected_sql, - "{}", - test_case.description - ); - assert_eq!(payload["changed"], test_case.expected_changed); - } - Ok(()) -} - -#[test] -fn given_unknown_rule_when_linting_then_request_is_rejected() -> Result<(), String> { - let test_cases = [test_types::InvalidRuleTestCase { - description: "unknown native lint rule", - rule: "SQBRSQL999", - expected_message: "unknown native lint rule 'SQBRSQL999'", - }]; - - for test_case in &test_cases { - let request = json!({"version": 1, "sql": "SELECT 1", "dialect": "snowflake", "enabled_rules": [test_case.rule]}).to_string(); - let error = lint_json(&request) - .err() - .ok_or_else(|| "unknown rule should fail".to_string())?; - assert!( - error.contains(test_case.expected_message), - "{}", - test_case.description - ); - } - Ok(()) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs deleted file mode 100644 index 6a3348bde3..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs +++ /dev/null @@ -1,182 +0,0 @@ -pub(crate) struct LintTestCase { - pub description: &'static str, - pub sql: &'static str, - pub expected_codes: &'static [&'static str], - pub expected_anchors: &'static [(&'static str, &'static str)], -} - -pub(crate) struct EmptyFixtureLintTestCase { - pub(crate) description: &'static str, - pub(crate) sql: &'static str, - pub(crate) allows_empty_fixture_star: bool, - pub(crate) expected_diagnostic_count: usize, -} - -pub(crate) struct BatchLintTestCase { - pub(crate) description: &'static str, - pub(crate) queries: &'static [&'static str], - pub(crate) expected_responses: usize, -} - -pub(crate) struct LintMetadataTestCase { - pub description: &'static str, - pub sql: &'static str, - pub expected_code: &'static str, - pub expected_message: &'static str, - pub expected_remediation: &'static str, -} - -pub(crate) struct LintFixTestCase { - pub description: &'static str, - pub sql: &'static str, - pub expected_code: &'static str, - pub expected_replacement: Option<&'static str>, -} - -pub(crate) struct AdditionalLintRuleTestCase { - pub description: &'static str, - pub sql: &'static str, - pub rule: &'static str, - pub expected_anchor: Option<&'static str>, - pub expected_replacement: Option<&'static str>, -} - -pub(crate) struct CeremonialCteLintTestCase { - pub description: &'static str, - pub sql: &'static str, - pub expected_count: usize, - pub expected_start: Option, -} - -pub(crate) struct PlainSqlLintTestCase { - pub description: &'static str, - pub sql: &'static str, - pub rule: &'static str, - pub expected_count: usize, -} - -pub(crate) struct SqlMacroOutputLintTestCase { - pub(crate) description: &'static str, - pub(crate) sql: &'static str, - /// The text a `@sql.` call generated, and the names the call passed. - pub(crate) output: &'static str, - pub(crate) relations: &'static [&'static str], - pub(crate) expected_count: usize, -} - -pub(crate) struct DynamicOutputStarLintTestCase { - pub(crate) description: &'static str, - pub(crate) sql: &'static str, - pub(crate) expected_count: usize, - pub(crate) expected_prefix: &'static str, -} - -pub(crate) struct DependencyImportLintTestCase { - pub(crate) description: &'static str, - pub(crate) sql: &'static str, - pub(crate) dependency_identifiers: &'static [&'static str], - pub(crate) expected_count: usize, -} - -pub(crate) struct DialectLintRuleTestCase { - pub description: &'static str, - pub sql: &'static str, - pub dialect: &'static str, - pub rule: &'static str, - pub expected_count: usize, - pub expected_fix: bool, - pub expected_reason: &'static str, -} - -pub(crate) struct LineWrapTestCase { - pub description: &'static str, - pub sql: &'static str, - pub line_width: usize, - pub token_widths: &'static [(&'static str, usize)], - pub expected_sql: &'static str, -} - -pub(crate) struct FunctionCallSpacingTestCase { - pub description: &'static str, - pub dialect: &'static str, - pub line_width: usize, - pub argument: &'static str, - pub expected_refused: &'static [&'static str], -} - -pub(crate) struct LayoutRuleTestCase { - pub description: &'static str, - pub dialect: &'static str, - pub sql: &'static str, - pub line_width: usize, - pub expected_sql: &'static str, -} - -pub(crate) struct FormatCorpusTestCase { - pub description: &'static str, - pub dialect: &'static str, - pub expected_formatted: usize, - pub expected_refused: usize, -} - -pub(crate) struct DialectFormatTestCase { - pub description: &'static str, - pub dialect: &'static str, - pub sql: &'static str, - pub expected_sql: &'static str, -} - -pub(crate) struct TokenInvariantTestCase { - pub description: &'static str, - pub before: &'static str, - pub after: &'static str, - pub recases: &'static [(usize, &'static str, bool)], - pub expected_error: Option<&'static str>, -} - -pub(crate) struct FormatTestCase { - pub description: &'static str, - pub sql: &'static str, - pub expected_sql: &'static str, - pub expected_changed: bool, -} - -pub(crate) struct ReprintTestCase { - pub description: &'static str, - pub sql: &'static str, - pub expected_changed: bool, - pub expected_reprints: usize, -} - -pub(crate) struct InvalidRuleTestCase { - pub description: &'static str, - pub rule: &'static str, - pub expected_message: &'static str, -} - -pub(crate) struct FunctionDepthTestCase { - pub description: &'static str, - pub depth: usize, - pub expected_diagnostic_count: usize, -} - -pub(crate) struct FunctionDepthFailureTestCase { - pub description: &'static str, - pub depth: usize, - pub expected_code: &'static str, - pub expected_limit: &'static str, -} - -pub(crate) struct CrossJoinTestCase { - pub description: &'static str, - pub sql: &'static str, - pub dialect: &'static str, - pub one_row_relations: &'static [&'static str], - pub expected_messages: &'static [&'static str], -} - -pub(crate) struct CrossJoinSubjectTestCase { - pub description: &'static str, - pub message: &'static str, - pub expected_subject: Option<(&'static str, Option<&'static str>)>, -} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_unused_table_alias.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_unused_table_alias.rs deleted file mode 100644 index 940b79c4f1..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_unused_table_alias.rs +++ /dev/null @@ -1,317 +0,0 @@ -use crate::sql_lint::tests::{helpers, test_types}; - -#[test] -fn given_implicit_and_explicit_table_aliases_when_linting_then_unused_aliases_match() --> Result<(), String> { - let test_cases = [ - test_types::AdditionalLintRuleTestCase { - description: "unused implicit table alias", - sql: "SELECT id FROM items unused", - rule: "SQBRSQL023", - expected_anchor: Some("unused"), - expected_replacement: Some(""), - }, - test_types::AdditionalLintRuleTestCase { - description: "unused implicit joined table alias", - sql: "SELECT o.id FROM orders o INNER JOIN customers unused ON o.customer_id = o.id", - rule: "SQBRSQL023", - expected_anchor: Some("unused"), - expected_replacement: Some(""), - }, - test_types::AdditionalLintRuleTestCase { - description: "unused implicit qualified table alias", - sql: "SELECT id FROM warehouse.analytics.items unused", - rule: "SQBRSQL023", - expected_anchor: Some("unused"), - expected_replacement: Some(""), - }, - test_types::AdditionalLintRuleTestCase { - description: "unused implicit subquery alias", - sql: "SELECT id FROM (SELECT id FROM items) unused", - rule: "SQBRSQL023", - expected_anchor: Some("unused"), - expected_replacement: Some(""), - }, - test_types::AdditionalLintRuleTestCase { - description: "unused implicit lateral table function alias", - sql: "SELECT src.id FROM items src, LATERAL FLATTEN(input => src.payload) unused", - rule: "SQBRSQL023", - expected_anchor: Some("unused"), - expected_replacement: Some(""), - }, - test_types::AdditionalLintRuleTestCase { - description: "used implicit table alias", - sql: "SELECT kept.id FROM items kept", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "used implicit joined table aliases", - sql: "SELECT o.id, c.name FROM orders o INNER JOIN customers c ON o.customer_id = c.id", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "used implicit qualified table alias", - sql: "SELECT kept.id FROM warehouse.analytics.items kept", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "used implicit subquery alias", - sql: "SELECT sub.id FROM (SELECT id FROM items) sub", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "used implicit lateral table function alias", - sql: "SELECT flattened.value FROM items src, LATERAL FLATTEN(input => src.payload) flattened", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "implicit alias used by correlated subquery", - sql: "SELECT id FROM items kept WHERE EXISTS (SELECT 1 FROM other WHERE other.id = kept.id)", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "implicit alias carrying a column list is retained", - sql: "SELECT a FROM items t(a)", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "qualified relation name is not an implicit alias", - sql: "SELECT id FROM warehouse.analytics.items", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "join modifier keywords are not implicit aliases", - sql: "SELECT o.id, c.name FROM orders o POSITIONAL JOIN customers c", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "join condition identifiers are not implicit aliases", - sql: "SELECT o.id, c.name FROM orders o INNER JOIN customers c USING (customer_id)", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - ]; - - for test_case in test_cases { - let diagnostics = helpers::diagnostics_for_rules(test_case.sql, &[test_case.rule])?; - assert_eq!( - diagnostics.len(), - usize::from(test_case.expected_anchor.is_some()), - "{}", - test_case.description - ); - let _ = test_case.expected_anchor.map(|expected_anchor| { - let diagnostic = &diagnostics[0]; - let start = diagnostic["start"].as_u64().unwrap_or_default() as usize; - let end = diagnostic["end"].as_u64().unwrap_or_default() as usize; - assert_eq!( - &test_case.sql[start..end], - expected_anchor, - "{}", - test_case.description - ); - assert_eq!( - diagnostic["fix"]["replacement"].as_str(), - test_case.expected_replacement, - "{}", - test_case.description - ); - }); - } - Ok(()) -} - -#[test] -fn given_join_modifiers_and_whole_row_uses_when_linting_duckdb_then_aliases_are_kept() --> Result<(), String> { - let test_cases = [ - test_types::AdditionalLintRuleTestCase { - description: "unaliased relation before positional join has no alias", - sql: "SELECT * FROM orders POSITIONAL JOIN customers", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "unaliased relation before asof join has no alias", - sql: "SELECT * FROM orders ASOF JOIN prices ON orders.item_id = prices.item_id AND orders.ordered_at >= prices.valid_from", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "unaliased relation before natural join has no alias", - sql: "SELECT * FROM orders NATURAL JOIN customers", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "unaliased relation before anti join has no alias", - sql: "SELECT * FROM orders ANTI JOIN refunds USING (order_id)", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "unaliased relation before semi join has no alias", - sql: "SELECT * FROM orders SEMI JOIN refunds USING (order_id)", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "unaliased relation before cross join has no alias", - sql: "SELECT * FROM orders CROSS JOIN customers", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "implicit alias repeating the relation name is unused", - sql: "SELECT id FROM orders orders", - rule: "SQBRSQL023", - expected_anchor: Some("orders"), - expected_replacement: Some(""), - }, - test_types::AdditionalLintRuleTestCase { - description: "explicit alias repeating the relation name is unused", - sql: "SELECT id FROM orders AS orders", - rule: "SQBRSQL023", - expected_anchor: Some("AS orders"), - expected_replacement: Some(""), - }, - test_types::AdditionalLintRuleTestCase { - description: "alias repeating the relation name is used when referenced", - sql: "SELECT orders.id FROM orders orders", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "unaliased relation before window clause has no alias", - sql: "SELECT SUM(amount) OVER w FROM orders WINDOW w AS (ORDER BY id)", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "unaliased relation before tablesample has no alias", - sql: "SELECT * FROM orders TABLESAMPLE BERNOULLI (10)", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "unaliased relation before unpivot has no alias", - sql: "SELECT * FROM orders UNPIVOT (amount FOR month IN (jan, feb))", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "implicit alias used as whole-row function argument", - sql: "SELECT row_to_json(o) AS payload FROM orders o", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "explicit alias used as whole-row function argument", - sql: "SELECT row_to_json(o) AS payload FROM orders AS o", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "implicit alias used as bare projection", - sql: "SELECT o FROM orders o", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "explicit alias used as bare projection", - sql: "SELECT o FROM orders AS o", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "implicit alias used in whole-row comparison", - sql: "SELECT 1 AS present FROM orders o WHERE o IS NOT NULL", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "explicit alias used in whole-row comparison", - sql: "SELECT 1 AS present FROM orders AS o WHERE o IS NOT NULL", - rule: "SQBRSQL023", - expected_anchor: None, - expected_replacement: None, - }, - test_types::AdditionalLintRuleTestCase { - description: "unused implicit alias beside an unrelated qualified column is reported", - sql: "SELECT c.o FROM customers c CROSS JOIN orders o", - rule: "SQBRSQL023", - expected_anchor: Some("o"), - expected_replacement: Some(""), - }, - test_types::AdditionalLintRuleTestCase { - description: "unused explicit alias shadowed by an output alias is reported", - sql: "SELECT order_id AS o FROM orders AS o", - rule: "SQBRSQL023", - expected_anchor: Some("AS o"), - expected_replacement: Some(""), - }, - ]; - - for test_case in test_cases { - let diagnostics = - helpers::diagnostics_for_dialect(test_case.sql, "duckdb", &[test_case.rule])?; - assert_eq!( - diagnostics.len(), - usize::from(test_case.expected_anchor.is_some()), - "{}", - test_case.description - ); - let _ = test_case.expected_anchor.map(|expected_anchor| { - let diagnostic = &diagnostics[0]; - let start = diagnostic["start"].as_u64().unwrap_or_default() as usize; - let end = diagnostic["end"].as_u64().unwrap_or_default() as usize; - assert_eq!( - &test_case.sql[start..end], - expected_anchor, - "{}", - test_case.description - ); - assert_eq!( - diagnostic["fix"]["replacement"].as_str(), - test_case.expected_replacement, - "{}", - test_case.description - ); - }); - } - Ok(()) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_values_formatter.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_values_formatter.rs deleted file mode 100644 index 6963aadf4b..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_values_formatter.rs +++ /dev/null @@ -1,53 +0,0 @@ -use serde_json::{Value, json}; - -use crate::sql_lint::main::formatter::format_json; -use crate::sql_lint::tests::test_types; - -#[test] -fn given_mixed_values_relations_when_formatting_then_each_authored_form_is_preserved() --> Result<(), String> { - let test_cases = [ - test_types::FormatTestCase { - description: "mixed values relations preserve parenthesized relation first", - sql: "SELECT * FROM (VALUES (1, 2)) AS first_values(a, b) UNION ALL SELECT * FROM VALUES (3, 4)", - expected_sql: "SELECT *\nFROM (VALUES (1, 2)) AS first_values(a, b)\nUNION ALL\nSELECT *\nFROM VALUES (3, 4)", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "mixed values relations preserve unparenthesized relation first", - sql: "SELECT * FROM VALUES (1, 2) UNION ALL SELECT * FROM (VALUES (3, 4)) AS second_values(a, b)", - expected_sql: "SELECT *\nFROM VALUES (1, 2)\nUNION ALL\nSELECT *\nFROM (VALUES (3, 4)) AS second_values(a, b)", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "mixed values relations inside joined subquery preserve authored forms", - sql: "SELECT base.id FROM items AS base JOIN (SELECT * FROM (VALUES (1)) AS first_values(id) UNION ALL SELECT * FROM VALUES (2)) AS mixed ON base.id = mixed.id", - expected_sql: "SELECT base.id\nFROM items AS base\nJOIN (\n SELECT *\n FROM (VALUES (1)) AS first_values(id)\n UNION ALL\n SELECT *\n FROM VALUES (2)\n) AS mixed\n ON base.id = mixed.id", - expected_changed: true, - }, - test_types::FormatTestCase { - description: "values text in string and comment is ignored", - sql: "-- VALUES and FROM VALUES are examples\nSELECT 'VALUES' AS label FROM (VALUES (1)) AS first_values(id) UNION ALL SELECT * FROM VALUES (2)", - expected_sql: "-- VALUES and FROM VALUES are examples\nSELECT 'VALUES' AS label\nFROM (VALUES (1)) AS first_values(id)\nUNION ALL\nSELECT *\nFROM VALUES (2)", - expected_changed: true, - }, - ]; - - for test_case in &test_cases { - let response = format_json( - &json!({"version": 1, "sql": test_case.sql, "dialect": "snowflake"}).to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - assert_eq!( - payload["sql"], test_case.expected_sql, - "{}", - test_case.description - ); - assert_eq!( - payload["changed"], test_case.expected_changed, - "{}", - test_case.description - ); - } - Ok(()) -} diff --git a/crates/sqlbuild-rules/src/sql_lint/types.rs b/crates/sqlbuild-rules/src/sql_lint/types.rs deleted file mode 100644 index ef7601e9a1..0000000000 --- a/crates/sqlbuild-rules/src/sql_lint/types.rs +++ /dev/null @@ -1,4 +0,0 @@ -use crate::lint_text::models::InterpolationSite; - -/// Lint text with sentinels, its interpolation sites and the CTEs interpolation references. -pub type PreparedSql = (String, Vec, Vec); diff --git a/crates/sqlbuild-rules/src/sql_quality/_helpers/cte_columns.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/cte_columns.rs deleted file mode 100644 index c029f0779c..0000000000 --- a/crates/sqlbuild-rules/src/sql_quality/_helpers/cte_columns.rs +++ /dev/null @@ -1,457 +0,0 @@ -//! SQBRSQL042: output columns of non-final CTEs that no later step reads. - -use std::borrow::Cow; - -use polyglot_sql::expressions::{Cte, JoinKind, Select}; -use polyglot_sql::scope::walk_in_scope; -use polyglot_sql::{Expression, ExpressionWalk}; - -use crate::sql_lint::models::{LintDiagnostic, LintEdit}; -use crate::sql_quality::constants::{ - DISTINCT_UNUSED_REMEDIATION, GROUP_BY_ALL_UNUSED_REMEDIATION, GROUP_KEY_UNUSED_REMEDIATION, - INTERPOLATION_SENTINEL_PREFIXES, UNUSED_CTE_OUTPUT, -}; -use crate::sql_quality::models::QualityRequest; -use crate::sql_quality::removal_safety::{expression_refusal, reads_whole_rows, select_refusal}; -use crate::sql_quality::syntax::{ - Projection, columns, cte_select_tokens, lower, projection_removal, projections, qualifier, - sources, -}; -use rustc_hash::{FxHashMap, FxHashSet}; -use sqlbuild_analysis::sql_tokens::main::projection_spans::projection_spans; - -const HARNESS_PREFIX: &str = "__"; -const COLUMNS_FUNCTION: &str = "COLUMNS"; -const LAYOUT_FIX_REFUSAL: &str = "The projection list could not be mapped to authored tokens"; - -/// What one later step reads from a CTE. -#[derive(Default)] -struct Usage { - names: FxHashSet, - everything: bool, - passes_to: Vec, - read: bool, -} - -impl Usage { - fn merged(mut self, other: Self) -> Self { - self.names.extend(other.names); - self.everything |= other.everything; - self.passes_to.extend(other.passes_to); - self.read |= other.read; - self - } -} - -/// One CTE whose unused outputs are being reported. -struct CteOutputs<'a> { - cte: &'a Cte, - name: &'a str, - usage: Option<&'a Usage>, -} - -pub(crate) fn diagnostics(request: &QualityRequest<'_>) -> Vec { - if request.fixture_body { - return Vec::new(); - } - let mut found: Vec = Vec::new(); - for statement in request.statements { - let Expression::Select(root) = statement else { - continue; - }; - let Some(with) = root.with.as_ref().filter(|with| !with.recursive) else { - continue; - }; - let mut terminal: Select = root.as_ref().clone(); - terminal.with = None; - let terminal: Expression = Expression::Select(Box::new(terminal)); - let names: Vec = with.ctes.iter().map(|cte| lower(&cte.alias.name)).collect(); - let used: FxHashMap = reads(&with.ctes, &names, &terminal); - let select_tokens: FxHashMap = cte_select_tokens(request.tokens); - for (index, cte) in with - .ctes - .iter() - .enumerate() - .take(with.ctes.len().saturating_sub(1)) - { - let name: &str = &names[index]; - if name.starts_with(HARNESS_PREFIX) || request.externally_referenced_ctes.contains(name) - { - continue; - } - found.extend(unused_outputs( - request, - &select_tokens, - CteOutputs { - cte, - name, - usage: used.get(name), - }, - )); - } - } - found -} - -/// Names each CTE's later readers use, following `SELECT *` pass-through to the reader. -fn reads(ctes: &[Cte], names: &[String], terminal: &Expression) -> FxHashMap { - let bodies: Vec<(Option, &Expression)> = ctes - .iter() - .map(|cte| (Some(lower(&cte.alias.name)), &cte.this)) - .chain(std::iter::once((None, terminal))) - .collect(); - let known: FxHashSet<&str> = names.iter().map(String::as_str).collect(); - let outputs: Vec>> = ctes.iter().map(output_names).collect(); - let mut readers: FxHashMap> = FxHashMap::default(); - for (index, (_, body)) in bodies.iter().enumerate() { - for name in read_relations(body) { - if known.contains(name.as_str()) { - readers.entry(name).or_default().push(index); - } - } - } - let mut usage: FxHashMap = FxHashMap::default(); - for (index, name) in names.iter().enumerate() { - let mut current: Usage = Usage::default(); - if index + 1 == names.len() { - current.everything = true; - } - for &reader in readers.get(name.as_str()).into_iter().flatten() { - if reader > index { - let (reader_name, body) = &bodies[reader]; - let target: Target<'_> = Target { - name, - outputs: outputs[index].as_ref(), - }; - current = current.merged(reader_usage(&target, reader_name.clone(), body)); - } - } - current.names.extend(self_reads(&ctes[index].this)); - usage.insert(name.clone(), current); - } - for index in (0..names.len()).rev() { - let passes: Vec = usage[&names[index]].passes_to.clone(); - for reader in passes { - let (everything, inherited): (bool, Vec) = - usage.get(&reader).map_or((true, Vec::new()), |next| { - (next.everything, next.names.iter().cloned().collect()) - }); - if let Some(current) = usage.get_mut(&names[index]) { - current.everything |= everything; - current.names.extend(inherited); - } - } - } - usage -} - -/// The output names of a CTE, or None when a star or unnamed projection hides them. -fn output_names(cte: &Cte) -> Option> { - let Expression::Select(select) = &cte.this else { - return None; - }; - projections(select) - .into_iter() - .map(|item| match item { - Projection::Named { name, .. } => Some(name), - _ => None, - }) - .collect() -} - -fn read_relations(body: &Expression) -> FxHashSet { - body.dfs() - .filter_map(|node| match node { - Expression::Table(table) => Some(lower(&table.name.name)), - _ => None, - }) - .collect() -} - -/// The CTE whose readers are being inspected. -struct Target<'a> { - name: &'a str, - outputs: Option<&'a FxHashSet>, -} - -fn reader_usage(target: &Target<'_>, reader: Option, body: &Expression) -> Usage { - let name: &str = target.name; - let mut usage: Usage = Usage::default(); - let mut aliases: FxHashSet = FxHashSet::default(); - let mut top_level_star: bool = false; - for node in body.dfs() { - let Expression::Select(select) = node else { - continue; - }; - let local: Vec = sources(select) - .into_iter() - .filter(|(_, relation, _)| relation.as_deref() == Some(name)) - .map(|(alias, _, _)| alias) - .collect(); - if local.is_empty() { - continue; - } - if select.joins.iter().any(|join| { - matches!( - join.kind, - JoinKind::Natural - | JoinKind::NaturalLeft - | JoinKind::NaturalRight - | JoinKind::NaturalFull - ) - }) { - usage.everything = true; - } - for join in &select.joins { - usage - .names - .extend(join.using.iter().map(|identifier| lower(&identifier.name))); - } - let top: bool = std::ptr::eq(node, body); - if reads_as_row(select, &local, target.outputs) { - usage.everything = true; - } - for projection in projections(select) { - if let Projection::Star(star) = projection - && star - .table - .as_ref() - .is_none_or(|table| local.contains(&lower(&table.name))) - { - if top - && reader.is_some() - && star.except.is_none() - && star.replace.is_none() - && !reads_whole_rows(select) - { - top_level_star = true; - } else { - usage.everything = true; - } - } - } - aliases.extend(local); - } - if aliases.is_empty() { - return usage; - } - usage.read = true; - usage.everything |= columns(body) - .into_iter() - .any(|column| is_interpolation(&column.name.name)); - if top_level_star && let Some(reader) = reader { - usage.passes_to.push(reader); - } - for column in body.dfs().filter_map(|node| match node { - Expression::Column(column) => Some(column), - _ => None, - }) { - if qualifier(column).is_none_or(|table| aliases.contains(&table)) { - usage.names.insert(lower(&column.name.name)); - } - } - usage -} - -/// Whether a reader consumes whole rows: a nested star, `COLUMNS(...)`, or the alias itself. -fn reads_as_row(select: &Select, local: &[String], outputs: Option<&FxHashSet>) -> bool { - for clause in clause_roots(select) { - for node in walk_in_scope(clause, false) { - if reads_row_at(select, node, local, outputs) { - return true; - } - } - } - false -} - -fn reads_row_at( - select: &Select, - node: &Expression, - local: &[String], - outputs: Option<&FxHashSet>, -) -> bool { - match node { - Expression::Star(star) => { - let projected: bool = select - .expressions - .iter() - .any(|item| std::ptr::eq(item, node)); - let table: Option = star.table.as_ref().map(|table| lower(&table.name)); - !projected && table.is_none_or(|table| local.contains(&table)) - } - Expression::BracedWildcard(_) => true, - Expression::Function(function) => function.name.eq_ignore_ascii_case(COLUMNS_FUNCTION), - Expression::Column(column) => { - let read: String = lower(&column.name.name); - let named_output: bool = outputs.is_some_and(|names| names.contains(&read)); - column.table.is_none() && local.contains(&read) && !named_output - } - _ => false, - } -} - -/// Root expressions of every clause of one select, without its FROM relations. -fn clause_roots(select: &Select) -> Vec<&Expression> { - let mut roots: Vec<&Expression> = select.expressions.iter().collect(); - roots.extend(select.where_clause.iter().map(|clause| &clause.this)); - roots.extend(select.having.iter().map(|clause| &clause.this)); - roots.extend(select.qualify.iter().map(|clause| &clause.this)); - if let Some(group) = &select.group_by { - roots.extend(group.expressions.iter()); - } - if let Some(order) = &select.order_by { - roots.extend(order.expressions.iter().map(|ordered| &ordered.this)); - } - roots.extend(select.joins.iter().filter_map(|join| join.on.as_ref())); - roots -} - -/// Lateral alias reuse: a later projection or a clause of the CTE reading an earlier output. -fn self_reads(body: &Expression) -> FxHashSet { - let mut reads: FxHashSet = FxHashSet::default(); - let Expression::Select(select) = body else { - return reads; - }; - let mut earlier: FxHashSet = FxHashSet::default(); - for item in projections(select) { - if let Projection::Named { name, expression } = item { - for read in unqualified_reads(expression) { - if earlier.contains(&read) && read != name { - reads.insert(read); - } - } - earlier.insert(name); - } - } - let mut clauses: Vec<&Expression> = Vec::new(); - clauses.extend(select.where_clause.iter().map(|clause| &clause.this)); - clauses.extend( - select - .group_by - .iter() - .flat_map(|group| group.expressions.iter()), - ); - clauses.extend(select.having.iter().map(|clause| &clause.this)); - clauses.extend(select.qualify.iter().map(|clause| &clause.this)); - if let Some(order) = &select.order_by { - clauses.extend(order.expressions.iter().map(|ordered| &ordered.this)); - } - for clause in clauses { - reads.extend( - unqualified_reads(clause) - .into_iter() - .filter(|read| earlier.contains(read)), - ); - } - reads -} - -fn unqualified_reads(expression: &Expression) -> Vec { - columns(expression) - .into_iter() - .filter(|column| qualifier(column).is_none()) - .map(|column| lower(&column.name.name)) - .collect() -} - -fn unused_outputs( - request: &QualityRequest<'_>, - select_tokens: &FxHashMap, - outputs: CteOutputs<'_>, -) -> Vec { - let CteOutputs { cte, name, usage } = outputs; - let Expression::Select(select) = &cte.this else { - return Vec::new(); - }; - let Some(usage) = usage.filter(|usage| !usage.everything && usage.read) else { - return Vec::new(); - }; - if !cte.columns.is_empty() { - return Vec::new(); - } - let output: Vec> = projections(select); - if !output - .iter() - .all(|item| matches!(item, Projection::Named { .. })) - { - return Vec::new(); - } - let spans: Vec<(usize, usize)> = select_tokens - .get(name) - .map(|&select_token| projection_spans(request.tokens, select_token)) - .unwrap_or_default(); - let grouped: FxHashSet = select - .group_by - .iter() - .flat_map(|group| group.expressions.iter()) - .flat_map(columns) - .map(|column| lower(&column.name.name)) - .collect(); - let select_refused: Option<&'static str> = select_refusal(select, output.len()); - let mut found: Vec = Vec::new(); - for (index, item) in output.iter().enumerate() { - let Projection::Named { - name: column, - expression, - } = item - else { - continue; - }; - if usage.names.contains(column) || is_interpolation(column) { - continue; - } - let grouping_key: bool = matches!(expression, Expression::Column(source) if grouped.contains(&lower(&source.name.name))); - let (start, end) = spans - .get(index) - .copied() - .filter(|_| spans.len() == output.len()) - .unwrap_or((0, 0)); - let refusal: Option<&'static str> = select_refused - .or_else(|| expression_refusal(expression)) - .or_else(|| (spans.len() != output.len()).then_some(LAYOUT_FIX_REFUSAL)); - let fix: Option = refusal - .is_none() - .then(|| projection_removal(&spans, index)) - .flatten() - .map(|(start, end)| LintEdit { - start, - end, - replacement: String::new(), - }); - found.push(LintDiagnostic { - code: UNUSED_CTE_OUTPUT.code, - message: Cow::Borrowed(UNUSED_CTE_OUTPUT.message), - remediation: Cow::Borrowed(if select.distinct { - DISTINCT_UNUSED_REMEDIATION - } else if select - .group_by - .as_ref() - .is_some_and(|group| group.all == Some(true)) - { - GROUP_BY_ALL_UNUSED_REMEDIATION - } else if grouping_key { - GROUP_KEY_UNUSED_REMEDIATION - } else { - UNUSED_CTE_OUTPUT.remediation - }), - start, - end, - fix_unavailable_reason: if fix.is_none() { - Some(refusal.unwrap_or(LAYOUT_FIX_REFUSAL)) - } else { - None - }, - fix, - }); - } - found -} - -/// Neutralized `@macro` / `@variable` interpolations can stand for any columns. -fn is_interpolation(name: &str) -> bool { - let name: String = lower(name); - INTERPOLATION_SENTINEL_PREFIXES - .iter() - .any(|prefix| name.starts_with(prefix)) -} diff --git a/crates/sqlbuild-rules/src/sql_quality/_helpers/json_parses.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/json_parses.rs deleted file mode 100644 index 6e95858613..0000000000 --- a/crates/sqlbuild-rules/src/sql_quality/_helpers/json_parses.rs +++ /dev/null @@ -1,201 +0,0 @@ -//! SQBRSQL045: the same JSON payload column parsed more than once in one SELECT scope. - -use std::borrow::Cow; -use std::collections::HashMap; - -use polyglot_sql::expressions::{Column, Identifier}; -use polyglot_sql::scope::walk_in_scope; -use polyglot_sql::tokens::Token; -use polyglot_sql::{DialectType, Expression, ExpressionWalk}; - -use crate::sql_lint::models::LintDiagnostic; -use crate::sql_quality::constants::REPEATED_JSON_PARSE; -use crate::sql_quality::models::QualityRequest; -use crate::sql_quality::syntax::{is_trivia, lower}; - -/// Significant tokens searched backwards from the column for the function name. -const MAX_FUNCTION_NAME_DISTANCE: usize = 32; -const PARSE_ONCE_FIX_REFUSAL: &str = - "Parsing the payload once requires naming a column in the CTE that reads it"; - -/// An identifier normalised the way the dialect resolves it. -type IdentifierKey = String; - -/// One JSON parse call whose argument resolves to a plain column. -struct ParseCall<'a> { - function: &'static str, - column: &'a Column, - column_start: usize, -} - -pub(crate) fn diagnostics(request: &QualityRequest<'_>) -> Vec { - let functions: &[&'static str] = parse_functions(request.dialect); - if functions.is_empty() { - return Vec::new(); - } - let mut found: Vec = Vec::new(); - for statement in request.statements { - for node in statement.dfs() { - if matches!(node, Expression::Select(_)) { - found.extend(scope_diagnostics( - request.tokens, - node, - functions, - request.dialect, - )); - } - } - } - found -} - -/// JSON parse functions that take one string argument, by dialect, in upper case. -fn parse_functions(dialect: DialectType) -> &'static [&'static str] { - match dialect { - DialectType::Snowflake | DialectType::Databricks => &["PARSE_JSON", "TRY_PARSE_JSON"], - DialectType::BigQuery => &["PARSE_JSON", "SAFE.PARSE_JSON"], - DialectType::DuckDB => &["JSON"], - _ => &[], - } -} - -fn scope_diagnostics( - tokens: &[Token], - select: &Expression, - functions: &[&'static str], - dialect: DialectType, -) -> Vec { - let quote: char = identifier_quote(dialect); - let mut calls: Vec> = walk_in_scope(select, false) - .filter_map(|node| parse_call(node, functions)) - .collect(); - calls.sort_by_key(|call| call.column_start); - let mut groups: HashMap< - (&'static str, Option, IdentifierKey), - Vec<&ParseCall<'_>>, - > = HashMap::new(); - for call in &calls { - let qualifier: Option = call - .column - .table - .as_ref() - .map(|table| identifier_key(table, dialect)); - groups - .entry(( - call.function, - qualifier, - identifier_key(&call.column.name, dialect), - )) - .or_default() - .push(call); - } - let mut found: Vec = Vec::new(); - for group in groups.values().filter(|group| group.len() > 1) { - let second: &ParseCall<'_> = group[1]; - let (start, end) = function_span(tokens, second); - found.push(LintDiagnostic { - code: REPEATED_JSON_PARSE.code, - message: Cow::Borrowed(REPEATED_JSON_PARSE.message), - remediation: Cow::Owned(remediation(second, group.len(), quote)), - start, - end, - fix: None, - fix_unavailable_reason: Some(PARSE_ONCE_FIX_REFUSAL), - }); - } - found -} - -/// Normalises a name the way the dialect resolves it (Snowflake folds unquoted names upward). -fn identifier_key(identifier: &Identifier, dialect: DialectType) -> IdentifierKey { - match dialect { - DialectType::Snowflake if identifier.quoted => identifier.name.clone(), - DialectType::Snowflake => identifier.name.to_uppercase(), - _ => lower(&identifier.name), - } -} - -fn parse_call<'a>(node: &'a Expression, functions: &[&'static str]) -> Option> { - let (name, argument): (&str, &Expression) = match node { - Expression::ParseJson(function) => ("PARSE_JSON", &function.this), - Expression::Function(function) if function.args.len() == 1 => { - (function.name.as_str(), &function.args[0]) - } - _ => return None, - }; - let function: &'static str = functions - .iter() - .copied() - .find(|candidate| candidate.eq_ignore_ascii_case(name))?; - let column: &Column = unwrap_column(argument)?; - let column_start: usize = column.span.or(column.name.span)?.start; - Some(ParseCall { - function, - column, - column_start, - }) -} - -/// The column under wrappers that keep the parsed value tied to that column. -fn unwrap_column(expression: &Expression) -> Option<&Column> { - let mut current: &Expression = expression; - loop { - current = match current { - Expression::Column(column) => return Some(column), - Expression::Paren(paren) => &paren.this, - Expression::Lower(function) | Expression::Upper(function) => &function.this, - Expression::Trim(trim) if trim.characters.is_none() => &trim.this, - Expression::Cast(cast) | Expression::TryCast(cast) | Expression::SafeCast(cast) => { - &cast.this - } - Expression::Coalesce(coalesce) if coalesce.expressions.len() == 1 => { - &coalesce.expressions[0] - } - _ => return None, - }; - } -} - -/// Span of the function-name token that opens `call`, or of its column when not found. -fn function_span(tokens: &[Token], call: &ParseCall<'_>) -> (usize, usize) { - let bare: &str = call.function.rsplit('.').next().unwrap_or(call.function); - let before: usize = tokens.partition_point(|token| token.span.start < call.column_start); - tokens[..before] - .iter() - .rev() - .filter(|token| !is_trivia(token)) - .take(MAX_FUNCTION_NAME_DISTANCE) - .find(|token| token.text.eq_ignore_ascii_case(bare)) - .map_or((call.column_start, call.column_start), |token| { - (token.span.start, token.span.end) - }) -} - -fn remediation(call: &ParseCall<'_>, count: usize, quote: char) -> String { - let name: String = written(&call.column.name, quote); - let column: String = call.column.table.as_ref().map_or_else( - || name.clone(), - |table| format!("{}.{name}", written(table, quote)), - ); - let function: &str = call.function; - format!( - "{function}({column}) is parsed {count} times in one SELECT. Parse it once in the CTE that reads it, for example `{function}({name}) AS payload`, and read every field from that column in a later step." - ) -} - -/// The identifier as authored, re-quoted with the dialect's identifier quote when it was quoted. -fn written(identifier: &Identifier, quote: char) -> String { - if identifier.quoted { - let escaped: String = identifier.name.replace(quote, &format!("{quote}{quote}")); - format!("{quote}{escaped}{quote}") - } else { - identifier.name.clone() - } -} - -fn identifier_quote(dialect: DialectType) -> char { - match dialect { - DialectType::BigQuery | DialectType::Databricks => '`', - _ => '"', - } -} diff --git a/crates/sqlbuild-rules/src/sql_quality/_helpers/keys.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/keys.rs deleted file mode 100644 index 2e006e53b1..0000000000 --- a/crates/sqlbuild-rules/src/sql_quality/_helpers/keys.rs +++ /dev/null @@ -1,437 +0,0 @@ -//! Unique-key propagation through one model's relations, for ranking determinism proofs. - -use std::collections::HashMap; - -use polyglot_sql::Expression; -use polyglot_sql::expressions::{JoinKind, Literal, Select}; - -use crate::sql_quality::constants::FIRST_ROW; -use crate::sql_quality::models::QualityRequest; -use crate::sql_quality::syntax::{Projection, columns, lower, projections, qualifier, sources}; -use crate::sql_quality::types::{InputKey, Keys}; - -/// Keys known for relations by name: declared upstream keys and earlier CTE outputs. -pub(crate) struct KeyEnvironment { - pub(crate) declared: HashMap, - pub(crate) ctes: HashMap, - /// Per CTE, `(row number output, partition output columns)` for deduplication filters. - pub(crate) row_numbers: HashMap)>>, -} - -impl KeyEnvironment { - /// Output keys of a query expression, by output column name. - pub(crate) fn query_keys(&self, expression: &Expression) -> Keys { - match expression { - Expression::Select(select) => self.select_keys(select), - Expression::Subquery(subquery) => self.query_keys(&subquery.this), - Expression::Paren(paren) => self.query_keys(&paren.this), - Expression::Union(union) if !union.all => all_columns_key(&union.left), - Expression::Intersect(intersect) => self.query_keys(&intersect.left), - Expression::Except(except) => self.query_keys(&except.left), - _ => Vec::new(), - } - } - - fn relation_keys(&self, relation: &Expression) -> Keys { - match relation { - Expression::Table(table) => { - let name: String = lower(&table.name.name); - self.ctes - .get(&name) - .cloned() - .or_else(|| self.declared.get(&name).cloned()) - .unwrap_or_default() - } - Expression::Subquery(subquery) => self.query_keys(&subquery.this), - _ => Vec::new(), - } - } - - /// Keys of a select's joined and filtered input rows, before projection. - pub(crate) fn input_keys(&self, select: &Select) -> Vec { - if select - .from - .as_ref() - .is_none_or(|from| from.expressions.len() != 1) - { - return Vec::new(); - } - let found: Vec<(String, Option, &Expression)> = sources(select); - let Some((first_alias, _, first)) = found.first() else { - return Vec::new(); - }; - let mut keys: Vec = qualify(first_alias, self.relation_keys(first)); - keys.extend(qualify(first_alias, self.deduplicated_keys(select, first))); - for (join, (alias, _, relation)) in select.joins.iter().zip(found.iter().skip(1)) { - let right: Vec = qualify(alias, self.relation_keys(relation)); - let equated: Vec<(String, String)> = join_equalities(join, alias); - let left_columns: Vec<&str> = equated.iter().map(|(_, left)| left.as_str()).collect(); - let right_columns: Vec<&str> = - equated.iter().map(|(right, _)| right.as_str()).collect(); - let left_unique: bool = keys.iter().any(|key| key_within(key, &left_columns)); - let right_unique: bool = right.iter().any(|key| key_within(key, &right_columns)); - let mut joined: Vec = Vec::new(); - if right_unique - && !matches!( - join.kind, - JoinKind::Right | JoinKind::Full | JoinKind::Cross - ) - { - joined.extend(keys.iter().cloned()); - } - if left_unique && matches!(join.kind, JoinKind::Inner | JoinKind::Right) { - joined.extend(right.iter().cloned()); - } - keys = joined; - } - keys - } - - /// Input keys plus a `QUALIFY ROW_NUMBER() ... = n` partition, which holds only for output rows. - fn filtered_input_keys(&self, select: &Select) -> Vec { - let mut keys: Vec = self.input_keys(select); - if let Some(qualify_clause) = &select.qualify { - keys.extend(qualified_rank_keys(&qualify_clause.this, &sources(select))); - } - keys - } - - /// Partition keys of a `ROW_NUMBER()` output that this select filters to `= 1`. - fn deduplicated_keys(&self, select: &Select, relation: &Expression) -> Keys { - let Expression::Table(table) = relation else { - return Vec::new(); - }; - let Some(ranks) = self.row_numbers.get(&lower(&table.name.name)) else { - return Vec::new(); - }; - let filtered: Vec = select - .where_clause - .as_ref() - .map(|clause| first_row_filters(&clause.this)) - .unwrap_or_default(); - ranks - .iter() - .filter(|(rank, _)| filtered.contains(rank)) - .map(|(_, key)| key.clone()) - .collect() - } - - fn select_keys(&self, select: &Select) -> Keys { - let output: Vec> = projections(select); - let mut keys: Keys = Vec::new(); - if select.distinct - && output - .iter() - .all(|item| matches!(item, Projection::Named { .. })) - { - keys.push(output_names(&output)); - } - if let Some(group_by) = &select.group_by - && group_by.all.is_none() - && let Some(grain) = output_key_of(&group_by.expressions, &output) - { - keys.push(grain); - return keys; - } - let star: bool = output - .iter() - .any(|item| matches!(item, Projection::Star(_))); - for input in self.filtered_input_keys(select) { - if let Some(key) = project_key(&input, &output, star) { - keys.push(key); - } - } - keys - } -} - -fn qualify(alias: &str, keys: Keys) -> Vec { - keys.into_iter() - .map(|key| qualified_key(alias, key)) - .collect() -} - -fn qualified_key(alias: &str, key: Vec) -> InputKey { - key.into_iter() - .map(|column| (alias.to_owned(), column)) - .collect() -} - -fn key_within(key: &InputKey, columns: &[&str]) -> bool { - key.iter() - .all(|(_, column)| columns.contains(&column.as_str())) -} - -/// `(right column, left column)` pairs equated by a join's ON or USING condition. -fn join_equalities( - join: &polyglot_sql::expressions::Join, - right_alias: &str, -) -> Vec<(String, String)> { - let mut pairs: Vec<(String, String)> = join - .using - .iter() - .map(|identifier| (lower(&identifier.name), lower(&identifier.name))) - .collect(); - let mut pending: Vec<&Expression> = join.on.iter().collect(); - while let Some(expression) = pending.pop() { - match expression { - Expression::And(operation) => { - pending.push(&operation.left); - pending.push(&operation.right); - } - Expression::Paren(paren) => pending.push(&paren.this), - Expression::Eq(operation) => { - if let (Expression::Column(left), Expression::Column(right)) = - (&operation.left, &operation.right) - { - let (inner, outer) = if qualifier(left).as_deref() == Some(right_alias) { - (left, right) - } else { - (right, left) - }; - pairs.push((lower(&inner.name.name), lower(&outer.name.name))); - } - } - _ => {} - } - } - pairs -} - -fn qualified_rank_keys( - condition: &Expression, - found: &[(String, Option, &Expression)], -) -> Vec { - let Expression::Eq(operation) = condition else { - return Vec::new(); - }; - let window = match (&operation.left, &operation.right) { - (Expression::WindowFunction(window), Expression::Literal(_)) - | (Expression::Literal(_), Expression::WindowFunction(window)) => window, - _ => return Vec::new(), - }; - if !matches!(window.this, Expression::RowNumber(_)) || window.over.partition_by.is_empty() { - return Vec::new(); - } - let mut key: InputKey = InputKey::new(); - for expression in &window.over.partition_by { - let Expression::Column(column) = expression else { - return Vec::new(); - }; - let alias: String = qualifier(column) - .or_else(|| (found.len() == 1).then(|| found[0].0.clone())) - .unwrap_or_default(); - key.insert((alias, lower(&column.name.name))); - } - vec![key] -} - -fn output_names(output: &[Projection<'_>]) -> Vec { - output - .iter() - .filter_map(|item| match item { - Projection::Named { name, .. } => Some(name.clone()), - _ => None, - }) - .collect() -} - -fn output_key_of(expressions: &[Expression], output: &[Projection<'_>]) -> Option> { - let mut key: Vec = Vec::new(); - for expression in expressions { - if let Expression::Literal(literal) = expression - && let Literal::Number(text) = literal.as_ref() - { - let Ok(ordinal) = text.parse::() else { - return None; - }; - let Some(Projection::Named { name, .. }) = ordinal - .checked_sub(1) - .and_then(|position| output.get(position)) - else { - return None; - }; - key.push(name.clone()); - continue; - } - let Expression::Column(column) = expression else { - return None; - }; - let name: String = lower(&column.name.name); - let projected: Option = output.iter().find_map(|item| match item { - Projection::Named { - name: output_name, - expression: Expression::Column(source), - } if lower(&source.name.name) == name => Some(output_name.clone()), - Projection::Named { - name: output_name, .. - } if *output_name == name => Some(output_name.clone()), - _ => None, - }); - key.push(projected?); - } - Some(key) -} - -fn project_key(input: &InputKey, output: &[Projection<'_>], star: bool) -> Option> { - let mut key: Vec = Vec::new(); - for (alias, column) in input { - let projected: Option = output.iter().find_map(|item| match item { - Projection::Named { - name, - expression: Expression::Column(source), - } if lower(&source.name.name) == *column - && qualifier(source).is_none_or(|table| table == *alias) => - { - Some(name.clone()) - } - Projection::Star(star) - if star - .table - .as_ref() - .is_none_or(|table| lower(&table.name) == *alias) => - { - Some(column.clone()) - } - _ => None, - }); - match projected { - Some(name) => key.push(name), - None if star => key.push(column.clone()), - None => return None, - } - } - Some(key) -} - -fn all_columns_key(left: &Expression) -> Keys { - let Expression::Select(select) = left else { - return Vec::new(); - }; - let output: Vec> = projections(select); - if output - .iter() - .all(|item| matches!(item, Projection::Named { .. })) - { - vec![output_names(&output)] - } else { - Vec::new() - } -} - -/// Columns a ranking window partitions and orders by, generously resolved to input aliases. -pub(crate) fn window_columns( - partition_by: &[Expression], - order_by: &[&Expression], -) -> Vec<(Option, String)> { - partition_by - .iter() - .chain(order_by.iter().copied()) - .flat_map(columns) - .map(|column| (qualifier(column), lower(&column.name.name))) - .collect() -} - -/// Whether some input key is covered by the window's partition and order columns. -pub(crate) fn covers(keys: &[InputKey], window: &[(Option, String)]) -> bool { - keys.iter().any(|key| key_covered(key, window)) -} - -fn key_covered(key: &InputKey, window: &[(Option, String)]) -> bool { - key.iter() - .all(|(alias, column)| window_has(window, alias, column)) -} - -fn window_has(window: &[(Option, String)], alias: &str, column: &str) -> bool { - window.iter().any(|(qualifier, name)| { - name == column && qualifier.as_deref().is_none_or(|table| table == alias) - }) -} - -/// Keys of declared relations and of each top-level CTE, in definition order. -pub(crate) fn key_environment(request: &QualityRequest<'_>) -> KeyEnvironment { - let mut environment: KeyEnvironment = KeyEnvironment { - declared: lowered_keys(request.relation_keys), - ctes: HashMap::new(), - row_numbers: HashMap::new(), - }; - for statement in request.statements { - let Expression::Select(select) = statement else { - continue; - }; - for cte in select.with.iter().flat_map(|with| with.ctes.iter()) { - let keys = environment.query_keys(&cte.this); - environment.ctes.insert(lower(&cte.alias.name), keys); - if let Expression::Select(body) = &cte.this { - environment - .row_numbers - .insert(lower(&cte.alias.name), row_number_outputs(body)); - } - } - } - environment -} - -fn lowered_keys(keys: &HashMap) -> HashMap { - let mut lowered: HashMap = HashMap::new(); - for (name, relation_keys) in keys { - let mut converted: Keys = Vec::new(); - for key in relation_keys { - let mut columns: Vec = Vec::new(); - for column in key { - columns.push(lower(column)); - } - converted.push(columns); - } - lowered.insert(lower(name), converted); - } - lowered -} - -/// `(output name, partition output columns)` for each `ROW_NUMBER() OVER (PARTITION BY ...)` output. -fn row_number_outputs(select: &Select) -> Vec<(String, Vec)> { - let output: Vec> = projections(select); - let mut found: Vec<(String, Vec)> = Vec::new(); - for item in &output { - let Projection::Named { - name, - expression: Expression::WindowFunction(window), - } = item - else { - continue; - }; - if !matches!(window.this, Expression::RowNumber(_)) || window.over.partition_by.is_empty() { - continue; - } - if let Some(key) = output_key_of(&window.over.partition_by, &output) { - found.push((name.clone(), key)); - } - } - found -} - -/// Columns a filter pins to the first row, as in `WHERE rn = 1 AND ...`. -fn first_row_filters(condition: &Expression) -> Vec { - let mut found: Vec = Vec::new(); - let mut pending: Vec<&Expression> = vec![condition]; - while let Some(expression) = pending.pop() { - match expression { - Expression::And(operation) => { - pending.push(&operation.left); - pending.push(&operation.right); - } - Expression::Paren(paren) => pending.push(&paren.this), - Expression::Eq(operation) => { - if let (Expression::Column(column), Expression::Literal(literal)) - | (Expression::Literal(literal), Expression::Column(column)) = - (&operation.left, &operation.right) - && matches!(literal.as_ref(), Literal::Number(text) if text == FIRST_ROW) - { - found.push(lower(&column.name.name)); - } - } - _ => {} - } - } - found -} diff --git a/crates/sqlbuild-rules/src/sql_quality/_helpers/literals.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/literals.rs deleted file mode 100644 index d052eb7085..0000000000 --- a/crates/sqlbuild-rules/src/sql_quality/_helpers/literals.rs +++ /dev/null @@ -1,64 +0,0 @@ -//! SQBRSQL044: string literals longer than the project limit. - -use std::borrow::Cow; - -use polyglot_sql::tokens::TokenType; - -use crate::configuration::main::setting_help::setting_help; -use crate::sql_lint::models::LintDiagnostic; -use crate::sql_quality::constants::{ - LONG_LITERAL, LONG_LITERAL_GUIDANCE, MAX_LITERAL_LENGTH_KEY, RULES_THRESHOLDS_SECTION, -}; -use crate::sql_quality::models::QualityRequest; - -const CONSTANT_FIX_REFUSAL: &str = - "Moving the value to a constant requires choosing its name and placement"; - -pub(crate) fn diagnostics(request: &QualityRequest<'_>) -> Vec { - request - .tokens - .iter() - .filter(|token| is_string(token.token_type)) - .filter_map(|token| { - let length: usize = token.text.chars().count(); - (length > request.max_literal_length).then(|| LintDiagnostic { - code: LONG_LITERAL.code, - message: Cow::Borrowed(LONG_LITERAL.message), - remediation: Cow::Owned(remediation(length, request.max_literal_length)), - start: token.span.start, - end: token.span.end, - fix: None, - fix_unavailable_reason: Some(CONSTANT_FIX_REFUSAL), - }) - }) - .collect() -} - -fn remediation(length: usize, limit: usize) -> String { - format!( - "{LONG_LITERAL_GUIDANCE} {}", - setting_help( - &format!("To allow literals of {length} characters (the current value is {limit})"), - RULES_THRESHOLDS_SECTION, - MAX_LITERAL_LENGTH_KEY, - &length.to_string(), - ) - ) -} - -fn is_string(kind: TokenType) -> bool { - matches!( - kind, - TokenType::String - | TokenType::DollarString - | TokenType::EscapeString - | TokenType::RawString - | TokenType::NationalString - | TokenType::UnicodeString - | TokenType::ByteString - | TokenType::TripleSingleQuotedString - | TokenType::TripleDoubleQuotedString - | TokenType::HeredocString - | TokenType::HeredocStringAlternative - ) -} diff --git a/crates/sqlbuild-rules/src/sql_quality/_helpers/ranking.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/ranking.rs deleted file mode 100644 index fb4e463d6a..0000000000 --- a/crates/sqlbuild-rules/src/sql_quality/_helpers/ranking.rs +++ /dev/null @@ -1,216 +0,0 @@ -//! SQBRSQL043 ranking sort caps and SQBRSQL018 ranking determinism proofs. - -use std::borrow::Cow; -use std::collections::{HashMap, HashSet}; - -use polyglot_sql::expressions::{Over, Select, WindowFunction}; -use polyglot_sql::scope::walk_in_scope; -use polyglot_sql::tokens::Token; -use polyglot_sql::{Expression, ExpressionWalk}; - -use crate::configuration::main::setting_help::setting_help; -use crate::sql_lint::models::LintDiagnostic; -use crate::sql_quality::constants::{ - MAX_RANKING_ORDER_BY_KEY, RANKING_SORT_CAP, RANKING_SORT_GUIDANCE, RULES_THRESHOLDS_SECTION, - TIE_SENSITIVE_FUNCTIONS, UNPROVEN_RANKING_MESSAGE, UNPROVEN_RANKING_REMEDIATION, - UNSTABLE_ROW_NUMBER_CODE, -}; -use crate::sql_quality::keys::{KeyEnvironment, covers, window_columns}; -use crate::sql_quality::models::QualityRequest; -use crate::sql_quality::syntax::{columns, lower}; -use crate::sql_quality::types::InputKey; - -const PRIORITY_FIX_REFUSAL: &str = - "Choosing a priority column and tie-breaker requires author intent"; -const KEY_FIX_REFUSAL: &str = "Declaring a key or adding a tie-breaker requires author intent"; - -struct RankingWindow<'a> { - function: &'static str, - window: &'a WindowFunction, - select: &'a Select, -} - -pub(crate) fn diagnostics( - request: &QualityRequest<'_>, - environment: &KeyEnvironment, -) -> Vec { - let cap: bool = request.enabled.contains(RANKING_SORT_CAP.code); - let proof: bool = request.enabled.contains(UNSTABLE_ROW_NUMBER_CODE); - let mut used: HashSet = HashSet::new(); - let mut function_tokens: HashMap> = HashMap::new(); - for (index, token) in request.tokens.iter().enumerate() { - if TIE_SENSITIVE_FUNCTIONS - .iter() - .chain(["RANK", "DENSE_RANK"].iter()) - .any(|name| token.text.eq_ignore_ascii_case(name)) - { - function_tokens - .entry(token.text.to_ascii_uppercase()) - .or_default() - .push(index); - } - } - let mut found: Vec = Vec::new(); - for ranking in ranking_windows(request.statements) { - let over: Option<&Over> = effective_over(&ranking.window.over, ranking.select); - let order: Vec<&Expression> = over.map_or_else(Vec::new, |over| { - over.order_by.iter().map(|ordered| &ordered.this).collect() - }); - let partition: &[Expression] = over.map_or(&[], |over| over.partition_by.as_slice()); - let exceeds: bool = cap && order.len() > request.max_ranking_order_by; - let unproven: bool = proof - && !order.is_empty() - && TIE_SENSITIVE_FUNCTIONS.contains(&ranking.function) - && !proven(environment, ranking.select, partition, &order); - if !exceeds && !unproven { - continue; - } - let candidates: &[usize] = function_tokens - .get(ranking.function) - .map_or(&[], Vec::as_slice); - let token: Option = function_token( - request.tokens, - candidates, - window_anchor(partition, &order), - &used, - ); - let (start, end) = token.map_or((0, 0), |index| { - ( - request.tokens[index].span.start, - request.tokens[index].span.end, - ) - }); - used.extend(token); - if exceeds { - found.push(LintDiagnostic { - code: RANKING_SORT_CAP.code, - message: Cow::Borrowed(RANKING_SORT_CAP.message), - remediation: Cow::Owned(sort_cap_remediation( - order.len(), - request.max_ranking_order_by, - )), - start, - end, - fix: None, - fix_unavailable_reason: Some(PRIORITY_FIX_REFUSAL), - }); - } - if unproven { - found.push(LintDiagnostic { - code: UNSTABLE_ROW_NUMBER_CODE, - message: Cow::Borrowed(UNPROVEN_RANKING_MESSAGE), - remediation: Cow::Borrowed(UNPROVEN_RANKING_REMEDIATION), - start, - end, - fix: None, - fix_unavailable_reason: Some(KEY_FIX_REFUSAL), - }); - } - } - found -} - -fn sort_cap_remediation(sort_keys: usize, limit: usize) -> String { - format!( - "{RANKING_SORT_GUIDANCE} {}", - setting_help( - &format!("To allow {sort_keys} sort keys (the current value is {limit})"), - RULES_THRESHOLDS_SECTION, - MAX_RANKING_ORDER_BY_KEY, - &sort_keys.to_string(), - ) - ) -} - -fn proven( - environment: &KeyEnvironment, - select: &Select, - partition: &[Expression], - order: &[&Expression], -) -> bool { - let keys: Vec = environment.input_keys(select); - covers(&keys, &window_columns(partition, order)) -} - -fn ranking_windows(statements: &[Expression]) -> Vec> { - let mut found: Vec> = Vec::new(); - for statement in statements { - for node in statement.dfs() { - let Expression::Select(select) = node else { - continue; - }; - for local in walk_in_scope(node, false) { - let Expression::WindowFunction(window) = local else { - continue; - }; - let function: &'static str = match &window.this { - Expression::RowNumber(_) => "ROW_NUMBER", - Expression::Rank(_) => "RANK", - Expression::DenseRank(_) => "DENSE_RANK", - Expression::FirstValue(_) => "FIRST_VALUE", - Expression::LastValue(_) => "LAST_VALUE", - _ => continue, - }; - found.push(RankingWindow { - function, - window, - select, - }); - } - } - } - found -} - -fn effective_over<'a>(over: &'a Over, select: &'a Select) -> Option<&'a Over> { - let mut current: &Over = over; - let mut seen: HashSet = HashSet::new(); - loop { - if !current.order_by.is_empty() || current.window_name.is_none() { - return Some(current); - } - let name: String = lower(¤t.window_name.as_ref()?.name); - if !seen.insert(name.clone()) { - return None; - } - current = &select - .windows - .iter() - .flatten() - .find(|window| lower(&window.name.name) == name)? - .spec; - } -} - -/// Start offset of the first located column a window partitions or orders by. -fn window_anchor(partition: &[Expression], order: &[&Expression]) -> Option { - let mut expressions: Vec<&Expression> = partition.iter().collect(); - expressions.extend(order.iter().copied()); - for expression in expressions { - for column in columns(expression) { - if let Some(span) = column.span { - return Some(span.start); - } - } - } - None -} - -/// The function-name token before the anchor, or the first token not reported yet. -fn function_token( - tokens: &[Token], - candidates: &[usize], - anchor: Option, - used: &HashSet, -) -> Option { - if let Some(anchor) = anchor { - let before: usize = candidates.partition_point(|&index| tokens[index].span.start < anchor); - if let Some(position) = before.checked_sub(1) { - return Some(candidates[position]); - } - } - candidates - .iter() - .copied() - .find(|index| !used.contains(index)) -} diff --git a/crates/sqlbuild-rules/src/sql_quality/_helpers/removal_safety.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/removal_safety.rs deleted file mode 100644 index eb5d8398a2..0000000000 --- a/crates/sqlbuild-rules/src/sql_quality/_helpers/removal_safety.rs +++ /dev/null @@ -1,102 +0,0 @@ -//! SQBRSQL042 allow-list: when removing an unused CTE output provably keeps every row. - -use polyglot_sql::expressions::Select; -use polyglot_sql::traversal::contains_aggregate; -use polyglot_sql::{Expression, ExpressionWalk}; - -use crate::sql_quality::constants::{ - DETERMINISTIC_SCALAR_FUNCTIONS, MIN_REMOVABLE_OUTPUTS, ROW_NEUTRAL_EXPRESSIONS, -}; - -const ORDER_BY_ALL: &str = "all"; -const DISTINCT_REFUSAL: &str = "SELECT DISTINCT depends on every output column"; -const DISTINCT_ON_REFUSAL: &str = "SELECT DISTINCT ON keeps one row per key chosen by the select"; -const GROUP_BY_ALL_REFUSAL: &str = "GROUP BY ALL groups by every non-aggregate output column"; -const ORDER_BY_ALL_REFUSAL: &str = "ORDER BY ALL sorts by every output column"; -const UNGROUPED_AGGREGATE_REFUSAL: &str = - "An aggregate select without GROUP BY depends on its whole projection"; -const POSITIONAL_REFUSAL: &str = "Positional GROUP BY or ORDER BY would shift"; -const ONLY_OUTPUT_REFUSAL: &str = "Removing the only output column leaves an empty select"; -const EXPRESSION_REFUSAL: &str = "Only column references and deterministic scalar expressions are removed automatically; this expression may change the rows the CTE returns"; - -/// Why removing any output of `select` could change its rows, if it could. -pub(super) fn select_refusal(select: &Select, outputs: usize) -> Option<&'static str> { - if select.distinct { - Some(DISTINCT_REFUSAL) - } else if select.distinct_on.is_some() { - Some(DISTINCT_ON_REFUSAL) - } else if select - .group_by - .as_ref() - .is_some_and(|group| group.all == Some(true)) - { - Some(GROUP_BY_ALL_REFUSAL) - } else if orders_by_all(select) { - Some(ORDER_BY_ALL_REFUSAL) - } else if select.group_by.is_none() - && (select.having.is_some() || select.expressions.iter().any(contains_aggregate)) - { - Some(UNGROUPED_AGGREGATE_REFUSAL) - } else if positional_clauses(select) { - Some(POSITIONAL_REFUSAL) - } else if outputs < MIN_REMOVABLE_OUTPUTS { - Some(ONLY_OUTPUT_REFUSAL) - } else { - None - } -} - -/// Why removing this one projection could change the rows, if it could. -pub(super) fn expression_refusal(expression: &Expression) -> Option<&'static str> { - (!row_neutral(expression)).then_some(EXPRESSION_REFUSAL) -} - -/// Whether a star select depends on every input column or on their positions. -pub(super) fn reads_whole_rows(select: &Select) -> bool { - select.distinct - || select.distinct_on.is_some() - || select.group_by.is_some() - || orders_by_all(select) - || positional_clauses(select) -} - -/// A column reference or deterministic scalar expression (no aggregate, window, SRF or subquery). -fn row_neutral(expression: &Expression) -> bool { - expression.dfs().all(|node| match node { - Expression::Function(function) => { - let name: String = function.name.to_ascii_uppercase(); - !function.distinct && DETERMINISTIC_SCALAR_FUNCTIONS.contains(&name.as_str()) - } - other => ROW_NEUTRAL_EXPRESSIONS.contains(&other.variant_name()), - }) -} - -fn orders_by_all(select: &Select) -> bool { - select - .order_by - .iter() - .flat_map(|order| order.expressions.iter()) - .any(|ordered| match &ordered.this { - Expression::Column(column) => { - column.table.is_none() - && !column.name.quoted - && column.name.name.eq_ignore_ascii_case(ORDER_BY_ALL) - } - Expression::Var(var) => var.this.eq_ignore_ascii_case(ORDER_BY_ALL), - _ => false, - }) -} - -fn positional_clauses(select: &Select) -> bool { - let numeric = |expression: &Expression| matches!(expression, Expression::Literal(_)); - select - .group_by - .iter() - .flat_map(|group| group.expressions.iter()) - .any(numeric) - || select - .order_by - .iter() - .flat_map(|order| order.expressions.iter()) - .any(|ordered| numeric(&ordered.this)) -} diff --git a/crates/sqlbuild-rules/src/sql_quality/_helpers/syntax.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/syntax.rs deleted file mode 100644 index 7f79990f0f..0000000000 --- a/crates/sqlbuild-rules/src/sql_quality/_helpers/syntax.rs +++ /dev/null @@ -1,124 +0,0 @@ -//! Shared syntax helpers for SQL quality Rules. - -use polyglot_sql::expressions::{Column, Select, Star}; -use polyglot_sql::tokens::{Token, TokenType}; -use polyglot_sql::{Expression, ExpressionWalk}; - -use crate::sql_quality::constants::MIN_REMOVABLE_OUTPUTS; -use rustc_hash::FxHashMap; - -/// One projection of a select list. -pub(super) enum Projection<'a> { - Named { - name: String, - expression: &'a Expression, - }, - Star(&'a Star), - Unnamed, -} - -pub(super) fn lower(name: &str) -> String { - name.to_ascii_lowercase() -} - -pub(super) fn projections(select: &Select) -> Vec> { - select - .expressions - .iter() - .map(|expression| match expression { - Expression::Alias(alias) => Projection::Named { - name: lower(&alias.alias.name), - expression: &alias.this, - }, - Expression::Column(column) => Projection::Named { - name: lower(&column.name.name), - expression, - }, - Expression::Star(star) => Projection::Star(star), - _ => Projection::Unnamed, - }) - .collect() -} - -pub(super) fn columns(expression: &Expression) -> Vec<&Column> { - expression - .dfs() - .filter_map(|node| match node { - Expression::Column(column) => Some(column.as_ref()), - _ => None, - }) - .collect() -} - -pub(super) fn qualifier(column: &Column) -> Option { - column.table.as_ref().map(|table| lower(&table.name)) -} - -/// Relations read directly by one select: `(alias, relation name, expression)`. -pub(super) fn sources(select: &Select) -> Vec<(String, Option, &Expression)> { - let mut found: Vec<(String, Option, &Expression)> = Vec::new(); - let from = select.from.iter().flat_map(|from| from.expressions.iter()); - for expression in from.chain(select.joins.iter().map(|join| &join.this)) { - match expression { - Expression::Table(table) => { - let name: String = lower(&table.name.name); - let alias: String = table - .alias - .as_ref() - .map_or_else(|| name.clone(), |alias| lower(&alias.name)); - found.push((alias, Some(name), expression)); - } - Expression::Subquery(subquery) => { - let alias: String = subquery - .alias - .as_ref() - .map_or_else(String::new, |alias| lower(&alias.name)); - found.push((alias, None, expression)); - } - _ => found.push((String::new(), None, expression)), - } - } - found -} - -/// Token index of the SELECT opening each `name AS (SELECT` CTE body, by lower-cased name. -pub(super) fn cte_select_tokens(tokens: &[Token]) -> FxHashMap { - let significant: Vec = (0..tokens.len()) - .filter(|&index| !is_trivia(&tokens[index])) - .collect(); - let mut found: FxHashMap = FxHashMap::default(); - for window in significant.windows(4) { - let [name_index, as_index, paren_index, select_index] = window else { - continue; - }; - if tokens[*as_index].token_type == TokenType::As - && tokens[*paren_index].token_type == TokenType::LParen - && tokens[*select_index].token_type == TokenType::Select - { - found - .entry(lower(&tokens[*name_index].text)) - .or_insert(*select_index); - } - } - found -} - -/// The comma-inclusive deletion span that removes projection `index` from `spans`. -pub(super) fn projection_removal(spans: &[(usize, usize)], index: usize) -> Option<(usize, usize)> { - if spans.len() < MIN_REMOVABLE_OUTPUTS { - return None; - } - let (start, end) = spans[index]; - if index + 1 < spans.len() { - Some((start, spans[index + 1].0)) - } else { - Some((spans[index - 1].1, end)) - } -} - -pub(super) fn is_trivia(token: &Token) -> bool { - matches!( - token.token_type, - TokenType::Space | TokenType::Break | TokenType::LineComment | TokenType::BlockComment - ) -} diff --git a/crates/sqlbuild-rules/src/sql_quality/constants.rs b/crates/sqlbuild-rules/src/sql_quality/constants.rs deleted file mode 100644 index fbf0f8a3f3..0000000000 --- a/crates/sqlbuild-rules/src/sql_quality/constants.rs +++ /dev/null @@ -1,202 +0,0 @@ -use crate::sql_lint::models::LintRuleMetadata; - -pub(crate) const UNUSED_CTE_OUTPUT: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL042", - message: "CTE output column is never read later in the model", - remediation: "Remove the column from the CTE output, or read it in a later CTE or the final query.", -}; -pub(crate) const RANKING_SORT_CAP: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL043", - message: "Ranking window sorts by more expressions than max_ranking_order_by", - remediation: "Order by the one or two columns that decide which row wins, then a unique row key as the final tie-breaker.", -}; -pub(crate) const LONG_LITERAL: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL044", - message: "String literal is longer than max_literal_length", - remediation: "Define the value once as a constant and reference it with `@const`, or shorten it.", -}; -pub(crate) const REPEATED_JSON_PARSE: LintRuleMetadata = LintRuleMetadata { - code: "SQBRSQL045", - message: "JSON payload is parsed more than once in one SELECT", - remediation: "Parse the payload once in the CTE that reads it, for example `TRY_PARSE_JSON(value) AS payload`, and read every field from that column in a later step.", -}; -pub(crate) const UNSTABLE_ROW_NUMBER_CODE: &str = "SQBRSQL018"; -pub(crate) const UNPROVEN_RANKING_MESSAGE: &str = - "Ranking window order is not proven unique within its partition"; -pub(crate) const UNPROVEN_RANKING_REMEDIATION: &str = "PARTITION BY and ORDER BY must together cover a declared key of the ranked rows. End the ORDER BY with a unique tie-breaker column, or declare the key upstream. A key qualifies when it is a `unique` column audit on a column that is also non-null (a `not_null` audit or `nullable false`), or the `unique_key` of an incremental `merge` model or of a `delete_insert` model without a cursor; snapshot, append, microbatch and cursor-ranged `unique_key`s do not prove one row per key. If the rows have no unique column, build one in the upstream model from the columns that identify a row (for example `HASH(source_partition, source_offset, line_index) AS row_key`), declare it with a `unique` audit, and order by it last."; -/// SQBRSQL043 guidance before the `max_ranking_order_by` setting snippet. -pub(crate) const RANKING_SORT_GUIDANCE: &str = "Order by the one or two columns that decide which row wins, then a unique row key as the final tie-breaker. If the rows have no unique column, build one in the upstream model from the columns that identify a row (for example `HASH(source_partition, source_offset, line_index) AS row_key`) and declare it with a `unique` audit, so SQBRSQL018 can prove the order is deterministic. If the order genuinely needs more keys, raise `max_ranking_order_by` for the project."; -/// SQBRSQL044 guidance before the `max_literal_length` setting snippet. -pub(crate) const LONG_LITERAL_GUIDANCE: &str = "Define the value once as a constant and reference it with `@const`, in the narrowest scope that covers every use. (1) Only this model: add `constants (_status_pattern '')` to its `MODEL (...)` header and write `@const(\"_status_pattern\")`. (2) Other files use the same value: declare `CONSTANT (name status_pattern, value '');` in `_sqlbuild/_constants/` of the nearest folder that contains every use when they all sit directly in that folder, or in its `_sqlbuild/constants/` to include subfolders, and write `@const(\"status_pattern\")`. (3) Use top-level `constants/` only when the value is needed project-wide. Otherwise shorten the literal."; -pub(crate) const RULES_THRESHOLDS_SECTION: &str = "rules.thresholds"; -pub(crate) const MAX_RANKING_ORDER_BY_KEY: &str = "max_ranking_order_by"; -pub(crate) const MAX_LITERAL_LENGTH_KEY: &str = "max_literal_length"; -pub(crate) const DISTINCT_UNUSED_REMEDIATION: &str = "This CTE uses SELECT DISTINCT, so the column still decides which rows are kept. Make the deduplication independent of the column, or restructure the CTE so it outputs only what later steps read."; -pub(crate) const GROUP_KEY_UNUSED_REMEDIATION: &str = "Drop the column from the CTE output but keep it in GROUP BY; the grouping grain stays the same."; -pub(crate) const GROUP_BY_ALL_UNUSED_REMEDIATION: &str = "This CTE uses GROUP BY ALL, so the column is part of the grouping grain. List the grouping columns explicitly and keep this one in GROUP BY, then drop it from the output."; -/// Expression kinds that compute one value per input row without changing the row set. -pub(crate) const ROW_NEUTRAL_EXPRESSIONS: &[&str] = &[ - "literal", - "boolean", - "null", - "identifier", - "column", - "paren", - "dot", - "subscript", - "data_type", - "interval", - "cast", - "try_cast", - "safe_cast", - "collation", - "case", - "and", - "or", - "not", - "neg", - "add", - "sub", - "mul", - "div", - "mod", - "eq", - "neq", - "lt", - "lte", - "gt", - "gte", - "like", - "i_like", - "is", - "is_null", - "is_true", - "is_false", - "in", - "between", - "null_safe_eq", - "bitwise_and", - "bitwise_or", - "bitwise_xor", - "concat", - "concat_ws", - "substring", - "upper", - "lower", - "length", - "trim", - "replace", - "left", - "right", - "lpad", - "rpad", - "split", - "split_part", - "regexp_replace", - "initcap", - "position", - "contains", - "starts_with", - "ends_with", - "abs", - "round", - "floor", - "ceil", - "greatest", - "least", - "coalesce", - "null_if", - "if_func", - "if_null", - "nvl", - "nvl2", - "decode", - "date_add", - "date_diff", - "date_trunc", - "timestamp_trunc", - "extract", - "to_date", - "to_timestamp", - "to_char", - "to_number", - "date", - "year", - "month", - "last_day", - "struct_extract", -]; -/// Generic function names known to be deterministic scalar functions in supported dialects. -pub(crate) const DETERMINISTIC_SCALAR_FUNCTIONS: &[&str] = &[ - "ABS", - "ARRAY_SIZE", - "CEIL", - "CHARINDEX", - "COALESCE", - "CONCAT", - "CONCAT_WS", - "DATEADD", - "DATEDIFF", - "DATE_ADD", - "DATE_DIFF", - "DATE_PART", - "DATE_TRUNC", - "DAY", - "DIV0", - "FLOOR", - "GREATEST", - "HOUR", - "IFF", - "IFNULL", - "LEAST", - "LEFT", - "LENGTH", - "LOWER", - "LPAD", - "LTRIM", - "MD5", - "MOD", - "MONTH", - "NULLIF", - "NVL", - "NVL2", - "POWER", - "REGEXP_REPLACE", - "REPLACE", - "RIGHT", - "ROUND", - "RPAD", - "RTRIM", - "SPLIT_PART", - "SQRT", - "STRPOS", - "SUBSTR", - "SUBSTRING", - "TO_CHAR", - "TO_DATE", - "TO_VARCHAR", - "TRIM", - "TRUNC", - "UPPER", - "YEAR", - "ZEROIFNULL", -]; -pub(crate) const DEFAULT_MAX_LITERAL_LENGTH: usize = 100; -pub(crate) const DEFAULT_MAX_RANKING_ORDER_BY: usize = 5; -pub(crate) const TIE_SENSITIVE_FUNCTIONS: [&str; 3] = ["ROW_NUMBER", "FIRST_VALUE", "LAST_VALUE"]; -pub(crate) const MIN_REMOVABLE_OUTPUTS: usize = 2; -pub(crate) const INTERPOLATION_SENTINEL_PREFIXES: [&str; 3] = [ - "__sqb_lint_", - "__sqlbuild_audit_parameter_", - "__sqlbuild_context_parameter_", -]; -pub(crate) const FIRST_ROW: &str = "1"; - -pub(crate) const UNIQUE_AUDIT: &str = "unique"; -pub(crate) const NOT_NULL_AUDIT: &str = "not_null"; -pub(crate) const INCREMENTAL_MATERIALIZATION: &str = "incremental"; -pub(crate) const MICROBATCH_MODE: &str = "microbatch"; -pub(crate) const KEYED_DELETE_INSERT_STRATEGY: &str = "delete_insert"; -pub(crate) const KEY_PRESERVING_STRATEGIES: [&str; 2] = ["merge", KEYED_DELETE_INSERT_STRATEGY]; -pub(crate) const RELATION_KIND_REF: &str = "ref"; -pub(crate) const RELATION_KIND_SEED: &str = "seed"; diff --git a/crates/sqlbuild-rules/src/sql_quality/main/diagnostics.rs b/crates/sqlbuild-rules/src/sql_quality/main/diagnostics.rs deleted file mode 100644 index 76848ccae4..0000000000 --- a/crates/sqlbuild-rules/src/sql_quality/main/diagnostics.rs +++ /dev/null @@ -1,32 +0,0 @@ -use crate::sql_lint::models::LintDiagnostic; -use crate::sql_quality::constants::{ - LONG_LITERAL, RANKING_SORT_CAP, REPEATED_JSON_PARSE, UNSTABLE_ROW_NUMBER_CODE, - UNUSED_CTE_OUTPUT, -}; -use crate::sql_quality::keys::{KeyEnvironment, key_environment}; -use crate::sql_quality::models::QualityRequest; -use crate::sql_quality::{cte_columns, json_parses, literals, ranking}; - -/// Run every enabled SQL quality Rule over one parsed body. -pub(crate) fn diagnostics(request: &QualityRequest<'_>) -> Result, String> { - if request.max_literal_length == 0 || request.max_ranking_order_by == 0 { - return Err("max_literal_length and max_ranking_order_by must be positive".to_owned()); - } - let mut found: Vec = Vec::new(); - if request.enabled.contains(LONG_LITERAL.code) { - found.extend(literals::diagnostics(request)); - } - if request.enabled.contains(REPEATED_JSON_PARSE.code) { - found.extend(json_parses::diagnostics(request)); - } - if request.enabled.contains(UNUSED_CTE_OUTPUT.code) { - found.extend(cte_columns::diagnostics(request)); - } - if request.enabled.contains(RANKING_SORT_CAP.code) - || request.enabled.contains(UNSTABLE_ROW_NUMBER_CODE) - { - let environment: KeyEnvironment = key_environment(request); - found.extend(ranking::diagnostics(request, &environment)); - } - Ok(found) -} diff --git a/crates/sqlbuild-rules/src/sql_quality/mod.rs b/crates/sqlbuild-rules/src/sql_quality/mod.rs deleted file mode 100644 index 2c9b78b9fc..0000000000 --- a/crates/sqlbuild-rules/src/sql_quality/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -//! Enforceable SQL quality Rules with project-wide limits and ranking determinism proofs. - -pub mod constants; -#[path = "_helpers/cte_columns.rs"] -mod cte_columns; -#[path = "_helpers/declared_keys.rs"] -mod declared_keys; -#[path = "_helpers/json_parses.rs"] -mod json_parses; -#[path = "_helpers/keys.rs"] -mod keys; -#[path = "_helpers/literals.rs"] -mod literals; -pub mod main; -pub mod models; -#[path = "_helpers/ranking.rs"] -mod ranking; -#[path = "_helpers/removal_safety.rs"] -mod removal_safety; -#[path = "_helpers/syntax.rs"] -mod syntax; -pub mod types; - -#[cfg(test)] -mod tests; diff --git a/crates/sqlbuild-rules/src/sql_quality/tests.rs b/crates/sqlbuild-rules/src/sql_quality/tests.rs deleted file mode 100644 index 2643da6c21..0000000000 --- a/crates/sqlbuild-rules/src/sql_quality/tests.rs +++ /dev/null @@ -1,10 +0,0 @@ -#[path = "tests/helpers.rs"] -mod helpers; -#[path = "tests/test_json_parses.rs"] -mod json_parses; -#[path = "tests/test_quality_rules.rs"] -mod quality_rules; -#[path = "tests/test_quality_scale.rs"] -mod quality_scale; -#[path = "tests/test_types.rs"] -mod test_types; diff --git a/crates/sqlbuild-rules/src/sql_quality/tests/helpers.rs b/crates/sqlbuild-rules/src/sql_quality/tests/helpers.rs deleted file mode 100644 index 3b8a43de34..0000000000 --- a/crates/sqlbuild-rules/src/sql_quality/tests/helpers.rs +++ /dev/null @@ -1,116 +0,0 @@ -use serde_json::{Value, json}; - -use crate::sql_lint::main::engine::lint_json; -use crate::sql_quality::tests::test_types::QualityRuleTestCase; - -pub(crate) fn lint(test_case: &QualityRuleTestCase) -> Result, String> { - let relation_keys: Value = - serde_json::from_str(test_case.relation_keys).map_err(|error| error.to_string())?; - let request: Value = json!({ - "version": 1, - "sql": test_case.sql, - "dialect": "duckdb", - "enabled_rules": [test_case.rule], - "max_literal_length": 24, - "max_ranking_order_by": 2, - "relation_keys": relation_keys, - }); - let response: Value = serde_json::from_str(&lint_json(&request.to_string())?) - .map_err(|error| error.to_string())?; - Ok(response["diagnostics"] - .as_array() - .cloned() - .unwrap_or_default()) -} - -/// Lint `sql` in `dialect` with only `rule` enabled. -pub(crate) fn lint_dialect(sql: &str, dialect: &str, rule: &str) -> Result, String> { - let request: Value = json!({ - "version": 1, - "sql": sql, - "dialect": dialect, - "enabled_rules": [rule], - }); - let response: Value = serde_json::from_str(&lint_json(&request.to_string())?) - .map_err(|error| error.to_string())?; - Ok(response["diagnostics"] - .as_array() - .cloned() - .unwrap_or_default()) -} - -/// One DuckDB SELECT that parses each of `column_count` payload columns twice. -pub(crate) fn wide_json_select(column_count: usize) -> String { - let projections: Vec = (0..column_count * 2) - .map(|index| format!("json(e.c{})->>'f{index}' AS f{index}", index % column_count)) - .collect(); - format!("SELECT {} FROM events AS e", projections.join(", ")) -} - -pub(crate) fn anchors(sql: &str, diagnostics: &[Value]) -> Vec { - diagnostics - .iter() - .map(|diagnostic| { - let start: usize = diagnostic["start"].as_u64().unwrap_or(0) as usize; - let end: usize = diagnostic["end"].as_u64().unwrap_or(0) as usize; - sql.chars() - .skip(start) - .take(end.saturating_sub(start)) - .collect() - }) - .collect() -} - -pub(crate) fn fixed(sql: &str, diagnostics: &[Value]) -> Option { - let mut edits: Vec<(usize, usize, String)> = diagnostics - .iter() - .filter_map(|diagnostic| { - let fix: &Value = diagnostic.get("fix")?; - Some(( - fix["start"].as_u64()? as usize, - fix["end"].as_u64()? as usize, - fix["replacement"].as_str()?.to_owned(), - )) - }) - .collect(); - edits.sort_by_key(|edit| std::cmp::Reverse(edit.0)); - let applied: bool = !edits.is_empty(); - let mut characters: Vec = sql.chars().collect(); - for (start, end, replacement) in edits { - characters.splice(start..end, replacement.chars()); - } - applied.then(|| characters.into_iter().collect()) -} - -/// A chain of CTEs that each drop their last column and rank without a declared key. -pub(crate) fn wide_cte_chain(cte_count: usize, column_count: usize) -> String { - let columns: Vec = (0..column_count).map(|index| format!("c{index}")).collect(); - let mut ctes: Vec = vec![format!( - "step_0 AS (SELECT {} FROM source_rows AS s)", - columns - .iter() - .map(|column| format!("s.{column}")) - .collect::>() - .join(", ") - )]; - for step in 1..cte_count { - let mut outputs: Vec = columns[..column_count - 1] - .iter() - .map(|column| format!("p.{column}")) - .collect(); - outputs.push(format!( - "ROW_NUMBER() OVER (PARTITION BY p.c0 ORDER BY p.c1) AS c{}", - column_count - 1 - )); - ctes.push(format!( - "step_{step} AS (SELECT {} FROM step_{} AS p)", - outputs.join(", "), - step - 1 - )); - } - format!( - "WITH {} SELECT * FROM step_{}", - ctes.join(",\n"), - cte_count - 1 - ) -} diff --git a/crates/sqlbuild-rules/src/sql_quality/tests/test_json_parses.rs b/crates/sqlbuild-rules/src/sql_quality/tests/test_json_parses.rs deleted file mode 100644 index 29a89e6b6c..0000000000 --- a/crates/sqlbuild-rules/src/sql_quality/tests/test_json_parses.rs +++ /dev/null @@ -1,282 +0,0 @@ -use std::time::Instant; - -use serde_json::Value; - -use crate::sql_quality::tests::helpers::{anchors, lint_dialect, wide_json_select}; -use crate::sql_quality::tests::test_types::{JsonParseScaleTestCase, JsonParseTestCase}; - -const RULE: &str = "SQBRSQL045"; - -#[test] -fn given_json_parse_calls_when_linting_then_repeats_in_one_select_are_reported() --> Result<(), String> { - let test_cases = [ - JsonParseTestCase { - description: "same column parsed twice reports the second call", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(e.value):order_id AS order_id, TRY_PARSE_JSON(e.value):status AS status FROM events AS e", - expected_anchors: &["TRY_PARSE_JSON"], - expected_details: &["TRY_PARSE_JSON(e.value) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "the finding is located at the second call", - dialect: "snowflake", - sql: "SELECT try_parse_json(value):a AS a, TRY_PARSE_JSON(value):b AS b FROM events", - expected_anchors: &["TRY_PARSE_JSON"], - expected_details: &["TRY_PARSE_JSON(value) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "three parses give one finding with the count", - dialect: "snowflake", - sql: "SELECT PARSE_JSON(value):a AS a, PARSE_JSON(value):b AS b FROM events WHERE PARSE_JSON(value):c IS NOT NULL", - expected_anchors: &["PARSE_JSON"], - expected_details: &["PARSE_JSON(value) is parsed 3 times in one SELECT."], - }, - JsonParseTestCase { - description: "column casing and wrappers do not hide a repeat", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(LOWER(E.Value)):a AS a, TRY_PARSE_JSON((CAST(TRIM(e.value) AS VARCHAR))):b AS b, TRY_PARSE_JSON(COALESCE(UPPER(e.VALUE))):c AS c FROM events AS e", - expected_anchors: &["TRY_PARSE_JSON"], - expected_details: &["TRY_PARSE_JSON(e.value) is parsed 3 times in one SELECT."], - }, - JsonParseTestCase { - description: "multi-argument COALESCE is not a plain column", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(COALESCE(e.value, e.backup)):a AS a, TRY_PARSE_JSON(COALESCE(e.value, e.fallback)):b AS b, TRY_PARSE_JSON(e.value):c AS c FROM events AS e", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "TRIM with characters is not a plain column", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(TRIM(e.value, ' ')):a AS a, TRY_PARSE_JSON(e.value):b AS b FROM events AS e", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "quoted names differing in case are different columns", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(e.\"Value\"):a AS a, TRY_PARSE_JSON(e.\"value\"):b AS b FROM events AS e", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "quoted and unquoted names are not merged", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(e.\"value\"):a AS a, TRY_PARSE_JSON(e.value):b AS b FROM events AS e", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "quoted qualifiers differing in case are different relations", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(\"E\".value):a AS a, TRY_PARSE_JSON(\"e\".value):b AS b FROM events AS \"E\" INNER JOIN events AS \"e\" ON \"E\".id = \"e\".id", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "identical quoted names are reported as written", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(\"E\".\"Value\"):a AS a, TRY_PARSE_JSON(\"E\".\"Value\"):b AS b FROM events AS \"E\"", - expected_anchors: &["TRY_PARSE_JSON"], - expected_details: &["TRY_PARSE_JSON(\"E\".\"Value\") is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "bigquery quoted names keep backticks", - dialect: "bigquery", - sql: "SELECT JSON_VALUE(PARSE_JSON(e.`Value`), '$.a') AS a, JSON_VALUE(PARSE_JSON(e.`Value`), '$.b') AS b FROM events AS e", - expected_anchors: &["PARSE_JSON"], - expected_details: &["PARSE_JSON(e.`Value`) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "snowflake quoted upper case matches the folded unquoted name", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(e.\"VALUE\"):a AS a, TRY_PARSE_JSON(e.value):b AS b FROM events AS e", - expected_anchors: &["TRY_PARSE_JSON"], - expected_details: &["TRY_PARSE_JSON(e.value) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "duckdb quoted names are case-insensitive", - dialect: "duckdb", - sql: "SELECT json(e.\"Value\") -> 'a' AS a, json(e.value) -> 'b' AS b FROM events AS e", - expected_anchors: &["json"], - expected_details: &["JSON(e.value) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "bigquery backticked names are case-insensitive", - dialect: "bigquery", - sql: "SELECT JSON_VALUE(PARSE_JSON(e.`Value`), '$.a') AS a, JSON_VALUE(PARSE_JSON(e.value), '$.b') AS b FROM events AS e", - expected_anchors: &["PARSE_JSON"], - expected_details: &["PARSE_JSON(e.value) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "databricks backticked names are case-insensitive", - dialect: "databricks", - sql: "SELECT try_parse_json(e.`v`) AS a, try_parse_json(e.V) AS b FROM events AS e", - expected_anchors: &["try_parse_json"], - expected_details: &["TRY_PARSE_JSON(e.V) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "references are grouped as written", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(value):a AS a, TRY_PARSE_JSON(e.value):b AS b, TRY_PARSE_JSON(events.value):c AS c FROM events AS e", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "different columns pass", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(e.payload):a AS a, TRY_PARSE_JSON(e.metadata):b AS b FROM events AS e", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "same column name on different tables passes", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(o.payload):a AS a, TRY_PARSE_JSON(c.payload):b AS b FROM orders AS o INNER JOIN customers AS c ON o.customer_id = c.customer_id", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "different parse functions are not merged", - dialect: "snowflake", - sql: "SELECT PARSE_JSON(value):a AS a, TRY_PARSE_JSON(value):b AS b FROM events", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "one parse per CTE scope passes", - dialect: "snowflake", - sql: "WITH parsed AS (SELECT TRY_PARSE_JSON(value) AS payload FROM events), final AS (SELECT TRY_PARSE_JSON(value) AS payload FROM events) SELECT payload FROM final", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "nested SELECT is its own scope", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(e.value):a AS a, (SELECT MAX(TRY_PARSE_JSON(x.value):b) FROM events AS x) AS b FROM events AS e", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "repeat inside a subquery is reported in that scope", - dialect: "snowflake", - sql: "SELECT e.order_id FROM events AS e WHERE e.order_id IN (SELECT TRY_PARSE_JSON(value):id FROM events WHERE TRY_PARSE_JSON(value):ok)", - expected_anchors: &["TRY_PARSE_JSON"], - expected_details: &["TRY_PARSE_JSON(value) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "union branches are separate scopes", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(value) AS payload FROM events UNION ALL SELECT TRY_PARSE_JSON(value) AS payload FROM events", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "a repeat inside one CTE body is reported", - dialect: "snowflake", - sql: "WITH parsed AS (SELECT TRY_PARSE_JSON(value):a AS a, TRY_PARSE_JSON(value):b AS b FROM events) SELECT a, b FROM parsed", - expected_anchors: &["TRY_PARSE_JSON"], - expected_details: &["TRY_PARSE_JSON(value) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "non-column arguments pass", - dialect: "snowflake", - sql: "SELECT TRY_PARSE_JSON(value || '') AS a, TRY_PARSE_JSON(value || '') AS b FROM events", - expected_anchors: &[], - expected_details: &[], - }, - JsonParseTestCase { - description: "bigquery safe parse is matched", - dialect: "bigquery", - sql: "SELECT JSON_VALUE(SAFE.PARSE_JSON(e.value), '$.a') AS a, JSON_VALUE(SAFE.PARSE_JSON(e.value), '$.b') AS b FROM events AS e", - expected_anchors: &["PARSE_JSON"], - expected_details: &["SAFE.PARSE_JSON(e.value) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "databricks try parse is matched", - dialect: "databricks", - sql: "SELECT try_parse_json(value) AS a, try_parse_json(value) AS b FROM events", - expected_anchors: &["try_parse_json"], - expected_details: &["TRY_PARSE_JSON(value) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "duckdb json is matched", - dialect: "duckdb", - sql: "SELECT json(e.value)->>'a' AS a, json(e.value)->>'b' AS b FROM events AS e", - expected_anchors: &["json"], - expected_details: &["JSON(e.value) is parsed 2 times in one SELECT."], - }, - JsonParseTestCase { - description: "dialects without an equivalent are not checked", - dialect: "postgres", - sql: "SELECT CAST(value AS JSONB) AS a, CAST(value AS JSONB) AS b FROM events", - expected_anchors: &[], - expected_details: &[], - }, - ]; - for test_case in &test_cases { - let diagnostics: Vec = lint_dialect(test_case.sql, test_case.dialect, RULE)?; - assert_eq!( - anchors(test_case.sql, &diagnostics), - test_case.expected_anchors, - "{}", - test_case.description - ); - let details: Vec = diagnostics - .iter() - .map(|diagnostic| { - diagnostic["remediation"] - .as_str() - .unwrap_or_default() - .split_inclusive(". ") - .next() - .unwrap_or_default() - .trim_end() - .to_owned() - }) - .collect(); - assert_eq!( - details, test_case.expected_details, - "{}", - test_case.description - ); - assert!( - diagnostics - .iter() - .all(|diagnostic| diagnostic["fix"].is_null()), - "{}", - test_case.description - ); - } - Ok(()) -} - -#[test] -fn given_wide_select_with_many_repeated_parses_when_linting_then_it_stays_linear() --> Result<(), String> { - let test_cases = [JsonParseScaleTestCase { - description: "3,000 projections parsing 1,500 payload columns twice each", - column_count: 1_500, - expected_findings: 1_500, - expected_max_seconds: 10.0, - }]; - for test_case in &test_cases { - let started: Instant = Instant::now(); - let diagnostics: Vec = - lint_dialect(&wide_json_select(test_case.column_count), "duckdb", RULE)?; - let elapsed: f64 = started.elapsed().as_secs_f64(); - assert_eq!( - diagnostics.len(), - test_case.expected_findings, - "{}", - test_case.description - ); - assert!( - elapsed <= test_case.expected_max_seconds, - "{}: {elapsed:.2}s", - test_case.description - ); - } - Ok(()) -} diff --git a/crates/sqlbuild-rules/src/sql_quality/tests/test_quality_rules.rs b/crates/sqlbuild-rules/src/sql_quality/tests/test_quality_rules.rs deleted file mode 100644 index ba5c068406..0000000000 --- a/crates/sqlbuild-rules/src/sql_quality/tests/test_quality_rules.rs +++ /dev/null @@ -1,405 +0,0 @@ -use serde_json::Value; - -use crate::sql_quality::tests::helpers::{anchors, fixed, lint}; -use crate::sql_quality::tests::test_types::{QualityRemediationTestCase, QualityRuleTestCase}; - -#[test] -fn given_quality_rule_inputs_when_linting_then_findings_and_fixes_match() -> Result<(), String> { - let test_cases = [ - QualityRuleTestCase { - description: "long literal is reported without a split fix", - sql: "SELECT 'pending orders awaiting shipment' AS label", - rule: "SQBRSQL044", - relation_keys: "{}", - expected_anchors: &["'pending orders awaiting shipment'"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "short literal passes", - sql: "SELECT 'pending' AS label", - rule: "SQBRSQL044", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "typed literal is reported without a fix", - sql: "SELECT TIMESTAMP '2026-04-01 10:00:00.000000000' AS ordered_at", - rule: "SQBRSQL044", - relation_keys: "{}", - expected_anchors: &["'2026-04-01 10:00:00.000000000'"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "ranking sort above the cap", - sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.a, o.b, o.order_id) AS rn FROM orders AS o", - rule: "SQBRSQL043", - relation_keys: "{}", - expected_anchors: &["ROW_NUMBER"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "ranking sort within the cap", - sql: "SELECT RANK() OVER (ORDER BY o.a, o.order_id) AS rn FROM orders AS o", - rule: "SQBRSQL043", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "ordering covers a declared key", - sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.ordered_at, o.order_id) AS rn FROM orders AS o", - rule: "SQBRSQL018", - relation_keys: r#"{"orders": [["order_id"]]}"#, - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "ordering without a key is unproven", - sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.ordered_at) AS rn FROM orders AS o", - rule: "SQBRSQL018", - relation_keys: r#"{"orders": [["order_id"]]}"#, - expected_anchors: &["ROW_NUMBER"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "key proven through a CTE and a one-to-one join", - sql: "WITH lines AS (SELECT l.line_id, l.order_id FROM order_lines AS l), final AS (SELECT l.line_id, o.customer_id, ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY l.line_id) AS rn FROM lines AS l INNER JOIN orders AS o ON l.order_id = o.order_id) SELECT f.line_id, f.customer_id, f.rn FROM final AS f", - rule: "SQBRSQL018", - relation_keys: r#"{"orders": [["order_id"]], "order_lines": [["line_id"]]}"#, - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "key lost through a fan-out join", - sql: "WITH final AS (SELECT o.order_id, ROW_NUMBER() OVER (ORDER BY o.order_id) AS rn FROM orders AS o INNER JOIN order_lines AS l ON l.order_id = o.order_id) SELECT f.order_id, f.rn FROM final AS f", - rule: "SQBRSQL018", - relation_keys: r#"{"orders": [["order_id"]], "order_lines": [["line_id"]]}"#, - expected_anchors: &["ROW_NUMBER"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "GROUP BY grain is a key", - sql: "WITH totals AS (SELECT o.customer_id, SUM(o.amount) AS amount FROM orders AS o GROUP BY o.customer_id), final AS (SELECT t.customer_id, ROW_NUMBER() OVER (ORDER BY t.amount, t.customer_id) AS rn FROM totals AS t) SELECT f.customer_id, f.rn FROM final AS f", - rule: "SQBRSQL018", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "positional GROUP BY grain is a key", - sql: "WITH totals AS (SELECT o.customer_id, SUM(o.amount) AS amount FROM orders AS o GROUP BY 1), final AS (SELECT t.customer_id, ROW_NUMBER() OVER (ORDER BY t.amount, t.customer_id) AS rn FROM totals AS t) SELECT f.customer_id, f.rn FROM final AS f", - rule: "SQBRSQL018", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "a later rn = 1 filter makes the partition a key", - sql: "WITH ranked AS (SELECT o.order_id, o.customer_id, ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.order_id) AS rn FROM orders AS o), latest AS (SELECT r.customer_id, r.order_id FROM ranked AS r WHERE r.rn = 1), final AS (SELECT l.customer_id, ROW_NUMBER() OVER (ORDER BY l.customer_id) AS position FROM latest AS l) SELECT f.customer_id, f.position FROM final AS f", - rule: "SQBRSQL018", - relation_keys: r#"{"orders": [["order_id"]]}"#, - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "a QUALIFY window cannot prove itself", - sql: "SELECT o.customer_id, o.order_id FROM orders AS o QUALIFY ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.amount DESC) = 1", - rule: "SQBRSQL018", - relation_keys: r#"{"orders": [["order_id"]]}"#, - expected_anchors: &["ROW_NUMBER"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "a QUALIFY partition is a key for later steps", - sql: "WITH latest AS (SELECT o.customer_id, o.order_id FROM orders AS o QUALIFY ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.order_id) = 1), final AS (SELECT l.customer_id, ROW_NUMBER() OVER (ORDER BY l.customer_id) AS position FROM latest AS l) SELECT f.customer_id, f.position FROM final AS f", - rule: "SQBRSQL018", - relation_keys: r#"{"orders": [["order_id"]]}"#, - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "RANK ties are deterministic", - sql: "SELECT RANK() OVER (ORDER BY o.ordered_at) AS rn FROM orders AS o", - rule: "SQBRSQL018", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "unused CTE output is removed", - sql: "WITH orders AS (SELECT o.order_id, o.status, o.note FROM raw_orders AS o), final AS (SELECT r.order_id, r.status FROM orders AS r) SELECT f.order_id, f.status FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &["o.note"], - expected_fixed_sql: Some( - "WITH orders AS (SELECT o.order_id, o.status FROM raw_orders AS o), final AS (SELECT r.order_id, r.status FROM orders AS r) SELECT f.order_id, f.status FROM final AS f", - ), - }, - QualityRuleTestCase { - description: "star pass-through into the final CTE keeps every column", - sql: "WITH orders AS (SELECT o.order_id, o.note FROM raw_orders AS o), final AS (SELECT * FROM orders) SELECT * FROM final", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "star pass-through follows the reader's own reads", - sql: "WITH orders AS (SELECT o.order_id, o.note FROM raw_orders AS o), staged AS (SELECT * FROM orders), final AS (SELECT s.order_id FROM staged AS s) SELECT f.order_id FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &["o.note"], - expected_fixed_sql: Some( - "WITH orders AS (SELECT o.order_id FROM raw_orders AS o), staged AS (SELECT * FROM orders), final AS (SELECT s.order_id FROM staged AS s) SELECT f.order_id FROM final AS f", - ), - }, - QualityRuleTestCase { - description: "a same-named cast of a source column is not a lateral read", - sql: "WITH orders AS (SELECT o.order_id, CAST(note AS VARCHAR) AS note, o.note || 'x' AS tagged FROM raw_orders AS o), final AS (SELECT r.order_id, r.tagged FROM orders AS r) SELECT f.order_id, f.tagged FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &["CAST(note AS VARCHAR) AS note"], - expected_fixed_sql: Some( - "WITH orders AS (SELECT o.order_id, o.note || 'x' AS tagged FROM raw_orders AS o), final AS (SELECT r.order_id, r.tagged FROM orders AS r) SELECT f.order_id, f.tagged FROM final AS f", - ), - }, - QualityRuleTestCase { - description: "an interpolated expression in a reader may read any column", - sql: "WITH orders AS (SELECT o.order_id, o.status FROM raw_orders AS o), final AS (SELECT r.order_id FROM orders AS r WHERE __sqlbuild_audit_parameter_0__) SELECT f.order_id FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "an interpolated output is never reported", - sql: "WITH orders AS (SELECT o.order_id, __SQB_LINT_0002 FROM raw_orders AS o), final AS (SELECT r.order_id FROM orders AS r) SELECT f.order_id FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "DISTINCT CTE reports without a fix", - sql: "WITH orders AS (SELECT DISTINCT o.order_id, o.note FROM raw_orders AS o), final AS (SELECT r.order_id FROM orders AS r) SELECT f.order_id FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &["o.note"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "columns read in later join and filter clauses are used", - sql: "WITH orders AS (SELECT o.order_id, o.customer_id, o.status FROM raw_orders AS o), final AS (SELECT c.name FROM orders AS r INNER JOIN customers AS c ON r.customer_id = c.customer_id WHERE r.status = 'open' QUALIFY ROW_NUMBER() OVER (PARTITION BY r.order_id ORDER BY c.name) = 1) SELECT f.name FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "a column read only through a nested star is used", - sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), final AS (SELECT g.order_id, HASH(g.*) AS row_hash FROM g) SELECT f.order_id, f.row_hash FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "a whole-row reference reads every column", - sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), final AS (SELECT g.order_id, HASH(g) AS row_hash FROM g) SELECT f.order_id, f.row_hash FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "COUNT(DISTINCT row) reads every column", - sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), final AS (SELECT COUNT(DISTINCT g) AS n FROM g) SELECT f.n FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "COLUMNS(*) reads every column", - sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), final AS (SELECT g.order_id FROM g WHERE COLUMNS(*) IS NOT NULL) SELECT f.order_id FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "COLUMNS with a pattern reads every column", - sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), final AS (SELECT MAX(COLUMNS('order.*')) FROM g) SELECT * FROM final", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "DISTINCT star pass-through reads every column", - sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), staged AS (SELECT DISTINCT * FROM g), final AS (SELECT s.order_id FROM staged AS s) SELECT f.order_id FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "a positional ORDER BY over a star pass-through reads every column", - sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), staged AS (SELECT * FROM g ORDER BY 2 LIMIT 1), final AS (SELECT s.order_id FROM staged AS s) SELECT f.order_id FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "an output alias read in WHERE or GROUP BY is used", - sql: "WITH g AS (SELECT r.order_id, r.amount * 2 AS doubled, r.customer_id % 2 AS bucket FROM raw_orders AS r WHERE doubled > 1 GROUP BY r.order_id, doubled, bucket), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "GROUP BY ALL grouping column is reported without a fix", - sql: "WITH g AS (SELECT r.customer_id, r.status, SUM(r.amount) AS total FROM raw_orders AS r GROUP BY ALL), final AS (SELECT g.customer_id, g.total FROM g) SELECT f.customer_id, f.total FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &["r.status"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "an aggregate in an ungrouped select is reported without a fix", - sql: "WITH g AS (SELECT 'all' AS label, COUNT(*) AS n FROM raw_orders AS r), final AS (SELECT g.label FROM g) SELECT f.label FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &["COUNT(*) AS n"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "a set-returning projection is reported without a fix", - sql: "WITH g AS (SELECT r.order_id, UNNEST(r.tags) AS tag FROM raw_orders AS r), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &["UNNEST(r.tags) AS tag"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "ORDER BY ALL with LIMIT is reported without a fix", - sql: "WITH g AS (SELECT r.order_id, r.customer_id FROM raw_orders AS r ORDER BY ALL LIMIT 1), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &["r.customer_id"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "DISTINCT ON is reported without a fix", - sql: "WITH g AS (SELECT DISTINCT ON (r.customer_id) r.order_id, r.amount FROM raw_orders AS r ORDER BY r.customer_id, r.order_id), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &["r.amount"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "a volatile projection is reported without a fix", - sql: "WITH g AS (SELECT r.order_id, RANDOM() AS draw FROM raw_orders AS r), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &["RANDOM() AS draw"], - expected_fixed_sql: None, - }, - QualityRuleTestCase { - description: "a deterministic scalar projection is removed", - sql: "WITH g AS (SELECT r.order_id, COALESCE(UPPER(r.status), 'none') AS label FROM raw_orders AS r), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", - rule: "SQBRSQL042", - relation_keys: "{}", - expected_anchors: &["COALESCE(UPPER(r.status), 'none') AS label"], - expected_fixed_sql: Some( - "WITH g AS (SELECT r.order_id FROM raw_orders AS r), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", - ), - }, - ]; - for test_case in &test_cases { - let diagnostics: Vec = lint(test_case)?; - assert_eq!( - anchors(test_case.sql, &diagnostics), - test_case.expected_anchors, - "{}", - test_case.description - ); - assert_eq!( - fixed(test_case.sql, &diagnostics).as_deref(), - test_case.expected_fixed_sql, - "{}", - test_case.description - ); - } - Ok(()) -} - -#[test] -fn given_threshold_findings_when_linting_then_remediation_shows_the_exact_setting() --> Result<(), String> { - let test_cases = [ - QualityRemediationTestCase { - description: "long literal points at a constant and the length setting", - sql: "SELECT 'pending orders awaiting shipment' AS label", - rule: "SQBRSQL044", - expected_fragments: &[ - "reference it with `@const`", - "(1) Only this model: add `constants (_status_pattern '')`", - "`_sqlbuild/_constants/` of the nearest folder", - "(3) Use top-level `constants/` only", - "the current value is 24", - "[rules.thresholds]\n max_literal_length = 32", - ], - }, - QualityRemediationTestCase { - description: "long ranking sort points at a row key and the sort setting", - sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.a, o.b, o.order_id) AS rn FROM orders AS o", - rule: "SQBRSQL043", - expected_fragments: &[ - "unique row key as the final tie-breaker", - "HASH(source_partition, source_offset, line_index) AS row_key", - "the current value is 2", - "[rules.thresholds]\n max_ranking_order_by = 3", - ], - }, - QualityRemediationTestCase { - description: "unproven ranking points at upstream keys and a built row key", - sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.a) AS rn FROM orders AS o", - rule: "SQBRSQL018", - expected_fragments: &[ - "a `unique` column audit on a column that is also non-null", - "the `unique_key` of an incremental `merge` model", - "HASH(source_partition, source_offset, line_index) AS row_key", - ], - }, - ]; - for test_case in &test_cases { - let diagnostics: Vec = lint(&QualityRuleTestCase { - description: test_case.description, - sql: test_case.sql, - rule: test_case.rule, - relation_keys: "{}", - expected_anchors: &[], - expected_fixed_sql: None, - })?; - let remediation: &str = diagnostics - .first() - .and_then(|diagnostic| diagnostic["remediation"].as_str()) - .ok_or_else(|| format!("{}: no finding", test_case.description))?; - let present: Vec<&str> = test_case - .expected_fragments - .iter() - .copied() - .filter(|fragment| remediation.contains(fragment)) - .collect(); - assert_eq!( - present, test_case.expected_fragments, - "{}: {remediation}", - test_case.description - ); - } - Ok(()) -} diff --git a/crates/sqlbuild-rules/src/sql_quality/tests/test_quality_scale.rs b/crates/sqlbuild-rules/src/sql_quality/tests/test_quality_scale.rs deleted file mode 100644 index b1bf2179f6..0000000000 --- a/crates/sqlbuild-rules/src/sql_quality/tests/test_quality_scale.rs +++ /dev/null @@ -1,43 +0,0 @@ -use std::time::Instant; - -use serde_json::{Value, json}; - -use crate::sql_lint::main::engine::lint_json; -use crate::sql_quality::tests::helpers::wide_cte_chain; -use crate::sql_quality::tests::test_types::QualityScaleTestCase; - -#[test] -fn given_many_wide_ctes_with_findings_when_linting_then_quality_rules_stay_linear() --> Result<(), String> { - let test_cases = [QualityScaleTestCase { - description: "1,500 wide CTEs, each dropping a column and ranking without a key", - cte_count: 1_500, - column_count: 24, - expected_findings: 2_998, - expected_max_seconds: 20.0, - }]; - for test_case in &test_cases { - let request: Value = json!({ - "version": 1, - "sql": wide_cte_chain(test_case.cte_count, test_case.column_count), - "dialect": "duckdb", - "enabled_rules": ["SQBRSQL018", "SQBRSQL042", "SQBRSQL043", "SQBRSQL044"], - }); - let started: Instant = Instant::now(); - let response: Value = serde_json::from_str(&lint_json(&request.to_string())?) - .map_err(|error| error.to_string())?; - let elapsed: f64 = started.elapsed().as_secs_f64(); - assert_eq!( - response["diagnostics"].as_array().map_or(0, Vec::len), - test_case.expected_findings, - "{}", - test_case.description - ); - assert!( - elapsed <= test_case.expected_max_seconds, - "{}: {elapsed:.2}s", - test_case.description - ); - } - Ok(()) -} diff --git a/crates/sqlbuild-rules/src/sql_quality/tests/test_types.rs b/crates/sqlbuild-rules/src/sql_quality/tests/test_types.rs deleted file mode 100644 index bc80b426ff..0000000000 --- a/crates/sqlbuild-rules/src/sql_quality/tests/test_types.rs +++ /dev/null @@ -1,38 +0,0 @@ -pub(crate) struct QualityRuleTestCase { - pub(crate) description: &'static str, - pub(crate) sql: &'static str, - pub(crate) rule: &'static str, - pub(crate) relation_keys: &'static str, - pub(crate) expected_anchors: &'static [&'static str], - pub(crate) expected_fixed_sql: Option<&'static str>, -} - -pub(crate) struct QualityScaleTestCase { - pub(crate) description: &'static str, - pub(crate) cte_count: usize, - pub(crate) column_count: usize, - pub(crate) expected_findings: usize, - pub(crate) expected_max_seconds: f64, -} - -pub(crate) struct QualityRemediationTestCase { - pub(crate) description: &'static str, - pub(crate) sql: &'static str, - pub(crate) rule: &'static str, - pub(crate) expected_fragments: &'static [&'static str], -} - -pub(crate) struct JsonParseTestCase { - pub(crate) description: &'static str, - pub(crate) dialect: &'static str, - pub(crate) sql: &'static str, - pub(crate) expected_anchors: &'static [&'static str], - pub(crate) expected_details: &'static [&'static str], -} - -pub(crate) struct JsonParseScaleTestCase { - pub(crate) description: &'static str, - pub(crate) column_count: usize, - pub(crate) expected_findings: usize, - pub(crate) expected_max_seconds: f64, -} diff --git a/crates/sqlbuild-rules/src/sql_quality/types.rs b/crates/sqlbuild-rules/src/sql_quality/types.rs deleted file mode 100644 index 717e03a5ee..0000000000 --- a/crates/sqlbuild-rules/src/sql_quality/types.rs +++ /dev/null @@ -1,6 +0,0 @@ -use std::collections::BTreeSet; - -/// Column sets that are unique for one relation, by lower-cased output column name. -pub(crate) type Keys = Vec>; -/// A unique column set over a select's joined input, as `(source alias, column)` pairs. -pub(crate) type InputKey = BTreeSet<(String, String)>; diff --git a/fensu.toml b/fensu.toml index 7e2ae7561a..1cf6f755f4 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1569,11 +1569,6 @@ rule = "FPRSS201" path = "crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs" reason = "NullabilityCallback implements Debug manually because it holds the adapter's nullability rule closure, which cannot implement Debug; other models derive it." -[[targets.rust.rule_exceptions]] -rule = "FPRSL105" -path = "crates/sqlbuild-analysis/src/sql_tokens/main/builtin_function_names.rs" -reason = "This test-support entry exists only for the SQL lint tests in sqlbuild-rules, which the analyzer does not count as importers." - [[targets.rust.rule_exceptions]] rule = "FPRSS130" path = "crates/sqlbuild-python/src/bindings/_helpers/native_threads/exit_signal.rs" diff --git a/scripts/cold_compile_performance/_helpers/dense_project.py b/scripts/cold_compile_performance/_helpers/dense_project.py index e651ead7a1..0403ec4395 100644 --- a/scripts/cold_compile_performance/_helpers/dense_project.py +++ b/scripts/cold_compile_performance/_helpers/dense_project.py @@ -76,7 +76,7 @@ def write_dense_compile_project(*, project_dir: Path, model_count: int) -> None: (project_dir / "sqlbuild_project.toml").write_text( 'name = "dense_orders"\nadapter = "snowflake"\n' '[defaults]\ndatabase = "warehouse"\nschema = "analytics"\n' - f'[rules]\nselect = ["SQBR", "XSQBR"]\ndomains = [{domains}]\n', + f'[rules]\nselect = ["SQBR", "SQLCLEAN", "XSQBR"]\ndomains = [{domains}]\n', encoding="utf-8", ) source_count: int = max(1, model_count * 24 // 100) diff --git a/scripts/compiler_differential/_helpers/corpus/failure_cases.py b/scripts/compiler_differential/_helpers/corpus/failure_cases.py index c9a877c914..f09db8c721 100644 --- a/scripts/compiler_differential/_helpers/corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/failure_cases.py @@ -416,11 +416,11 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ), failure_case( name="rule-unused-cte-after-variable-and-macro", - expected_code="SQBRSQL005", + expected_code="SQLCLEAN005", files={ FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + '\n[vars]\nfloor = "2"\n' - + _RULES_CONFIG.format(codes="SQBRSQL005"), + + _RULES_CONFIG.format(codes="SQLCLEAN005"), "models/marts/_macros/python/cents.py": _ENGINE_ERROR_CENTS, **mart_body_files( 'WITH totals AS (SELECT customer_id, @py.cents("amount") AS cents, ' @@ -432,9 +432,9 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ), failure_case( name="rule-unused-cte", - expected_code="SQBRSQL005", + expected_code="SQLCLEAN005", files={ - **_rules("SQBRSQL005"), + **_rules("SQLCLEAN005"), **mart_body_files( "WITH spare AS (SELECT 1 AS one)\n" "SELECT customer_id, SUM(amount) AS total_amount\n" diff --git a/scripts/rules_benchmark/_helpers/workflow.py b/scripts/rules_benchmark/_helpers/workflow.py index 4b39685374..e9ad10ca0a 100644 --- a/scripts/rules_benchmark/_helpers/workflow.py +++ b/scripts/rules_benchmark/_helpers/workflow.py @@ -604,7 +604,7 @@ def _write_project(*, project_dir: Path, model_count: int) -> None: config.write_text( config.read_text(encoding="utf-8") + ( - '\n[rules]\nselect = ["SQBRSQL001", "XSQBR"]\n' + '\n[rules]\nselect = ["SQLCLEAN001", "XSQBR"]\n' "[rules.thresholds]\nmin_custom_rule_test_cases = 0\n" "[rules.cache]\nenabled = true\n" ), diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 4a78cb2ea7..d12632c725 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3352,6 +3352,7 @@ def format_compile_text( use_color: bool, selected_keys: frozenset[CompiledObjectKey] | None = None, sql_validation_enabled: bool = True, + default_rules_notice: str | None = None, ) -> str: ... def format_compile_json( *, @@ -5813,7 +5814,9 @@ class EarlySqlLint: def enabled(self) -> bool: ... def __enter__(self) -> EarlySqlLint: ... def __exit__(self, *exception: object) -> None: ... - def start(self, *, inputs: CompileInputsView, dialect: str) -> None: ... + def start( + self, *, inputs: CompileInputsView, dialect: str, value_renderer: TypedSqlValueRenderer + ) -> None: ... def stop(self) -> int: ... def scan_headers(*, contents: str, first_only: bool = False) -> tuple[HeaderSpan, ...]: ... @@ -15879,7 +15882,7 @@ class LayoutConfig(NativeRecord): ) -> None: ... class RulesConfig(NativeRecord): - select: tuple[str, ...] + select: tuple[str, ...] | None ignore: tuple[str, ...] thresholds: dict[str, int] threshold_overrides: tuple[ThresholdOverride, ...] @@ -15896,7 +15899,7 @@ class RulesConfig(NativeRecord): cache: RulesCacheConfig def __init__( self, - select: tuple[str, ...] = ..., + select: tuple[str, ...] | None = ..., ignore: tuple[str, ...] = ..., thresholds: dict[str, int] = ..., threshold_overrides: tuple[ThresholdOverride, ...] = ..., diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 2c40057a26..2262e3926c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -2893,7 +2893,7 @@ def write_set_operation_project( }[adapter == "duckdb"] files: dict[str, str] = { "sqlbuild_project.toml": ( - f'name = "orders"\nadapter = "{adapter}"\n{target}[rules]\nselect = ["SQBRSQL020"]\n' + f'name = "orders"\nadapter = "{adapter}"\n{target}[rules]\nselect = ["SQLCLEAN020"]\n' ), } for model in models: @@ -2962,7 +2962,7 @@ def run_set_operation_lifecycle( def run_set_arity_rule(*, project_dir: Path) -> subprocess.CompletedProcess[str]: """Run the set-operation arity Rule over every model.""" - return run_sqb(project_dir=project_dir, command=("rules", "run", "SQBRSQL020")) + return run_sqb(project_dir=project_dir, command=("rules", "run", "SQLCLEAN020")) def format_set_operation_models( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py index b80afc7bce..8d3c8d95af 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py @@ -558,7 +558,7 @@ def scoped_model(macros: dict[str, str], body: str) -> dict[str, str]: SQL_RULES: dict[str, str] = { "sqlbuild_project.toml": BASE_FILES["sqlbuild_project.toml"] - + '\n[rules]\nselect = ["SQBRSQL028", "SQBRSQL029"]\n' + + '\n[rules]\nselect = ["SQLCLEAN028", "SQLCLEAN029"]\n' } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_body_rules.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_body_rules.py index f2ec2a0a55..d37353cbd4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_body_rules.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_body_rules.py @@ -32,7 +32,7 @@ }, expected_returncode=0, expected_fragments=("Project compiled",), - expected_absent_fragments=("SQBRSQL",), + expected_absent_fragments=("SQLCLEAN",), ), SqlMacroBodyRulesTestCase( description="plain_single_slot_and_undeclared_reads_through_the_relation_pass", @@ -49,7 +49,7 @@ }, expected_returncode=0, expected_fragments=("Project compiled",), - expected_absent_fragments=("SQBRSQL",), + expected_absent_fragments=("SQLCLEAN",), ), SqlMacroBodyRulesTestCase( description="a_real_mixed_qualification_beside_a_plain_group_read_is_reported", @@ -62,12 +62,12 @@ }, expected_returncode=1, expected_fragments=( - "error[SQBRSQL028]: Single-relation query mixes qualified and unqualified columns " + "error[SQLCLEAN028]: Single-relation query mixes qualified and unqualified columns " "(in SQL generated by @weekly)\n --> macros/sql/weekly_rates.sql:14:18", - "error[SQBRSQL028]: Single-relation query mixes qualified and unqualified columns\n" + "error[SQLCLEAN028]: Single-relation query mixes qualified and unqualified columns\n" " --> macros/sql/weekly_rates.sql:14:42", ), - expected_absent_fragments=("SQBRSQL029",), + expected_absent_fragments=("SQLCLEAN029",), ), SqlMacroBodyRulesTestCase( description="a_qualifier_absent_from_scope_beside_a_plain_group_read_is_reported", @@ -88,11 +88,11 @@ }, expected_returncode=1, expected_fragments=( - "error[SQBRSQL029]: Qualified reference uses a relation absent from this query " + "error[SQLCLEAN029]: Qualified reference uses a relation absent from this query " "scope\n --> macros/sql/uncalled_rates.sql:10:32", ), expected_absent_fragments=( - "error[SQBRSQL029]: Qualified reference uses a relation absent from this query " + "error[SQLCLEAN029]: Qualified reference uses a relation absent from this query " "scope (in SQL generated by @weekly)", "--> macros/sql/weekly_rates.sql", ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_expansion_text.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_expansion_text.py index 9d6c734443..a94e146263 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_expansion_text.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_expansion_text.py @@ -38,7 +38,7 @@ SqlMacroCompileTestCase( description="parameter_before_words_and_a_parenthesis", files={ - **with_rules("SQBRSQL001"), + **with_rules("SQLCLEAN001"), **check_macro("value VARCHAR", "@value IS NULL OR TRIM(@value) = ''"), **_CHECK_STATUS, }, @@ -50,7 +50,7 @@ SqlMacroCompileTestCase( description="parameter_before_in_inside_case", files={ - **with_rules("SQBRSQL001"), + **with_rules("SQLCLEAN001"), **check_macro( "value VARCHAR", "CASE WHEN @value IN ('open', 'closed') THEN TRUE ELSE FALSE END", @@ -67,7 +67,7 @@ SqlMacroCompileTestCase( description="parameter_before_and_a_parenthesised_predicate", files={ - **with_rules("SQBRSQL001"), + **with_rules("SQLCLEAN001"), **check_macro("value VARCHAR", "@value IS NULL AND (@value IS NULL)"), **_CHECK_STATUS, }, @@ -79,7 +79,7 @@ SqlMacroCompileTestCase( description="parameter_before_in_alone", files={ - **with_rules("SQBRSQL001"), + **with_rules("SQLCLEAN001"), **check_macro("value VARCHAR", "@value IN ('', ' ')"), **_CHECK_STATUS, }, @@ -91,7 +91,7 @@ SqlMacroCompileTestCase( description="parameter_compared_then_trimmed", files={ - **with_rules("SQBRSQL001"), + **with_rules("SQLCLEAN001"), **check_macro("value VARCHAR", "@value = '' OR TRIM(@value) = ''"), **_CHECK_STATUS, }, @@ -103,7 +103,7 @@ SqlMacroCompileTestCase( description="parenthesised_parameter", files={ - **with_rules("SQBRSQL001"), + **with_rules("SQLCLEAN001"), **check_macro("value VARCHAR", "(@value) IS NULL OR TRIM(@value) = ''"), **_CHECK_STATUS, }, @@ -115,7 +115,7 @@ SqlMacroCompileTestCase( description="body_opening_with_two_parentheses", files={ - **with_rules("SQBRSQL001"), + **with_rules("SQLCLEAN001"), **check_macro( "value VARCHAR", "((@value) IS NULL AND (@value) IS NULL) OR TRIM(@value) = ''" ), @@ -131,7 +131,7 @@ SqlMacroCompileTestCase( description="spaced_enum_and_constant_references_in_a_body_are_declarations", files={ - **with_rules("SQBRSQL001"), + **with_rules("SQLCLEAN001"), "constants/fee_divisor.sql": "CONSTANT (name fee_divisor, value 100);\n", **check_macro( "value VARCHAR, cents BIGINT", @@ -161,7 +161,7 @@ SqlMacroCompileTestCase( description="two_parameters_before_and_a_parenthesised_predicate", files={ - **with_rules("SQBRSQL001"), + **with_rules("SQLCLEAN001"), **check_macro("x VARCHAR, y VARCHAR", "@x IS NULL AND (@y IS NULL)"), **orders_model( "SELECT order_id, @sql.check(status, country_code) AS checked " @@ -176,18 +176,18 @@ SqlMacroCompileTestCase( description="a_finding_in_a_macro_body_is_reported_in_the_macro_file", files={ - **with_rules("SQBRSQL001"), + **with_rules("SQLCLEAN001"), **check_macro("value VARCHAR", "((@value) IS NULL) OR @value = NULL"), **_CHECK_STATUS, }, expected_returncode=1, - expected_fragments=("[SQBRSQL001]", f"--> {CHECK_MACRO_PATH}:7:"), + expected_fragments=("[SQLCLEAN001]", f"--> {CHECK_MACRO_PATH}:7:"), unexpected_fragments=_RULE_FAILURES, ), SqlMacroCompileTestCase( description="a_cte_list_body_lints_without_parse_or_unused_cte_findings", files={ - **with_rules("SQBRSQL001", "SQBRSQL005"), + **with_rules("SQLCLEAN001", "SQLCLEAN005"), "macros/sql/live_orders.sql": ( 'MACRO (\n description "Live orders",\n' " arguments (src RELATION (order_id BIGINT, is_deleted BOOLEAN)),\n);\n\n" @@ -201,7 +201,7 @@ }, expected_returncode=0, expected_fragments=("Project compiled",), - unexpected_fragments=(*_RULE_FAILURES, "SQBRSQL005"), + unexpected_fragments=(*_RULE_FAILURES, "SQLCLEAN005"), ), ], ids=lambda case: case.description, @@ -375,7 +375,7 @@ def test_given_a_sql_macro_call_in_python_macro_arguments_when_compiling_then_py SqlMacroCompileTestCase( description="comments_from_a_macro_expansion_are_not_judged_in_the_caller", files={ - **with_rules("SQBRSQL033"), + **with_rules("SQLCLEAN033"), **_COMMENTED_CHECK, **orders_model( "SELECT order_id, NOT @sql.check(status) AS plain, " @@ -389,7 +389,7 @@ def test_given_a_sql_macro_call_in_python_macro_arguments_when_compiling_then_py SqlMacroCompileTestCase( description="comments_a_python_macro_returns_are_judged_in_the_caller", files={ - **with_rules("SQBRSQL033"), + **with_rules("SQLCLEAN033"), **WRAPPING_MACROS, **orders_model( "SELECT order_id, @py.noted('status') AS noted FROM __ref(\"stg_orders\")\n" @@ -397,7 +397,7 @@ def test_given_a_sql_macro_call_in_python_macro_arguments_when_compiling_then_py }, expected_returncode=1, expected_fragments=( - "[SQBRSQL033]", + "[SQLCLEAN033]", "(in SQL generated by @py.noted('status'))", "--> models/marts/orders.sql:3:", ), @@ -405,14 +405,14 @@ def test_given_a_sql_macro_call_in_python_macro_arguments_when_compiling_then_py SqlMacroCompileTestCase( description="the_same_comments_written_in_a_model_are_judged", files={ - **with_rules("SQBRSQL033"), + **with_rules("SQLCLEAN033"), **orders_model( "SELECT order_id, NOT -- a status holding an at sign is flagged\n" "CONTAINS(status, /* the sign */ '@') AS plain FROM __ref(\"stg_orders\")\n" ), }, expected_returncode=1, - expected_fragments=("[SQBRSQL033]", "--> models/marts/orders.sql:3:"), + expected_fragments=("[SQLCLEAN033]", "--> models/marts/orders.sql:3:"), ), ], ids=lambda case: case.description, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_relation_slots.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_relation_slots.py index aca782590c..2eea85e385 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_relation_slots.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_relation_slots.py @@ -258,7 +258,7 @@ def test_given_wrong_slot_mappings_when_compiling_then_the_error_names_the_slot( files={ **WEEKLY_ORDERS, "sqlbuild_project.toml": BASE_FILES["sqlbuild_project.toml"] - + '\n[rules]\nselect = ["SQBRSQL001"]\n', + + '\n[rules]\nselect = ["SQLCLEAN001"]\n', "macros/sql/order_columns.sql": ( 'MACRO (\n description "Order count column",\n' " arguments (weekly RELATION (order_count BIGINT)),\n);\n\n" @@ -273,7 +273,7 @@ def test_given_wrong_slot_mappings_when_compiling_then_the_error_names_the_slot( command=("compile",), expected_returncode=1, expected_fragments=( - "error[SQBRSQL001]: Comparison with NULL is never true (in SQL generated by " + "error[SQLCLEAN001]: Comparison with NULL is never true (in SQL generated by " "@sql.order_columns(w, weekly.order_count := orders_placed))", "--> models/marts/order_counts.sql:3:8", ), @@ -283,7 +283,7 @@ def test_given_wrong_slot_mappings_when_compiling_then_the_error_names_the_slot( files={ **WEEKLY_ORDERS, "sqlbuild_project.toml": BASE_FILES["sqlbuild_project.toml"] - + '\n[rules]\nselect = ["SQBRSQL016"]\n', + + '\n[rules]\nselect = ["SQLCLEAN016"]\n', "macros/sql/order_columns.sql": ( 'MACRO (\n description "Order count column",\n' " arguments (weekly RELATION (order_count BIGINT)),\n);\n\n" @@ -306,7 +306,7 @@ def test_given_wrong_slot_mappings_when_compiling_then_the_error_names_the_slot( files={ **WEEKLY_ORDERS, "sqlbuild_project.toml": BASE_FILES["sqlbuild_project.toml"] - + '\n[rules]\nselect = ["SQBRSQL016"]\n', + + '\n[rules]\nselect = ["SQLCLEAN016"]\n', "macros/sql/order_columns.sql": ( 'MACRO (\n description "Order count column",\n' " arguments (weekly RELATION (order_count BIGINT)),\n);\n\n" @@ -322,7 +322,7 @@ def test_given_wrong_slot_mappings_when_compiling_then_the_error_names_the_slot( command=("compile",), expected_returncode=1, expected_fragments=( - "SQBRSQL016]: Expression is aliased to its existing name", + "SQLCLEAN016]: Expression is aliased to its existing name", "--> macros/sql/order_columns.sql:6:", ), ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py index 52d64bb4aa..5786204a87 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_delta.py @@ -204,7 +204,7 @@ def order_folder_siblings(*, model: Model, ctx: RuleContext) -> list[Finding]: # Built-in Rules cannot parse PIVOT models, which would fail the stored compile's Rules. _BUILT_IN_RULES_OFF: tuple[str, str, str] = ( "sqlbuild_project.toml", - 'select = ["SQBR", "XSQBR"]', + 'select = ["SQBR", "SQLCLEAN", "XSQBR"]', 'select = ["XSQBR"]', ) _RULE_READ_MORE: str = "delta compile fell back: a custom rule read more than the changed models" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_lint.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_lint.py index 77d262f4b3..4f29767741 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_lint.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_lint.py @@ -24,7 +24,7 @@ ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project -_PROJECT_TOML: str = 'name = "commerce"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL005"]\n' +_PROJECT_TOML: str = 'name = "commerce"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN005"]\n' _ORDERS_SQL: str = 'MODEL (description "Orders");\nSELECT 1 AS order_id, 2 AS amount\n' @@ -59,9 +59,9 @@ ), expected_exit_code=1, expected_diagnostics=( - ("SQBRSQL005", "models/order_totals.sql", 3, 3), - ("SQBRSQL005", "models/order_totals.sql", 6, 3), - ("SQBRSQL005", "models/order_totals.sql", 9, 3), + ("SQLCLEAN005", "models/order_totals.sql", 3, 3), + ("SQLCLEAN005", "models/order_totals.sql", 6, 3), + ("SQLCLEAN005", "models/order_totals.sql", 9, 3), ), ), ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_parameterised_test_lint.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_parameterised_test_lint.py index f1d374bb3a..b379042ad2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_parameterised_test_lint.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_parameterised_test_lint.py @@ -44,13 +44,13 @@ [ ParameterisedTestLintTestCase( description="compile lints the parameterised test instead of failing with L005", - selected_rules='"SQBRSQL004"', + selected_rules='"SQLCLEAN004"', command=("--no-color", "compile"), expected_fragment="0 errors", ), ParameterisedTestLintTestCase( description="the cases still run once the test file is linted", - selected_rules='"SQBRSQL004"', + selected_rules='"SQLCLEAN004"', command=("--no-color", "test"), expected_fragment="PASS=2 FAIL=0 TOTAL=2", ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/format/test_rule_fixes.py b/tests/e2e/src/sqlbuild/cli/commands/main/format/test_rule_fixes.py index 08cbb50548..b4da626db9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/format/test_rule_fixes.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/format/test_rule_fixes.py @@ -23,7 +23,7 @@ "verification failure", 'MODEL (description "Test model.", sql_analysis false);\nWITH unused AS (SELECT 1 AS item) SELECT 2 AS item', "refused", - "SQBRSQL005", + "SQLCLEAN005", 1, True, True, @@ -33,7 +33,7 @@ 'MODEL (description "Test model.");\n' 'WITH unused AS (SELECT customer_id FROM __ref("customers")) SELECT 2 AS item', "refused", - "SQBRSQL005", + "SQLCLEAN005", 0, False, True, @@ -44,7 +44,7 @@ "WITH first AS (SELECT 1 AS item), second AS (SELECT item FROM first), " "third AS (SELECT item FROM second) SELECT 2 AS item", "applied", - "SQBRSQL005", + "SQLCLEAN005", 0, False, False, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_unicode_empty_fixture.py b/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_unicode_empty_fixture.py index 3a35161fe5..610ab526ae 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_unicode_empty_fixture.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_unicode_empty_fixture.py @@ -39,14 +39,14 @@ def test_given_unicode_playground_fixture_when_running_rules_and_format_then_emp ) fixture.write_text(sql, encoding="utf-8") baseline: subprocess.CompletedProcess[str] = run_sqb( - command=("rules", "--json", "run", "SQBRSQL021"), project_dir=project + command=("rules", "--json", "run", "SQLCLEAN021"), project_dir=project ) baseline_count: int = json.loads(baseline.stdout)["finding_count"] fixture.write_text( sql.replace("'completed' AS status", f"'{test_case.status}' AS status"), encoding="utf-8" ) before: subprocess.CompletedProcess[str] = run_sqb( - command=("rules", "--json", "run", "SQBRSQL021"), project_dir=project + command=("rules", "--json", "run", "SQLCLEAN021"), project_dir=project ) assert before.returncode == baseline.returncode, before.stdout + before.stderr assert ( @@ -62,7 +62,7 @@ def test_given_unicode_playground_fixture_when_running_rules_and_format_then_emp assert test_case.status in formatted assert "__EMPTY_FIXTURE()" in formatted after: subprocess.CompletedProcess[str] = run_sqb( - command=("rules", "--json", "run", "SQBRSQL021"), project_dir=project + command=("rules", "--json", "run", "SQLCLEAN021"), project_dir=project ) assert after.returncode == baseline.returncode, after.stdout + after.stderr assert ( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index bd6089a214..26dc9feaef 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -48,7 +48,7 @@ class UnevaluatedResourceCase: expected_reason: str = "E_GUARD_FUNCTION_NESTING_DEPTH_EXCEEDED" adapter: str = "duckdb" extra_files: tuple[tuple[str, str], ...] = () - rule: str = "SQBRSQL035" + rule: str = "SQLCLEAN035" @dataclass(frozen=True) @@ -356,3 +356,14 @@ class ClickHouseCommentE2ETestCase: description: str comment: str expected_location: str + + +@dataclass(frozen=True) +class DefaultRuleSelectionCase: + description: str + rules_section: str + sql: str + command: tuple[str, ...] + expected_exit: int + expected_finding: bool + expected_notices: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py index f9cf7dd094..42c1bcbcbc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py @@ -23,7 +23,7 @@ def write_unevaluated_rules_project( model_options: str = "", configuration: str = "", adapter: str = "duckdb", - selected_rules: tuple[str, ...] = ("SQBRSQL035",), + selected_rules: tuple[str, ...] = ("SQLCLEAN035",), resource_path: str = "models/staging/orders.sql", resource_template: str = ( "MODEL (description 'Test model orders.', {options});\nSELECT {expression} AS order_id" @@ -317,7 +317,7 @@ def help_fragment_presence( def long_literal_hints(result: subprocess.CompletedProcess[str]) -> dict[str, str]: - """Map each finding path of a SQBRSQL044-only run to the help before the generic guidance.""" + """Map each finding path of a SQLCLEAN044-only run to the help before the generic guidance.""" return { str(item["path"]): str(item["remediation"]).partition("Define the value once")[0].strip() @@ -331,7 +331,7 @@ def long_literal_hints(result: subprocess.CompletedProcess[str]) -> dict[str, st '[connection]\ndatabase = "warehouse.duckdb"\n\n' '[targets.dev]\nschema = "dev"\n\n[targets.prod]\nschema = "prod"\n\n' '[vars]\ndiscount_rate = "0.1"\n\n' - '[rules]\nselect = ["SQBR", "XSQBR"]\n\n' + '[rules]\nselect = ["SQBR", "SQLCLEAN", "XSQBR"]\n\n' "[rules.thresholds]\nmin_custom_rule_test_cases = 0\n" ), "models/staging/stg_orders.sql": ( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_cross_join_one_row.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_cross_join_one_row.py index 7b9d8ca95f..9289e03b24 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_cross_join_one_row.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_cross_join_one_row.py @@ -1,4 +1,4 @@ -"""E2E coverage of SQBRSQL038 inferring that one side of a CROSS JOIN holds at most one row.""" +"""E2E coverage of SQLCLEAN038 inferring that one side of a CROSS JOIN holds at most one row.""" from __future__ import annotations @@ -14,7 +14,7 @@ write_one_row_cross_join_project, ) -_PROJECT_TOML: str = 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL038"]\n' +_PROJECT_TOML: str = 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN038"]\n' _SEED_YAML: str = ( "seeds:\n" " - name: order_policy\n" @@ -141,7 +141,7 @@ def test_given_cross_join_when_compiling_then_only_multiplying_joins_are_reporte result: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path) - assert diagnostic_messages(result, "SQBRSQL038") == test_case.expected_messages, ( + assert diagnostic_messages(result, "SQLCLEAN038") == test_case.expected_messages, ( result.stdout + result.stderr ) assert result.returncode == test_case.expected_exit, result.stderr @@ -180,8 +180,8 @@ def test_given_one_row_seed_when_a_row_is_added_then_the_cached_compile_reports_ second: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path) assert first.returncode == 0, first.stdout + first.stderr - assert diagnostic_messages(first, "SQBRSQL038") == () - assert diagnostic_messages(second, "SQBRSQL038") == test_case.expected_messages, ( + assert diagnostic_messages(first, "SQLCLEAN038") == () + assert diagnostic_messages(second, "SQLCLEAN038") == test_case.expected_messages, ( second.stdout + second.stderr ) assert second.returncode == test_case.expected_exit diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_default_rule_selection.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_default_rule_selection.py new file mode 100644 index 0000000000..5261589d0b --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_default_rule_selection.py @@ -0,0 +1,117 @@ +"""E2E coverage of omitted versus explicit `[rules] select` and the default Rules notice.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import DefaultRuleSelectionCase +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_PROJECT_TOML: str = ( + 'name = "shop"\nadapter = "duckdb"\n\n[connection]\ndatabase = "warehouse.duckdb"\n' +) +_HEADER: str = "MODEL (description 'Test model orders.', materialized table);\n" +_NULL_COMPARISON_SQL: str = "WITH o AS (SELECT 1 AS id)\nSELECT o.id FROM o WHERE o.id = NULL\n" +_CLEAN_SQL: str = "SELECT 1 AS id\n" +_NOTICE: str = "Using default rules. To disable configurable checks, add to sqlbuild_project.toml:" +_FINDING: str = "SQLCLEAN001" + + +@pytest.mark.parametrize( + "test_case", + [ + DefaultRuleSelectionCase( + description="omitted select runs the default Rules and notes how to turn them off", + rules_section="", + sql=_NULL_COMPARISON_SQL, + command=("compile", "--no-cache"), + expected_exit=1, + expected_finding=True, + expected_notices=1, + ), + DefaultRuleSelectionCase( + description="omitted select notes the defaults once when build stops on a finding", + rules_section="", + sql=_NULL_COMPARISON_SQL, + command=("build",), + expected_exit=1, + expected_finding=True, + expected_notices=1, + ), + DefaultRuleSelectionCase( + description="explicit empty select turns off configurable Rules", + rules_section="\n[rules]\nselect = []\n", + sql=_NULL_COMPARISON_SQL, + command=("compile", "--no-cache"), + expected_exit=0, + expected_finding=False, + expected_notices=0, + ), + DefaultRuleSelectionCase( + description="explicit selection reports its finding without the notice", + rules_section='\n[rules]\nselect = ["SQLCLEAN001"]\n', + sql=_NULL_COMPARISON_SQL, + command=("compile", "--no-cache"), + expected_exit=1, + expected_finding=True, + expected_notices=0, + ), + DefaultRuleSelectionCase( + description="omitted select on clean SQL prints no notice", + rules_section="", + sql=_CLEAN_SQL, + command=("compile", "--no-cache"), + expected_exit=0, + expected_finding=False, + expected_notices=0, + ), + DefaultRuleSelectionCase( + description="JSON compile output carries the finding without the notice", + rules_section="", + sql=_NULL_COMPARISON_SQL, + command=("compile", "--no-cache", "--json"), + expected_exit=1, + expected_finding=True, + expected_notices=0, + ), + DefaultRuleSelectionCase( + description="JSON plan error carries the finding without the notice", + rules_section="", + sql=_NULL_COMPARISON_SQL, + command=("plan", "--json"), + expected_exit=1, + expected_finding=True, + expected_notices=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_rule_selection_when_running_commands_then_defaults_and_notice_follow_it( + test_case: DefaultRuleSelectionCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML + test_case.rules_section, + "models/orders.sql": _HEADER + test_case.sql, + }, + ) + + ran: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", *test_case.command), project_dir=project_dir + ) + + output: str = ran.stdout + ran.stderr + assert (ran.returncode, _FINDING in output, output.count(_NOTICE)) == ( + test_case.expected_exit, + test_case.expected_finding, + test_case.expected_notices, + ), output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_exception_config.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_exception_config.py index 2d3d84d997..b23a1409ee 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_exception_config.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_exception_config.py @@ -15,13 +15,15 @@ write_model_files, ) -_PROJECT_TOML: str = 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL038"]\n' +_PROJECT_TOML: str = 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN038"]\n' _MODEL_PATH: str = "models/order_checks.sql" _CUSTOMERS: str = ( "CROSS JOIN with `customers` in CTE `order_checks` creates an explicit cartesian product" ) _REGIONS: str = "CROSS JOIN with `regions` in CTE `final` creates an explicit cartesian product" -_EXCEPTION_HEAD: str = f'\n[[rules.rule_exceptions]]\nrule = "SQBRSQL038"\npath = "{_MODEL_PATH}"\n' +_EXCEPTION_HEAD: str = ( + f'\n[[rules.rule_exceptions]]\nrule = "SQLCLEAN038"\npath = "{_MODEL_PATH}"\n' +) _REASON: str = 'reason = "Every order is checked for every customer and region."\n' _CUSTOMERS_EXCEPTION: str = f'{_EXCEPTION_HEAD}relation = "customers"\n{_REASON}' _REGIONS_EXCEPTION: str = f'{_EXCEPTION_HEAD}relation = "regions"\n{_REASON}' @@ -45,15 +47,15 @@ ) _PLAIN_SQL: str = _MODEL_SQL.format(directive="") _DIRECTIVE_SQL: str = _MODEL_SQL.format( - directive=" -- sqb: ignore SQBRSQL038 because every order meets every customer\n" + directive=" -- sqb: ignore SQLCLEAN038 because every order meets every customer\n" ) _LITERAL_SQL: str = _PLAIN_SQL.replace( "SELECT f.order_id, f.customer_id, f.region_id FROM final AS f", - "SELECT f.order_id, f.customer_id, f.region_id, 'sqb: ignore SQBRSQL038' AS note FROM final AS f", + "SELECT f.order_id, f.customer_id, f.region_id, 'sqb: ignore SQLCLEAN038' AS note FROM final AS f", ) _TRAILING_SQL: str = _PLAIN_SQL.replace( ' CROSS JOIN __ref("customers") AS c\n', - ' CROSS JOIN __ref("customers") AS c -- sqb: ignore SQBRSQL038 because deliberate\n', + ' CROSS JOIN __ref("customers") AS c -- sqb: ignore SQLCLEAN038 because deliberate\n', ) _UNNAMED_SQL: str = ( "MODEL (description 'Order checks.');\n" @@ -98,7 +100,7 @@ model_sql=_PLAIN_SQL, expected_exit=1, expected_fragments=( - "stale rule exception suppresses no finding: SQBRSQL038 at " + "stale rule exception suppresses no finding: SQLCLEAN038 at " "models/order_checks.sql (relation shipments)", ), ), @@ -110,7 +112,7 @@ expected_messages=(_CUSTOMERS, _REGIONS), expected_fragments=( "Inline Rule suppressions are not supported", - '[[rules.rule_exceptions]]\n rule = "SQBRSQL038"\n' + '[[rules.rule_exceptions]]\n rule = "SQLCLEAN038"\n' ' path = "models/order_checks.sql"\n' ' relation = "customers"\n' ' reason = "every order meets every customer"', @@ -142,7 +144,7 @@ "only a file-wide exception (rule and path) can cover it", ), expected_fragments=( - '[[rules.rule_exceptions]]\n rule = "SQBRSQL038"\n' + '[[rules.rule_exceptions]]\n rule = "SQLCLEAN038"\n' ' path = "models/order_checks.sql"\n', ), ), @@ -173,7 +175,7 @@ def test_given_rule_exception_config_when_compiling_then_only_named_findings_are text: str = diagnostic_text(result) assert result.returncode == test_case.expected_exit, text - assert diagnostic_messages(result, "SQBRSQL038") == test_case.expected_messages, text + assert diagnostic_messages(result, "SQLCLEAN038") == test_case.expected_messages, text for fragment in test_case.expected_fragments: assert fragment in text diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_performance.py index 0b5c329f1d..27657fd2d9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rules_performance.py @@ -49,7 +49,7 @@ def test_given_advanced_project_and_disabled_cache_when_running_rules_then_finis config_path: Path = project_dir / "sqlbuild_project.toml" config_path.write_text( config_path.read_text(encoding="utf-8") - + ('\n[rules]\nselect = ["SQBR", "XSQBR"]\n[rules.cache]\nenabled = true\n'), + + ('\n[rules]\nselect = ["SQBR", "SQLCLEAN", "XSQBR"]\n[rules.cache]\nenabled = true\n'), encoding="utf-8", ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py index 76d4d67345..09961d554d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py @@ -30,7 +30,7 @@ _PROJECT_TOML: str = ( 'name = "orders"\nadapter = "duckdb"\n\n[rules]\n' - 'select = ["SQBRSQL018", "SQBRSQL042", "SQBRSQL043", "SQBRSQL044"]\n\n' + 'select = ["SQLCLEAN018", "SQLCLEAN042", "SQLCLEAN043", "SQLCLEAN044"]\n\n' "[rules.thresholds]\nmax_literal_length = 30\nmax_ranking_order_by = 2\n" ) _UPSTREAM_SQL: str = ( @@ -135,12 +135,12 @@ "SELECT f.order_id, f.label, f.amount_rank, f.stable_rank FROM final AS f\n" ), expected_findings=( - ("SQBRSQL042", 3), - ("SQBRSQL044", 9), - ("SQBRSQL018", 10), - ("SQBRSQL043", 11), + ("SQLCLEAN042", 3), + ("SQLCLEAN044", 9), + ("SQLCLEAN018", 10), + ("SQLCLEAN043", 11), ), - expected_after_fix=(("SQBRSQL044", 16), ("SQBRSQL018", 19), ("SQBRSQL043", 20)), + expected_after_fix=(("SQLCLEAN044", 16), ("SQLCLEAN018", 19), ("SQLCLEAN043", 20)), ), SqlQualityRuleCase( description="a ranking order without a declared key cannot be proven", @@ -157,8 +157,8 @@ ")\n" "SELECT f.order_id, f.rn FROM final AS f\n" ), - expected_findings=(("SQBRSQL018", 7),), - expected_after_fix=(("SQBRSQL018", 12),), + expected_findings=(("SQLCLEAN018", 7),), + expected_after_fix=(("SQLCLEAN018", 12),), ), ), ids=lambda case: case.description, @@ -226,7 +226,7 @@ def test_given_quality_rule_violations_when_compiling_and_fixing_then_findings_a ), upstream_sql=_CURSOR_INCREMENTAL_SQL, query_sql=_RENAMED_CHAIN_SQL, - expected_codes=("SQBRSQL018",), + expected_codes=("SQLCLEAN018",), extra_files=(("sources/landing_events.yml", _CURSOR_SOURCE_YML),), ), RankingKeyProofCase( @@ -234,14 +234,14 @@ def test_given_quality_rule_violations_when_compiling_and_fixing_then_findings_a upstream_header=_SNAPSHOT_HEADER, upstream_sql=_INCREMENTAL_SQL, query_sql=_RENAMED_CHAIN_SQL, - expected_codes=("SQBRSQL018",), + expected_codes=("SQLCLEAN018",), ), RankingKeyProofCase( description="a unique audit on a nullable column is not a key", upstream_header='MODEL (description "Raw orders", columns (order_id (audits [unique])));\n', upstream_sql=_UPSTREAM_SQL, query_sql=_RENAMED_CHAIN_SQL, - expected_codes=("SQBRSQL018",), + expected_codes=("SQLCLEAN018",), ), RankingKeyProofCase( description="a unique audit on a nullable false column is a key", @@ -272,7 +272,7 @@ def test_given_quality_rule_violations_when_compiling_and_fixing_then_findings_a 'FROM __ref("raw_orders") AS r\n' "QUALIFY ROW_NUMBER() OVER (PARTITION BY r.customer_id ORDER BY r.amount DESC) = 1\n" ), - expected_codes=("SQBRSQL018",), + expected_codes=("SQLCLEAN018",), ), RankingKeyProofCase( description="an upstream hashed row key with a unique audit is a tie-breaker", @@ -299,7 +299,7 @@ def test_given_quality_rule_violations_when_compiling_and_fixing_then_findings_a upstream_header='MODEL (description "Raw orders");\n', upstream_sql=_UPSTREAM_SQL, query_sql=_RENAMED_CHAIN_SQL, - expected_codes=("SQBRSQL018",), + expected_codes=("SQLCLEAN018",), ), ), ids=lambda case: case.description, @@ -335,7 +335,7 @@ def test_given_declared_upstream_keys_when_compiling_then_ranking_proof_follows_ RankingSortDefaultCase( description="six sort keys exceed the default", order_by="o.amount, o.status, o.note, o.customer_id, o.amount + 1, o.order_id", - expected_codes=("SQBRSQL043",), + expected_codes=("SQLCLEAN043",), ), ), ids=lambda case: case.description, @@ -345,7 +345,7 @@ def test_given_default_ranking_limit_when_compiling_then_five_sort_keys_are_allo ) -> None: write_quality_project( project_dir=tmp_path, - toml='name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL043"]\n', + toml='name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN043"]\n', upstream=_RAW_HEADER + _UPSTREAM_SQL, summary=( _SUMMARY_HEADER + "SELECT o.order_id," @@ -379,20 +379,20 @@ def test_given_default_ranking_limit_when_compiling_then_five_sort_keys_are_allo "SELECT f.order_id, f.label, f.rn FROM final AS f\n" ), expected_fixability=( - ("SQBRSQL042", True), - ("SQBRSQL043", False), - ("SQBRSQL044", False), + ("SQLCLEAN042", True), + ("SQLCLEAN043", False), + ("SQLCLEAN044", False), ), expected_notes=( - ("SQBRSQL042", (_FIX_NOTE,)), - ("SQBRSQL043", ()), - ("SQBRSQL044", ()), + ("SQLCLEAN042", (_FIX_NOTE,)), + ("SQLCLEAN043", ()), + ("SQLCLEAN044", ()), ), expected_help_fragments=( - ("SQBRSQL043", "(the current value is 2)"), - ("SQBRSQL043", "[rules.thresholds]\n max_ranking_order_by = 3"), - ("SQBRSQL044", '@const("_status_pattern")'), - ("SQBRSQL044", "[rules.thresholds]\n max_literal_length = 39"), + ("SQLCLEAN043", "(the current value is 2)"), + ("SQLCLEAN043", "[rules.thresholds]\n max_ranking_order_by = 3"), + ("SQLCLEAN044", '@const("_status_pattern")'), + ("SQLCLEAN044", "[rules.thresholds]\n max_literal_length = 39"), ), ), ), @@ -409,10 +409,10 @@ def test_given_fixable_and_manual_findings_when_reporting_then_only_fixable_ones ) rules_json: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( - tmp_path, "rules", "--json", "run", "SQBRSQL" + tmp_path, "rules", "--json", "run", "SQLCLEAN" ) rules_text: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( - tmp_path, "rules", "run", "SQBRSQL" + tmp_path, "rules", "run", "SQLCLEAN" ) compiled: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( tmp_path, "compile", "--json", "--no-cache" @@ -453,7 +453,7 @@ def test_given_fixable_and_manual_findings_when_reporting_then_only_fixable_ones ' FROM __ref("raw_orders") AS o\n' ), expected_exit=1, - expected_codes=("SQBRSQL044",), + expected_codes=("SQLCLEAN044",), ), ), ids=lambda case: case.description, @@ -548,7 +548,7 @@ def test_given_repeated_long_literal_when_running_rules_then_help_names_the_shar model.write_text(f"{_SUMMARY_HEADER}SELECT {literal} AS label\n", encoding="utf-8") result: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( - tmp_path, "rules", "--json", "run", "SQBRSQL044" + tmp_path, "rules", "--json", "run", "SQLCLEAN044" ) hints: dict[str, str] = long_literal_hints(result) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_rule_paths.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_rule_paths.py index ec174f93d5..35561375d0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_rule_paths.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_rule_paths.py @@ -20,7 +20,7 @@ description="macro, CRLF model and CRLF SQL test", files={ "sqlbuild_project.toml": ( - 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL"]\n' + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN"]\n' ), "models/staging/orders.sql": ( 'MODEL (description "Orders");\n' @@ -44,15 +44,15 @@ ), }, expected_findings=( - ("models/staging/customers.sql", 2, 6, "SQBRSQL005"), - ("models/staging/customers.sql", 2, 6, "SQBRSQL041"), - ("models/staging/customers.sql", 3, 1, "SQBRSQL035"), - ("models/staging/orders.sql", 2, 6, "SQBRSQL005"), - ("models/staging/orders.sql", 2, 47, "SQBRSQL041"), - ("models/staging/orders.sql", 3, 1, "SQBRSQL034"), - ("models/staging/orders.sql", 3, 1, "SQBRSQL035"), - ("tests/unit/test_orders.sql", 4, 1, "SQBRSQL005"), - ("tests/unit/test_orders.sql", 5, 59, "SQBRSQL004"), + ("models/staging/customers.sql", 2, 6, "SQLCLEAN005"), + ("models/staging/customers.sql", 2, 6, "SQLCLEAN041"), + ("models/staging/customers.sql", 3, 1, "SQLCLEAN035"), + ("models/staging/orders.sql", 2, 6, "SQLCLEAN005"), + ("models/staging/orders.sql", 2, 47, "SQLCLEAN041"), + ("models/staging/orders.sql", 3, 1, "SQLCLEAN034"), + ("models/staging/orders.sql", 3, 1, "SQLCLEAN035"), + ("tests/unit/test_orders.sql", 4, 1, "SQLCLEAN005"), + ("tests/unit/test_orders.sql", 5, 59, "SQLCLEAN004"), ), ) ], @@ -73,7 +73,7 @@ def test_given_sql_findings_when_compiling_or_running_rules_then_findings_are_id command=("--no-color", "compile", "--json", "--no-cache"), project_dir=compile_dir ) ruled: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "rules", "--json", "run", "SQBRSQL"), project_dir=rules_dir + command=("--no-color", "rules", "--json", "run", "SQLCLEAN"), project_dir=rules_dir ) compile_payload: dict[str, Any] = json.loads(compiled.stdout) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py index ddb2405d1e..abdf5bf998 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py @@ -183,7 +183,7 @@ def test_given_prefix_not_like_when_running_rules_then_evaluates_the_complete_mo "rules", "--json", "run", - "SQBRSQL035", + "SQLCLEAN035", ) assert result.returncode == 0, result.stdout + result.stderr payload: dict[str, Any] = json.loads(result.stdout) @@ -213,7 +213,7 @@ def test_given_prefix_not_like_when_running_rules_then_evaluates_the_complete_mo ), UnevaluatedRuleCase( "explicit scoped rule ignore", - configuration='\n[[rules.rule_ignores]]\nrules = ["SQBRSQL035"]\npaths = ["models/staging/orders.sql"]\nreason = "External SQL syntax"\n', + configuration='\n[[rules.rule_ignores]]\nrules = ["SQLCLEAN035"]\npaths = ["models/staging/orders.sql"]\nreason = "External SQL syntax"\n', expected_exit=0, ), ], @@ -229,7 +229,7 @@ def test_given_explicit_escape_hatch_when_running_rules_then_honors_policy( configuration=test_case.configuration, ) result: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( - tmp_path, "rules", "--json", "run", "SQBRSQL035" + tmp_path, "rules", "--json", "run", "SQLCLEAN035" ) assert result.returncode == test_case.expected_exit, result.stdout + result.stderr payload: dict[str, Any] = json.loads(result.stdout) @@ -249,20 +249,20 @@ def test_given_explicit_escape_hatch_when_running_rules_then_honors_policy( "compile groups two Rules", ("compile", "--json", "--no-cache"), "diagnostics", - ("SQBRSQL034", "SQBRSQL035"), + ("SQLCLEAN034", "SQLCLEAN035"), ), GroupedRulesCase( "Rules CLI groups selected family", - ("rules", "--json", "run", "SQBRSQL"), + ("rules", "--json", "run", "SQLCLEAN"), "findings", - tuple(f"SQBRSQL{number:03}" for number in range(1, 46)), + tuple(f"SQLCLEAN{number:03}" for number in range(1, 54)), ), GroupedRulesCase( "suppression precedes grouping", ("compile", "--json", "--no-cache"), "diagnostics", - ("SQBRSQL034",), - '\n[[rules.rule_ignores]]\nrules = ["SQBRSQL035"]\npaths = ["models/staging/orders.sql"]\nreason = "External SQL syntax"\n', + ("SQLCLEAN034",), + '\n[[rules.rule_ignores]]\nrules = ["SQLCLEAN035"]\npaths = ["models/staging/orders.sql"]\nreason = "External SQL syntax"\n', ), ], ids=lambda case: case.description, @@ -273,7 +273,7 @@ def test_given_one_failed_resource_and_multiple_rules_when_running_cli_then_grou ) -> None: write_unevaluated_rules_project( project_dir=tmp_path, - selected_rules=("SQBRSQL034", "SQBRSQL035"), + selected_rules=("SQLCLEAN034", "SQLCLEAN035"), configuration=test_case.configuration, ) for _ in range(2): diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unused_cte_contexts.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unused_cte_contexts.py index 5423fcdc15..36048d3422 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unused_cte_contexts.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unused_cte_contexts.py @@ -195,9 +195,9 @@ def test_given_framework_cte_context_when_running_rule_then_live_cte_is_not_repo target.write_text(contents, encoding="utf-8") result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "rules", "run", "SQBRSQL005"), + command=("--no-color", "rules", "run", "SQLCLEAN005"), project_dir=project_dir, ) assert result.returncode == test_case.expected_returncode, result.stdout + result.stderr - assert result.stdout.count("[SQBRSQL005]") == test_case.expected_finding_count + assert result.stdout.count("[SQLCLEAN005]") == test_case.expected_finding_count diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py index 0cc7b5aa84..daa26ee567 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py @@ -683,7 +683,7 @@ def build_empty_fixture_scenario_project_files(*, customer_columns_yaml: str) -> "[defaults]\n" 'materialized = "table"\n\n' "[rules]\n" - 'select = ["SQBRSQL021"]\n' + 'select = ["SQLCLEAN021"]\n' ), "sources/raw.yml": ( "sources:\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_empty_fixture.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_empty_fixture.py index fc9df8604f..925b7ec1be 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_empty_fixture.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_empty_fixture.py @@ -63,14 +63,14 @@ def test_given_empty_fixture_scenario_when_running_rules_and_scenario_then_types ) rules: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "rules", "run", "SQBRSQL021"), project_dir=project_dir + command=("--no-color", "rules", "run", "SQLCLEAN021"), project_dir=project_dir ) scenario: subprocess.CompletedProcess[str] = run_sqb( command=("--no-color", "scenario", "test"), project_dir=project_dir ) assert rules.returncode == 0, rules.stdout + rules.stderr - assert "SQBRSQL021" not in rules.stdout.replace("Rule selection SQBRSQL021 passed", "") + assert "SQLCLEAN021" not in rules.stdout.replace("Rule selection SQLCLEAN021 passed", "") assert scenario.returncode == test_case.expected_exit_code, scenario.stdout + scenario.stderr for fragment in test_case.expected_fragments: assert fragment in scenario.stdout, scenario.stdout diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_macro_mock_alias.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_macro_mock_alias.py index 39d0fbf094..9b7cb84400 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_macro_mock_alias.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_macro_mock_alias.py @@ -1,4 +1,4 @@ -"""E2E coverage of macro mocks that name their value, as SQBRSQL022 asks.""" +"""E2E coverage of macro mocks that name their value, as SQLCLEAN022 asks.""" from __future__ import annotations @@ -12,7 +12,7 @@ _PROJECT_TOML: str = ( 'name = "shop"\nadapter = "duckdb"\n\n[connection]\ndatabase = "shop.duckdb"\n\n' - '[scopes]\nenforce_placement = false\n\n[rules]\nselect = ["SQBRSQL022"]\n' + '[scopes]\nenforce_placement = false\n\n[rules]\nselect = ["SQLCLEAN022"]\n' ) _REGION_MACRO: str = ( "MACRO (\n description \"The sales region filter\",\n returns BOOLEAN,\n);\n\nregion = 'NA'\n" @@ -38,7 +38,7 @@ "test_case", [ MacroMockAliasTestCase( - description="an aliased mock satisfies SQBRSQL022", + description="an aliased mock satisfies SQLCLEAN022", mock_select="SELECT 'region = ''EU''' AS region_filter", command=("--no-color", "compile"), expected_exit_code=0, @@ -52,11 +52,11 @@ expected_fragment="PASS=1 FAIL=0", ), MacroMockAliasTestCase( - description="an unaliased mock is reported by SQBRSQL022", + description="an unaliased mock is reported by SQLCLEAN022", mock_select="SELECT 'region = ''EU'''", command=("--no-color", "compile"), expected_exit_code=1, - expected_fragment="SQBRSQL022", + expected_fragment="SQLCLEAN022", ), ], ids=lambda case: case.description, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_optional_test_ceremony.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_optional_test_ceremony.py index 9a3e9fed7a..fc2789a8cc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_optional_test_ceremony.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_optional_test_ceremony.py @@ -81,7 +81,7 @@ def test_given_optional_test_ceremony_when_testing_then_short_and_explicit_forms command=("--no-color", "test"), project_dir=project_dir ) linted: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "rules", "run", "SQBRSQL"), project_dir=project_dir + command=("--no-color", "rules", "run", "SQLCLEAN"), project_dir=project_dir ) formatted: subprocess.CompletedProcess[str] = run_sqb( command=("--no-color", "format"), project_dir=project_dir diff --git a/tests/goldens/compiler/failure/rule-unused-cte-after-variable-and-macro.json b/tests/goldens/compiler/failure/rule-unused-cte-after-variable-and-macro.json index 19fd87eae7..5b2ab41638 100644 --- a/tests/goldens/compiler/failure/rule-unused-cte-after-variable-and-macro.json +++ b/tests/goldens/compiler/failure/rule-unused-cte-after-variable-and-macro.json @@ -7,7 +7,7 @@ { "phase": "rule", "severity": "error", - "code": "SQBRSQL005", + "code": "SQLCLEAN005", "message": "CTE is unreachable from the final query", "path": "models/marts/customer_totals.sql", "line": 5, @@ -29,7 +29,7 @@ "compiled_failed": { "models/marts/customer_totals.sql": [ "-- sqb: compile failed for this model; not buildable", - "-- SQBRSQL005 at models/marts/customer_totals.sql:5:117: CTE is unreachable from the final query", + "-- SQLCLEAN005 at models/marts/customer_totals.sql:5:117: CTE is unreachable from the final query", "WITH totals AS (SELECT customer_id, (amount * 100) AS cents, 2 AS floor_cents FROM __ref(\"stg_orders\")), spare AS (SELECT 1 AS one)", "SELECT customer_id, SUM(cents) AS total_amount", "FROM totals WHERE cents > floor_cents", diff --git a/tests/goldens/compiler/failure/rule-unused-cte.json b/tests/goldens/compiler/failure/rule-unused-cte.json index 7146a6cfca..b9c998fdd2 100644 --- a/tests/goldens/compiler/failure/rule-unused-cte.json +++ b/tests/goldens/compiler/failure/rule-unused-cte.json @@ -7,7 +7,7 @@ { "phase": "rule", "severity": "error", - "code": "SQBRSQL005", + "code": "SQLCLEAN005", "message": "CTE is unreachable from the final query", "path": "models/marts/customer_totals.sql", "line": 5, @@ -29,7 +29,7 @@ "compiled_failed": { "models/marts/customer_totals.sql": [ "-- sqb: compile failed for this model; not buildable", - "-- SQBRSQL005 at models/marts/customer_totals.sql:5:6: CTE is unreachable from the final query", + "-- SQLCLEAN005 at models/marts/customer_totals.sql:5:6: CTE is unreachable from the final query", "WITH spare AS (SELECT 1 AS one)", "SELECT customer_id, SUM(amount) AS total_amount", "FROM __ref(\"stg_orders\")", diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index d858fb7f28..4857d0e4eb 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -17,6 +17,7 @@ class SemanticFixTestCase: expected_code: str dialect: str = "duckdb" expected_status: str = "applied" + expected_exit_code: int = 0 @dataclass(frozen=True) @@ -101,10 +102,10 @@ class DerivedNativeCompileTestCase: @dataclass(frozen=True) class KeywordFunctionCompileTestCase: - """Expected compile result for a DuckDB SQL keyword function projection.""" + """Expected compile result for a DuckDB SQL keyword function query.""" description: str - projection: str + query: str expected_exit_code: int expected_diagnostics: tuple[str, ...] @@ -327,7 +328,7 @@ class FormatCompileIntegrationTestCase: upstream_sql: str = ( "SELECT CAST(1 AS INTEGER) AS order_id, CAST('new' AS VARCHAR) AS order_label\n" ) - selected_rules: str = '["SQBRSQL021"]' + selected_rules: str = '["SQLCLEAN021"]' @dataclass(frozen=True) @@ -916,7 +917,7 @@ class SqlReadabilityTestCase: description: str sql: str expected_fault: bool - rule_code: str = "SQBRSQL041" + rule_code: str = "SQLCLEAN041" @dataclass(frozen=True) @@ -971,7 +972,7 @@ class NativeAllocatorEntryTestCase: @dataclass(frozen=True) class UnusedOutputDifferentialCase: - """One SQBRSQL042 CTE whose rows must survive `format --fix` unchanged.""" + """One SQLCLEAN042 CTE whose rows must survive `format --fix` unchanged.""" description: str cte_sql: str @@ -1031,7 +1032,7 @@ class FormatExceptionPolicyTestCase: @dataclass(frozen=True) class RepeatedJsonParseTestCase: - """One model body checked by SQBRSQL045 through the CLI.""" + """One model body checked by SQLCLEAN045 through the CLI.""" description: str adapter: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py index 64177a1fcf..2a4aeb426d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/helpers.py @@ -15,6 +15,7 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.executor.build._helpers import scheduler as scheduler_module from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( + PROJECT_TOML, execute, query, write_project, @@ -183,10 +184,10 @@ def migrate_columns(declarations: str) -> str: return f" columns ({declarations}),\n" -def write_model(*, project_dir: Path, sql: str) -> None: +def write_model(*, project_dir: Path, sql: str, project_toml: str = PROJECT_TOML) -> None: """Replace the project with exactly the orders model.""" - write_project(project_dir=project_dir, models={MODEL_NAME: sql}) + write_project(project_dir=project_dir, models={MODEL_NAME: sql}, project_toml=project_toml) def load_orders(*, project_dir: Path, last_day: int, failing_day: int | None = None) -> None: diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_equivalence.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_equivalence.py index 3abaad406a..c4dc6db084 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_equivalence.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/test_column_rename_equivalence.py @@ -32,6 +32,7 @@ state_table_names, write_model, ) +from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import PROJECT_TOML _ORDERS: str = 'FROM __source("raw_orders")' _REBOUND_BODY: str = ( @@ -45,6 +46,8 @@ f"SELECT order_id, order_date, amount AS tax {_ORDERS} " "QUALIFY row_number() OVER (ORDER BY tax) = 1" ) +# The QUALIFY window ranks by a non-unique column on purpose, which SQLCLEAN018 reports. +_QUALIFY_PROJECT_TOML: str = PROJECT_TOML + '\n[rules]\nignore = ["SQLCLEAN018"]\n' _FILTERED_BODY: str = f"SELECT order_id, order_date, amount AS revenue {_ORDERS} WHERE amount > 0" _REPLAY_BOUNDED: str = " replay_on_change bounded-2d,\n" _HISTORY: tuple[tuple[int, Any], ...] = ((1, 101), (2, 102), (3, 103), (4, 104), (5, 105)) @@ -243,11 +246,11 @@ def test_given_rename_with_other_query_changes_when_building_then_replay_matches def test_given_renamed_alias_colliding_with_an_input_column_when_building_then_it_replays( test_case: CollidingAliasRenameTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: - write_model(project_dir=tmp_path, sql=test_case.initial_sql) + write_model(project_dir=tmp_path, sql=test_case.initial_sql, project_toml=_QUALIFY_PROJECT_TOML) load_descending_amounts(project_dir=tmp_path, last_day=3) _ = build_ok(project_dir=tmp_path, capsys=capsys) load_descending_amounts(project_dir=tmp_path, last_day=5) - write_model(project_dir=tmp_path, sql=test_case.renamed_sql) + write_model(project_dir=tmp_path, sql=test_case.renamed_sql, project_toml=_QUALIFY_PROJECT_TOML) plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) _ = build_ok(project_dir=tmp_path, capsys=capsys) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py index 87b09cc628..90a40c207b 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py @@ -37,7 +37,7 @@ class _StagingEveryProject(PreparedCompileArtifacts): ) OUTPUT_PROJECT_FILES: dict[str, str] = { "sqlbuild_project.toml": ( - 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL021"]\n' + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN021"]\n' ), **{ f"models/orders_{name}.sql": OUTPUT_MODEL_HEADER.format(name=name) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index a8b8acd2f9..ca3df86814 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -89,7 +89,7 @@ def write_prepared_artifacts_project(*, project_dir: Path, model_count: int) -> """Write a DuckDB project large enough to stage compile artifacts in the background.""" (project_dir / "sqlbuild_project.toml").write_text( - 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = ["SQBRSQL021"]\n', + 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = ["SQLCLEAN021"]\n', encoding="utf-8", ) models: Path = project_dir / "models" @@ -103,7 +103,8 @@ def write_prepared_artifacts_project(*, project_dir: Path, model_count: int) -> ) -SELECTED_RULES_CONFIG: str = '[rules]\nselect = ["SQBRSQL021"]\n' +SELECTED_RULES_CONFIG: str = '[rules]\nselect = ["SQLCLEAN021"]\n' +NO_RULES_CONFIG: str = "[rules]\nselect = []\n" TEST_PLANNING_MODELS: int = 6 @@ -271,7 +272,7 @@ def write_from_values_format_project(*, tmp_path: Path, adapter: str) -> tuple[P project_dir: Path = tmp_path / adapter project_dir.mkdir() (project_dir / "sqlbuild_project.toml").write_text( - f'name = "orders"\nadapter = "{adapter}"\n\n[rules]\nselect = ["SQBRSQL039"]\n', + f'name = "orders"\nadapter = "{adapter}"\n\n[rules]\nselect = ["SQLCLEAN039"]\n', encoding="utf-8", ) models: Path = project_dir / "models" @@ -637,17 +638,15 @@ def dropped_relation_microbatch_sql(*, batch_concurrency: int) -> str: ) -def compile_duckdb_projection( - *, tmp_path: Path, capsys: pytest.CaptureFixture[str], projection: str +def compile_duckdb_query( + *, tmp_path: Path, capsys: pytest.CaptureFixture[str], query: str ) -> tuple[int, tuple[object, ...]]: - """Compile one DuckDB model projection and return the exit and diagnostic codes.""" + """Compile one DuckDB model query and return the exit and diagnostic codes.""" (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') models: Path = tmp_path / "models" models.mkdir() - (models / "orders.sql").write_text( - f"MODEL (description 'Test model orders.');\nSELECT {projection}" - ) + (models / "orders.sql").write_text(f"MODEL (description 'Test model orders.');\n{query}") exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) result: dict[str, object] = json.loads(capsys.readouterr().out) diagnostics: list[dict[str, object]] = cast(list[dict[str, object]], result["diagnostics"]) @@ -705,10 +704,10 @@ def record(item: dict[str, object]) -> dict[str, object]: def write_repeated_json_parse_project( *, tmp_path: Path, test_case: RepeatedJsonParseTestCase ) -> None: - """Write a one-model project that selects only SQBRSQL045.""" + """Write a one-model project that selects only SQLCLEAN045.""" (tmp_path / "sqlbuild_project.toml").write_text( - f'name = "orders"\nadapter = "{test_case.adapter}"\n\n[rules]\nselect = ["SQBRSQL045"]\n', + f'name = "orders"\nadapter = "{test_case.adapter}"\n\n[rules]\nselect = ["SQLCLEAN045"]\n', encoding="utf-8", ) files: tuple[tuple[str, str], ...] = ( @@ -856,7 +855,7 @@ def write_project_files(*, project_dir: Path, files: dict[str, str]) -> None: EARLY_LINT_PROJECT_FILES: dict[str, str] = { "sqlbuild_project.toml": ( 'name = "orders"\nadapter = "duckdb"\n\n[vars]\nfloor = "2"\n\n' - '[rules]\nselect = ["SQBRSQL005"]\n' + '[rules]\nselect = ["SQLCLEAN005"]\n' ), "models/_macros/python/cents.py": ( 'def cents(expression: str) -> str:\n """Convert to cents."""\n' diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile.py index 488e0d48d0..380744a427 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile.py @@ -128,7 +128,7 @@ def test_given_model_macro_when_compiling_with_sql_rules_then_expands_once_and_p capsys: pytest.CaptureFixture[str], ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( - 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = ["SQBRSQL001"]\n' + 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = ["SQLCLEAN001"]\n' ) macro_dir: Path = tmp_path / "models" / "orders" / "_sqlbuild" / "_macros" / "python" macro_dir.mkdir(parents=True) @@ -148,7 +148,7 @@ def test_given_model_macro_when_compiling_with_sql_rules_then_expands_once_and_p diagnostics: list[dict[str, object]] = cast(list[dict[str, object]], result["diagnostics"]) assert exit_code == 1 assert int((macro_dir / "missing_value.calls").read_text()) == test_case.expected_count - assert tuple(item["code"] for item in diagnostics) == ("SQBRSQL001",) + assert tuple(item["code"] for item in diagnostics) == ("SQLCLEAN001",) assert diagnostics[0]["line"] == 3 diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_native_facts.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_native_facts.py index 24861e12f8..2c61d74947 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_native_facts.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_native_facts.py @@ -19,7 +19,12 @@ StarExpansionCompileTestCase, StarLineageCompileTestCase, ) -from tests.integration.src.sqlbuild.cli.commands.main.helpers import compile_duckdb_projection +from tests.integration.src.sqlbuild.cli.commands.main.helpers import compile_duckdb_query + +# Cases that check facts through derived tables turn off the Rule that asks for a CTE instead. +_DERIVED_RELATION_PROJECT: str = ( + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nignore = ["SQLCLEAN039"]\n' +) @pytest.mark.parametrize( @@ -64,7 +69,7 @@ def test_given_derived_native_facts_when_compiling_then_contract_and_execution_a tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: - (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + (tmp_path / "sqlbuild_project.toml").write_text(_DERIVED_RELATION_PROJECT) models: Path = tmp_path / "models" models.mkdir() (models / "orders.sql").write_text( @@ -112,7 +117,7 @@ def test_given_star_over_inferred_upstream_when_compiling_then_downstream_bindin tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: - (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + (tmp_path / "sqlbuild_project.toml").write_text(_DERIVED_RELATION_PROJECT) models: Path = tmp_path / "models" models.mkdir() (models / "staged_orders.sql").write_text( @@ -155,7 +160,7 @@ def test_given_star_over_derived_table_when_compiling_then_lineage_matches_proje tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: - (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + (tmp_path / "sqlbuild_project.toml").write_text(_DERIVED_RELATION_PROJECT) seeds: Path = tmp_path / "seeds" seeds.mkdir() (seeds / "order_labels.csv").write_text("order_id,label\n1,first\n") @@ -255,8 +260,8 @@ def test_given_snapshot_reader_when_compiling_then_validity_columns_are_known( ( KeywordFunctionCompileTestCase( description="keyword timestamps compile and execute", - projection=( - "CURRENT_TIMESTAMP AS created_at, LOCALTIMESTAMP AS local_created_at, " + query=( + "SELECT CURRENT_TIMESTAMP AS created_at, LOCALTIMESTAMP AS local_created_at, " "CURRENT_TIME AS created_time, LOCALTIME AS local_created_time" ), expected_exit_code=0, @@ -264,9 +269,9 @@ def test_given_snapshot_reader_when_compiling_then_validity_columns_are_known( ), KeywordFunctionCompileTestCase( description="keyword timestamp in a CTE arithmetic filter", - projection=( - "created_at FROM (SELECT CURRENT_TIMESTAMP AS created_at) AS recent " - "WHERE created_at > CURRENT_TIMESTAMP - INTERVAL 1 DAY" + query=( + "WITH recent AS (SELECT CURRENT_TIMESTAMP AS created_at) " + "SELECT created_at FROM recent WHERE created_at > CURRENT_TIMESTAMP - INTERVAL 1 DAY" ), expected_exit_code=0, expected_diagnostics=(), @@ -279,9 +284,7 @@ def test_given_duckdb_keyword_function_when_compiling_then_compiled_sql_executes tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: - exit_code, codes = compile_duckdb_projection( - tmp_path=tmp_path, capsys=capsys, projection=test_case.projection - ) + exit_code, codes = compile_duckdb_query(tmp_path=tmp_path, capsys=capsys, query=test_case.query) assert exit_code == test_case.expected_exit_code assert codes == test_case.expected_diagnostics @@ -295,7 +298,7 @@ def test_given_duckdb_keyword_function_when_compiling_then_compiled_sql_executes ( KeywordFunctionCompileTestCase( description="parenthesized keyword timestamp is still unknown", - projection="CURRENT_TIMESTAMP() AS created_at", + query="SELECT CURRENT_TIMESTAMP() AS created_at", expected_exit_code=1, expected_diagnostics=("B101",), ), @@ -307,9 +310,7 @@ def test_given_invalid_duckdb_keyword_call_when_compiling_then_function_is_rejec tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: - exit_code, codes = compile_duckdb_projection( - tmp_path=tmp_path, capsys=capsys, projection=test_case.projection - ) + exit_code, codes = compile_duckdb_query(tmp_path=tmp_path, capsys=capsys, query=test_case.query) assert exit_code == test_case.expected_exit_code assert codes == test_case.expected_diagnostics @@ -444,7 +445,7 @@ def test_given_bound_wildcard_or_using_join_when_compiling_then_facts_match_exec tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: - (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + (tmp_path / "sqlbuild_project.toml").write_text(_DERIVED_RELATION_PROJECT) sources: Path = tmp_path / "sources" sources.mkdir() (sources / "orders.yml").write_text( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py index 240e547810..1ebc4f3c5d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py @@ -21,6 +21,7 @@ PreparedArtifactsCompileTestCase, ) from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + NO_RULES_CONFIG, SELECTED_RULES_CONFIG, TEST_PLANNING_MODELS, compile_project_json, @@ -68,7 +69,10 @@ max_prepared_models=1, ), BackgroundTestPlanningCompileTestCase( - "no-cache compile without selected rules", ("--no-cache",), "", keep_project_unchanged + "no-cache compile without selected rules", + ("--no-cache",), + NO_RULES_CONFIG, + keep_project_unchanged, ), BackgroundTestPlanningCompileTestCase( "no-cache compile that stages every artifact", @@ -188,7 +192,7 @@ def test_given_existing_artifacts_when_rules_fail_then_staged_changes_are_not_pu ) -> None: use_staging_directory_factory(monkeypatch, create_staging=test_case.staging_directory_factory) config_path: Path = tmp_path / "sqlbuild_project.toml" - config: str = 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = ["SQBRSQL021"]\n' + config: str = 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = ["SQLCLEAN021"]\n' config_path.write_text(config, encoding="utf-8") models: Path = tmp_path / "models" models.mkdir() @@ -228,7 +232,7 @@ def test_given_existing_artifacts_when_rules_fail_then_staged_changes_are_not_pu encoding="utf-8", ) config_path.write_text( - config.replace('select = ["SQBRSQL021"]', 'select = ["XSQBRARCH001"]'), encoding="utf-8" + config.replace('select = ["SQLCLEAN021"]', 'select = ["XSQBRARCH001"]'), encoding="utf-8" ) assert main(arguments) == 1 failed: dict[str, object] = json.loads(capsys.readouterr().out) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py index 4faa1eb01e..f9732b5574 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py @@ -32,7 +32,7 @@ 'connection = "local"\n' 'schema = "dev"\n\n' "[rules]\n" - 'select = ["SQBRSQL028"]\n' + 'select = ["SQLCLEAN028"]\n' ) _ORDERS: str = ( 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' @@ -47,7 +47,7 @@ _FAILING: str = 'SELECT o.order_id, status FROM __ref("orders") AS o' _FAILED_OPEN_ORDERS: bytes = ( b"-- sqb: compile failed for this model; not buildable\n" - b"-- SQBRSQL028 at models/open_orders.sql:6:20: " + b"-- SQLCLEAN028 at models/open_orders.sql:6:20: " b"Single-relation query mixes qualified and unqualified columns\n" b'SELECT o.order_id, status FROM __ref("orders") AS o\n' ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format.py index d15e674dcd..ad63cf2219 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_format.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format.py @@ -1077,7 +1077,7 @@ def test_given_leading_cte_comment_when_formatting_then_comment_rule_still_passe capsys: pytest.CaptureFixture[str], ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( - 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL033"]\n', + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN033"]\n', encoding="utf-8", ) models: Path = tmp_path / "models" @@ -1098,7 +1098,7 @@ def test_given_leading_cte_comment_when_formatting_then_comment_rule_still_passe assert (format_exit, second_format_exit, compile_exit) == (0, 0, 0), output.out + output.err assert test_case.expected_fragment in formatted_sql assert model_path.read_text(encoding="utf-8") == formatted_sql - assert "SQBRSQL033" not in output.out + output.err + assert "SQLCLEAN033" not in output.out + output.err @pytest.mark.parametrize( @@ -1289,8 +1289,8 @@ def test_given_select_all_and_implicit_aliases_when_formatting_then_only_layout_ "-- paid orders only\n" "paid as (select o.order_id, cast(coalesce(o.amount, o.fallback_amount, 0) " "+ o.tax as decimal(18, 2)) as total from orders o where o.amount > 0),\n" - "labels as (select * from (values (1, 'paid and shipped to the customer'), " - "(2, 'cancelled by the customer')) as l(order_id, label))\n" + "labels(order_id, label) as (values (1, 'paid and shipped to the customer'), " + "(2, 'cancelled by the customer'))\n" "select p.order_id, p.total from paid p join labels l on p.order_id = l.order_id\n" ), expected_sql=( @@ -1313,13 +1313,10 @@ def test_given_select_all_and_implicit_aliases_when_formatting_then_only_layout_ " WHERE o.amount > 0\n" "),\n" "\n" - "labels AS (\n" - " SELECT *\n" - " FROM (\n" - " VALUES\n" - " (1, 'paid and shipped to the customer'),\n" - " (2, 'cancelled by the customer')\n" - " ) AS l(order_id, label)\n" + "labels(order_id, label) AS (\n" + " VALUES\n" + " (1, 'paid and shipped to the customer'),\n" + " (2, 'cancelled by the customer')\n" ")\n" "\n" "SELECT\n" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py index fc720f9ebb..46ded87c9c 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py @@ -25,9 +25,9 @@ ) _DIFFERENTIAL_RAW_SQL: str = ( 'MODEL (description "Raw orders");\n' - "SELECT * FROM (VALUES (1, 7, 'open', 3, [1, 2]), (2, 7, 'done', 5, [3])," - " (3, 8, 'done', 3, [4, 5, 6]), (4, 9, 'done', 2, [7])) AS v" - " (order_id, customer_id, status, amount, tags)\n" + "WITH v (order_id, customer_id, status, amount, tags) AS (VALUES (1, 7, 'open', 3, [1, 2])," + " (2, 7, 'done', 5, [3]), (3, 8, 'done', 3, [4, 5, 6]), (4, 9, 'done', 2, [7]))\n" + "SELECT v.order_id, v.customer_id, v.status, v.amount, v.tags FROM v\n" ) @@ -48,7 +48,7 @@ def test_given_bare_union_when_fixing_then_preserves_output_and_is_idempotent( payload: dict[str, Any] = json.loads(capsys.readouterr().out) assert model.read_text() == original assert any( - fix["code"] == "SQBRSQL008" and fix["status"] == "applied" for fix in payload["rule_fixes"] + fix["code"] == "SQLCLEAN008" and fix["status"] == "applied" for fix in payload["rule_fixes"] ) assert main(["--project-dir", str(tmp_path), "format", "--fix", "--json"]) == 0 capsys.readouterr() @@ -71,7 +71,7 @@ def test_given_null_comparison_when_fixing_then_refuses_semantics_change( main(["--project-dir", str(tmp_path), "format", "--fix", "--json"]) payload: dict[str, Any] = json.loads(capsys.readouterr().out) assert any( - fix["code"] == "SQBRSQL001" and fix["status"] == "refused" for fix in payload["rule_fixes"] + fix["code"] == "SQLCLEAN001" and fix["status"] == "refused" for fix in payload["rule_fixes"] ) assert test_case.expected_literal in model.read_text() @@ -109,28 +109,28 @@ def test_given_unanalysed_model_when_fixing_then_verification_preserves_original "comma product", "SELECT a.order_id, b.customer_id FROM " "(SELECT 1 AS order_id) AS a, (SELECT 2 AS customer_id) AS b", - "SQBRSQL002", + "SQLCLEAN002", ), SemanticFixTestCase( "unused CTE", "WITH unused AS (SELECT 2 AS customer_id) SELECT 1 AS order_id", - "SQBRSQL005", + "SQLCLEAN005", ), SemanticFixTestCase( "constant row count", "SELECT COUNT(1) AS order_count FROM (SELECT 1 AS order_id UNION ALL SELECT 2) AS o", - "SQBRSQL017", + "SQLCLEAN017", ), SemanticFixTestCase( "implicit inner join", "SELECT a.order_id, b.customer_id FROM (SELECT 1 AS order_id) AS a " "JOIN (SELECT 1 AS order_id, 2 AS customer_id) AS b ON a.order_id = b.order_id", - "SQBRSQL025", + "SQLCLEAN025", ), SemanticFixTestCase( "redundant distinct", "SELECT DISTINCT order_id FROM (SELECT 1 AS order_id) AS orders GROUP BY order_id", - "SQBRSQL006", + "SQLCLEAN006", ), ], ids=lambda case: case.description, @@ -163,33 +163,33 @@ def test_given_safe_native_fix_when_formatting_then_compiler_and_query_results_a "Snowflake cartesian JOIN", "SELECT a.order_id, b.customer_id FROM " "(SELECT 1 AS order_id) AS a JOIN (SELECT 2 AS customer_id) AS b", - "SQBRSQL003", + "SQLCLEAN003", dialect="snowflake", ), SemanticFixTestCase( "DuckDB invalid JOIN", "SELECT a.order_id, b.customer_id FROM " "(SELECT 1 AS order_id) AS a JOIN (SELECT 2 AS customer_id) AS b", - "SQBRSQL003", + "SQLCLEAN003", expected_status="refused", ), SemanticFixTestCase( "parentheses precedence", "SELECT DISTINCT (1 + 2) * 3 AS order_id", - "SQBRSQL013", + "SQLCLEAN013", expected_status="refused", ), SemanticFixTestCase( - "unused table alias", + "unused table alias proven by recompiling", "SELECT order_id FROM (SELECT 1 AS order_id) AS o", - "SQBRSQL023", - expected_status="refused", + "SQLCLEAN023", ), SemanticFixTestCase( - "boolean CASE", + "boolean CASE whose rewrite changes inferred nullability", "SELECT CASE WHEN 1 = 2 THEN TRUE ELSE FALSE END AS active", - "SQBRSQL030", + "SQLCLEAN030", expected_status="refused", + expected_exit_code=1, ), ], ids=lambda case: case.description, @@ -205,8 +205,9 @@ def test_given_dialect_sensitive_rewrite_when_fixing_then_requires_semantic_proo model.write_text( 'MODEL (description "Orders", database example, schema analytics);\n' + test_case.sql ) - assert main(["--project-dir", str(tmp_path), "format", "--fix", "--json"]) == 0 + exit_code: int = main(["--project-dir", str(tmp_path), "format", "--fix", "--json"]) payload: dict[str, Any] = json.loads(capsys.readouterr().out) + assert exit_code == test_case.expected_exit_code assert any( fix["code"] == test_case.expected_code and fix["status"] == test_case.expected_status for fix in payload["rule_fixes"] @@ -244,8 +245,8 @@ def test_given_one_unverifiable_file_when_fixing_then_only_that_file_keeps_its_o assert { (Path(fix["file"]).name, fix["code"], fix["status"]) for fix in payload["rule_fixes"] } >= { - ("orders.sql", "SQBRSQL008", "applied"), - ("customers.sql", "SQBRSQL005", "refused"), + ("orders.sql", "SQLCLEAN008", "applied"), + ("customers.sql", "SQLCLEAN005", "refused"), } faults: set[tuple[str, str]] = { (Path(violation["file"]).name, violation["code"]) for violation in payload["violations"] @@ -295,7 +296,7 @@ def test_given_existing_errors_and_unexpandable_file_when_fixing_then_other_file assert unexpandable.read_text() == unexpandable_sql assert { (Path(fix["file"]).name, fix["code"], fix["status"]) for fix in payload["rule_fixes"] - } >= {("orders.sql", "SQBRSQL008", "applied")} + } >= {("orders.sql", "SQLCLEAN008", "applied")} assert { ( Path(violation["file"]).name, @@ -336,7 +337,7 @@ def test_given_only_unverifiable_fixes_when_fixing_then_every_refusal_is_listed_ assert exit_code == 1 assert { (Path(fix["file"]).name, fix["code"], fix["status"]) for fix in payload["rule_fixes"] - } >= {("orders.sql", "SQBRSQL001", "refused"), ("customers.sql", "SQBRSQL005", "refused")} + } >= {("orders.sql", "SQLCLEAN001", "refused"), ("customers.sql", "SQLCLEAN005", "refused")} assert { (Path(violation["file"]).name, violation["message"]) for violation in payload["violations"] } >= {("customers.sql", test_case.expected_literal + ", so the fix cannot be verified")} @@ -419,7 +420,7 @@ def test_given_unused_cte_output_when_fixing_then_model_rows_are_unchanged( assert after == before assert ( - rule_fix_statuses(payload=payload, code="SQBRSQL042", file_name="summary.sql") + rule_fix_statuses(payload=payload, code="SQLCLEAN042", file_name="summary.sql") == test_case.expected_statuses ) @@ -456,7 +457,7 @@ def test_given_macro_and_crlf_source_when_fixing_then_previews_preserve_bytes_an assert main(["--project-dir", str(tmp_path), "format", "--fix", "--json"]) == 0 payload: dict[str, Any] = json.loads(capsys.readouterr().out) assert any( - fix["code"] == "SQBRSQL008" and fix["status"] == "applied" for fix in payload["rule_fixes"] + fix["code"] == "SQLCLEAN008" and fix["status"] == "applied" for fix in payload["rule_fixes"] ) written: bytes = model.read_bytes() assert b"@py.order_value()" in written diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py index 962bca87b8..f7e146fec1 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py @@ -8,6 +8,11 @@ from sqlbuild._native import run_cli as main from tests.integration.src.sqlbuild.cli.commands.main._test_types import FormatterSyntaxTestCase +# The window syntax cases order by a non-unique column, which SQLCLEAN018 reports. +_WINDOW_SYNTAX_PROJECT_TOML: str = ( + 'name = "orders"\nadapter = "snowflake"\n\n[rules]\nignore = ["SQLCLEAN018"]\n' +) + @pytest.mark.parametrize( "test_case", @@ -76,7 +81,7 @@ def test_given_authored_syntax_when_formatting_then_spelling_and_idempotence_are_preserved( test_case: FormatterSyntaxTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: - (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "snowflake"\n') + (tmp_path / "sqlbuild_project.toml").write_text(_WINDOW_SYNTAX_PROJECT_TOML) macros: Path = tmp_path / "models" / "_macros" / "python" macros.mkdir(parents=True) (macros / "orders.py").write_text(test_case.macro_source) @@ -213,7 +218,7 @@ def test_given_cte_macros_and_comments_when_formatting_then_layout_and_rules_sta test_case: FormatterSyntaxTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( - 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = ["SQBRSQL033"]\n' + 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = ["SQLCLEAN033"]\n' ) macros: Path = tmp_path / "models" / "_macros" / "python" macros.mkdir(parents=True) @@ -235,7 +240,7 @@ def test_given_cte_macros_and_comments_when_formatting_then_layout_and_rules_sta assert model.read_text() == formatted assert main(["--project-dir", str(tmp_path), "compile", "--no-cache"]) == 0 capsys.readouterr() - assert main(["--project-dir", str(tmp_path), "rules", "--json", "run", "SQBRSQL033"]) == 0 + assert main(["--project-dir", str(tmp_path), "rules", "--json", "run", "SQLCLEAN033"]) == 0 payload: dict[str, object] = json.loads(capsys.readouterr().out) assert payload["findings"] == [] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py b/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py index 3c739adeb2..d4776644b9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py @@ -1,4 +1,4 @@ -"""Real CLI coverage for how SQBRSQL028 judges the qualification of SQL macro slot reads.""" +"""Real CLI coverage for how SQLCLEAN028 judges the qualification of SQL macro slot reads.""" from __future__ import annotations @@ -22,7 +22,7 @@ 'connection = "local"\n' 'schema = "dev"\n\n' "[rules]\n" - 'select = ["SQBRSQL028"]\n' + 'select = ["SQLCLEAN028"]\n' ) @@ -40,7 +40,7 @@ ), expected_exit_code=0, expected_fragments=("Project compiled",), - absent_fragments=("SQBRSQL028",), + absent_fragments=("SQLCLEAN028",), ), MacroSlotQualificationTestCase( description="aliased slot reads beside a plain group read fail once, at the macro", @@ -79,7 +79,7 @@ }, expected_exit_code=0, expected_fragments=("Project compiled",), - absent_fragments=("SQBRSQL028",), + absent_fragments=("SQLCLEAN028",), ), MacroSlotQualificationTestCase( description="an expression macro's plain reads pass beside hand-qualified columns", @@ -102,7 +102,7 @@ }, expected_exit_code=0, expected_fragments=("Project compiled",), - absent_fragments=("SQBRSQL028",), + absent_fragments=("SQLCLEAN028",), ), MacroSlotQualificationTestCase( description="an expression macro's bare column beside a hand-qualified column fails", @@ -120,7 +120,7 @@ ), }, expected_exit_code=1, - expected_fragments=("error[SQBRSQL028]", "--> models/open_orders.sql"), + expected_fragments=("error[SQLCLEAN028]", "--> models/open_orders.sql"), absent_fragments=(), ), MacroSlotQualificationTestCase( @@ -139,7 +139,7 @@ ), }, expected_exit_code=1, - expected_fragments=("error[SQBRSQL028]", "--> models/open_orders.sql"), + expected_fragments=("error[SQLCLEAN028]", "--> models/open_orders.sql"), absent_fragments=(), ), MacroSlotQualificationTestCase( @@ -155,7 +155,7 @@ ), }, expected_exit_code=1, - expected_fragments=("error[SQBRSQL028]", "--> models/open_orders.sql"), + expected_fragments=("error[SQLCLEAN028]", "--> models/open_orders.sql"), absent_fragments=(), ), ], diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_repeated_json_parse.py b/tests/integration/src/sqlbuild/cli/commands/main/test_repeated_json_parse.py index 6dff1da5db..ea95f31645 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_repeated_json_parse.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_repeated_json_parse.py @@ -1,4 +1,4 @@ -"""Real-CLI coverage for SQBRSQL045, a JSON payload parsed more than once in one SELECT.""" +"""Real-CLI coverage for SQLCLEAN045, a JSON payload parsed more than once in one SELECT.""" from __future__ import annotations @@ -96,7 +96,7 @@ def test_given_json_parses_when_compiling_and_running_rules_then_repeats_fail_at assert main(["--project-dir", str(tmp_path), "compile", "--no-cache"]) == expected_exit_code capsys.readouterr() - result: int = main(["--project-dir", str(tmp_path), "rules", "--json", "run", "SQBRSQL045"]) + result: int = main(["--project-dir", str(tmp_path), "rules", "--json", "run", "SQLCLEAN045"]) findings: list[dict[str, object]] = json.loads(capsys.readouterr().out)["findings"] assert result == expected_exit_code @@ -147,7 +147,7 @@ def test_given_snowflake_json_parses_when_running_rules_then_repeats_are_reporte ) -> None: write_repeated_json_parse_project(tmp_path=tmp_path, test_case=test_case) - result: int = main(["--project-dir", str(tmp_path), "rules", "--json", "run", "SQBRSQL045"]) + result: int = main(["--project-dir", str(tmp_path), "rules", "--json", "run", "SQLCLEAN045"]) findings: list[dict[str, object]] = json.loads(capsys.readouterr().out)["findings"] assert result == int(bool(test_case.expected_locations)) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rule_exception_policy.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rule_exception_policy.py index 29dbf2e504..3f81a09054 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rule_exception_policy.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rule_exception_policy.py @@ -14,17 +14,17 @@ ) _EXCEPTION: str = ( - '[[rules.rule_exceptions]]\nrule = "SQBRSQL004"\npath = "models/customers.sql"\n' + '[[rules.rule_exceptions]]\nrule = "SQLCLEAN004"\npath = "models/customers.sql"\n' 'reason = "The sample intentionally keeps one row."\n' ) _IGNORE: str = ( - '[[rules.rule_ignores]]\nrules = ["SQBRSQL004"]\npaths = ["models/**"]\n' + '[[rules.rule_ignores]]\nrules = ["SQLCLEAN004"]\npaths = ["models/**"]\n' 'reason = "Samples intentionally keep one row."\n' ) -_DIRECTIVE: str = "-- sqb: ignore SQBRSQL004 because the sample intentionally keeps one row\n" +_DIRECTIVE: str = "-- sqb: ignore SQLCLEAN004 because the sample intentionally keeps one row\n" _PRINTED_ENTRY: str = ( "[[rules.rule_exceptions]]\n" - ' rule = "SQBRSQL004"\n' + ' rule = "SQLCLEAN004"\n' ' path = "models/customers.sql"\n' ' reason = "the sample intentionally keeps one row"' ) @@ -40,10 +40,10 @@ directive=_DIRECTIVE, expected_exit_code=1, expected_fragments=( - "SQBRSQL000", + "SQBRSUPPRESSION000", "Inline Rule suppressions are not supported", _PRINTED_ENTRY, - "SQBRSQL004", + "SQLCLEAN004", ), ), RuleExceptionPolicyTestCase( @@ -74,7 +74,7 @@ ), RuleExceptionPolicyTestCase( description="project-wide ignore", - rules_toml='ignore = ["SQBRSQL004"]\n', + rules_toml='ignore = ["SQLCLEAN004"]\n', directive="", expected_exit_code=0, ), @@ -87,7 +87,7 @@ def test_given_rule_exception_mechanism_when_compiling_then_only_configuration_a capsys: pytest.CaptureFixture[str], ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( - 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL004"]\n' + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN004"]\n' f"{test_case.rules_toml}", encoding="utf-8", ) @@ -134,7 +134,7 @@ def test_given_exception_policy_when_formatting_then_it_is_validated_like_compil capsys: pytest.CaptureFixture[str], ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( - 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL004"]\n' + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN004"]\n' f"{test_case.setting}\n", encoding="utf-8", ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index a8f1a66c9c..66c39436dc 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -54,10 +54,12 @@ write_project_scratch_rules, ) -_CACHE_RULES_TOML: str = 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL004"]\n' +_CACHE_RULES_TOML: str = ( + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN004"]\n' +) _CACHE_THRESHOLD_TOML: str = _CACHE_RULES_TOML + "thresholds = { max_literal_length = 7 }\n" _CACHE_SELECTION_TOML: str = _CACHE_THRESHOLD_TOML.replace( - '"SQBRSQL004"', '"SQBRSQL004", "SQBRSQL001"' + '"SQLCLEAN004"', '"SQLCLEAN004", "SQLCLEAN001"' ) _CACHE_MODEL_SQL: str = 'MODEL (description "Orders");\nSELECT order_id FROM orders LIMIT 1\n' _CACHE_EDITED_SQL: str = _CACHE_MODEL_SQL.replace("LIMIT 1", "LIMIT 2") @@ -188,7 +190,7 @@ def project_facts(*, project: Project, ctx: RuleContext) -> list[Finding]: _EARLY_LINT_SLOW_EXPANSION_S: float = 0.5 _EARLY_LINT_EXPANSION_DELAYS_S: dict[str, float] = {"orders": 0.0} _EARLY_LINT_FINDINGS: tuple[tuple[str, str, int | None, int | None], ...] = ( - ("SQBRSQL005", "models/order_cents.sql", 2, 119), + ("SQLCLEAN005", "models/order_cents.sql", 2, 119), ) @@ -646,7 +648,7 @@ def test_given_proven_dynamic_pivot_when_compiling_contract_rules_then_wildcard_ ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( 'name = "orders"\nadapter = "snowflake"\n\n[rules]\n' - 'select = ["SQBRMODEL102", "SQBRSQL021", "SQBRCONTRACT101", ' + 'select = ["SQBRMODEL102", "SQLCLEAN021", "SQBRCONTRACT101", ' '"SQBRCONTRACT105", "SQBRCONTRACT106"]\n\n' "[defaults]\n" 'contract = "enforced"\n', @@ -706,7 +708,7 @@ def test_given_proven_dynamic_pivot_when_compiling_contract_rules_then_wildcard_ assert invalid_exit_code == test_case.expected_invalid_exit_code assert invalid_codes.count("SQBRMODEL102") == test_case.expected_invalid_model_findings - assert invalid_codes.count("SQBRSQL021") == test_case.expected_invalid_sql_findings + assert invalid_codes.count("SQLCLEAN021") == test_case.expected_invalid_sql_findings @pytest.mark.parametrize( @@ -765,7 +767,7 @@ def test_given_contract_rules_when_compiling_then_contract_is_enforced_typed_and @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("selected SQL rule blocks artifacts", 1, "SQBRSQL004")], + [RulesIntegrationTestCase("selected SQL rule blocks artifacts", 1, "SQLCLEAN004")], ids=lambda case: case.description, ) def test_given_selected_sql_rule_when_compiling_then_authored_diagnostic_blocks_artifacts( @@ -774,7 +776,7 @@ def test_given_selected_sql_rule_when_compiling_then_authored_diagnostic_blocks_ capsys: pytest.CaptureFixture[str], ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( - 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL004"]\n', + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN004"]\n', encoding="utf-8", ) model: Path = tmp_path / "models" / "orders.sql" @@ -809,7 +811,7 @@ def test_given_selected_sql_rule_when_compiling_then_authored_diagnostic_blocks_ @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("SQL rule cache follows the native build", 1, "SQBRSQL004")], + [RulesIntegrationTestCase("SQL rule cache follows the native build", 1, "SQLCLEAN004")], ids=lambda case: case.description, ) def test_given_sql_rule_cache_from_another_native_build_when_compiling_then_rules_rerun( @@ -853,7 +855,7 @@ def test_given_sql_rule_cache_from_another_native_build_when_compiling_then_rule @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("warm SQL rules track non-model file edits", 1, "SQBRSQL004")], + [RulesIntegrationTestCase("warm SQL rules track non-model file edits", 1, "SQLCLEAN004")], ids=lambda case: case.description, ) def test_given_cached_sql_test_rules_when_test_or_macro_changes_then_findings_are_current( @@ -904,7 +906,7 @@ def test_given_cached_sql_test_rules_when_test_or_macro_changes_then_findings_ar @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("warm SQL rules reuse compiler declaration scope", 1, "SQBRSQL004")], + [RulesIntegrationTestCase("warm SQL rules reuse compiler declaration scope", 1, "SQLCLEAN004")], ids=lambda case: case.description, ) def test_given_warm_scoped_macro_sql_test_when_compiling_then_rules_reuse_compiler_scope( @@ -942,7 +944,7 @@ def test_given_warm_scoped_macro_sql_test_when_compiling_then_rules_reuse_compil @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("cold SQL rules rebind the compiler scope index", 1, "SQBRSQL004")], + [RulesIntegrationTestCase("cold SQL rules rebind the compiler scope index", 1, "SQLCLEAN004")], ids=lambda case: case.description, ) def test_given_cold_scoped_macro_sql_test_when_compiling_then_early_lint_uses_private_macros( @@ -989,7 +991,7 @@ def recording_load(macro_files: tuple[DiscoveredMacroFile, ...]) -> dict[str, Lo @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("early file read gives the cached findings", 1, "SQBRSQL004")], + [RulesIntegrationTestCase("early file read gives the cached findings", 1, "SQLCLEAN004")], ids=lambda case: case.description, ) def test_given_cold_non_model_sql_finding_when_compiling_twice_then_warm_findings_match( @@ -1087,7 +1089,7 @@ def test_given_graph_error_when_compiling_then_early_lint_is_stopped_instead_of_ monkeypatch: pytest.MonkeyPatch, ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( - 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL005"]\n', + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN005"]\n', encoding="utf-8", ) models_dir: Path = tmp_path / "models" @@ -1149,7 +1151,7 @@ def test_given_sql_and_custom_rules_when_compiling_twice_then_custom_host_runs_o ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( 'name = "orders"\nadapter = "duckdb"\n\n' - f'[rules]\nselect = ["SQBRSQL004", "{test_case.expected_code}"]\n\n' + f'[rules]\nselect = ["SQLCLEAN004", "{test_case.expected_code}"]\n\n' "[rules.thresholds]\nmin_custom_rule_test_cases = 0\n", encoding="utf-8", ) @@ -1189,7 +1191,7 @@ def final_directory(*, model: Model, ctx: RuleContext) -> list[Finding]: assert warm == cold assert { f"models/orders.sql:{test_case.expected_code}", - "tests/unit/test_orders.sql:SQBRSQL004", + "tests/unit/test_orders.sql:SQLCLEAN004", } <= cold @@ -1451,7 +1453,7 @@ def final_directory(*, model: Model, ctx: RuleContext) -> list[Finding]: @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("focused SQL rule reports requested code", 1, "SQBRSQL004")], + [RulesIntegrationTestCase("focused SQL rule reports requested code", 1, "SQLCLEAN004")], ids=lambda case: case.description, ) def test_given_project_when_running_focused_rule_family_then_only_that_family_runs( @@ -1476,7 +1478,7 @@ def test_given_project_when_running_focused_rule_family_then_only_that_family_ru "rules", "--json", "run", - "SQBRSQL004", + "SQLCLEAN004", ] ) @@ -1484,12 +1486,12 @@ def test_given_project_when_running_focused_rule_family_then_only_that_family_ru captured: CaptureResult[str] = capsys.readouterr() payload: dict[str, object] = json.loads(captured.out) assert test_case.expected_code in json.dumps(payload["findings"]) - assert "Rule selection SQBRSQL004 failed with 1 finding(s)." in captured.err + assert "Rule selection SQLCLEAN004 failed with 1 finding(s)." in captured.err @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("inline join query is rejected", 1, "SQBRSQL039")], + [RulesIntegrationTestCase("inline join query is rejected", 1, "SQLCLEAN039")], ids=lambda case: case.description, ) def test_given_inline_query_relation_when_running_readability_rule_then_finding_is_reported( @@ -1538,7 +1540,7 @@ def test_given_inline_query_relation_when_running_readability_rule_then_finding_ @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("scalar query internals are excluded", 0, "SQBRSQL039")], + [RulesIntegrationTestCase("scalar query internals are excluded", 0, "SQLCLEAN039")], ids=lambda case: case.description, ) def test_given_scalar_query_with_derived_relation_when_running_readability_rule_then_no_finding_is_reported( @@ -1583,7 +1585,7 @@ def test_given_scalar_query_with_derived_relation_when_running_readability_rule_ @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("SQL function arguments are not columns", 0, "SQBRSQL027")], + [RulesIntegrationTestCase("SQL function arguments are not columns", 0, "SQLCLEAN027")], ids=lambda case: case.description, ) def test_given_sql_function_argument_when_running_qualification_rule_then_argument_is_not_reported( @@ -1629,7 +1631,7 @@ def test_given_sql_function_argument_when_running_qualification_rule_then_argume @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("table function argument preserves left alias", 0, "SQBRSQL023")], + [RulesIntegrationTestCase("table function argument preserves left alias", 0, "SQLCLEAN023")], ids=lambda case: case.description, ) def test_given_table_function_argument_when_running_alias_rule_then_left_alias_is_preserved( @@ -1749,7 +1751,7 @@ def test_given_implicit_table_alias_when_running_alias_rule_then_usage_decides_f 'MODEL (description "Order customers");\n' + test_case.query_sql, encoding="utf-8" ) - exit_code: int = main(["--project-dir", str(tmp_path), "rules", "--json", "run", "SQBRSQL023"]) + exit_code: int = main(["--project-dir", str(tmp_path), "rules", "--json", "run", "SQLCLEAN023"]) payload: dict[str, object] = json.loads(capsys.readouterr().out) findings: list[dict[str, object]] = cast(list[dict[str, object]], payload["findings"]) @@ -1762,7 +1764,7 @@ def test_given_implicit_table_alias_when_running_alias_rule_then_usage_decides_f @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("ceremonial SQL test passes terminal rule", 0, "SQBRSQL035")], + [RulesIntegrationTestCase("ceremonial SQL test passes terminal rule", 0, "SQLCLEAN035")], ids=lambda case: case.description, ) def test_given_ceremonial_sql_test_when_compiling_with_rule_035_then_project_is_valid( @@ -1771,7 +1773,7 @@ def test_given_ceremonial_sql_test_when_compiling_with_rule_035_then_project_is_ capsys: pytest.CaptureFixture[str], ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( - 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL035"]\n', + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN035"]\n', encoding="utf-8", ) model: Path = tmp_path / "models" / "orders.sql" @@ -1804,7 +1806,7 @@ def test_given_ceremonial_sql_test_when_compiling_with_rule_035_then_project_is_ @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("contextual keywords are plain terminal columns", 0, "SQBRSQL035")], + [RulesIntegrationTestCase("contextual keywords are plain terminal columns", 0, "SQLCLEAN035")], ids=lambda case: case.description, ) def test_given_keyword_named_columns_when_running_terminal_rule_then_projection_is_accepted( @@ -1973,7 +1975,7 @@ def test_given_seed_and_function_references_when_running_source_rule_then_they_a RulesIntegrationTestCase( "dependency import star is compatible with output-shape rule", 0, - "SQBRSQL021", + "SQLCLEAN021", ) ], ids=lambda case: case.description, @@ -2027,7 +2029,7 @@ def test_given_dependency_import_star_when_running_output_shape_rule_then_import @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("invocation-dependent stars are accepted", 0, "SQBRSQL021")], + [RulesIntegrationTestCase("invocation-dependent stars are accepted", 0, "SQLCLEAN021")], ids=lambda case: case.description, ) def test_given_invocation_dependent_shapes_when_running_output_shape_rule_then_stars_are_accepted( @@ -2256,7 +2258,7 @@ def test_given_postgres_nested_comment_when_running_import_rule_then_comment_is_ @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("cross join used by filter is not unused", 0, "SQBRSQL032")], + [RulesIntegrationTestCase("cross join used by filter is not unused", 0, "SQLCLEAN032")], ids=lambda case: case.description, ) def test_given_cross_join_used_by_filter_when_running_unused_join_rule_then_no_finding_is_reported( @@ -2298,7 +2300,7 @@ def test_given_cross_join_used_by_filter_when_running_unused_join_rule_then_no_f @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("ceremonial SQL test select is ignored", 0, "SQBRSQL022")], + [RulesIntegrationTestCase("ceremonial SQL test select is ignored", 0, "SQLCLEAN022")], ids=lambda case: case.description, ) def test_given_ceremonial_test_select_when_running_alias_rule_then_control_projection_is_not_reported( @@ -2341,7 +2343,7 @@ def test_given_ceremonial_test_select_when_running_alias_rule_then_control_proje "test_case", [ RulesIntegrationTestCase( - "direct ceremonial select satisfies terminal shape", 0, "SQBRSQL035" + "direct ceremonial select satisfies terminal shape", 0, "SQLCLEAN035" ) ], ids=lambda case: case.description, @@ -2384,7 +2386,7 @@ def test_given_direct_ceremonial_select_when_running_terminal_rule_then_test_is_ @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("scalar subquery after prior CTE is stable", 0, "SQBRSQL022")], + [RulesIntegrationTestCase("scalar subquery after prior CTE is stable", 0, "SQLCLEAN022")], ids=lambda case: case.description, ) def test_given_prior_cte_when_scalar_subquery_is_aliased_then_inner_aggregate_is_not_reported( @@ -2432,7 +2434,7 @@ def test_given_prior_cte_when_scalar_subquery_is_aliased_then_inner_aggregate_is @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("cast type is not a self alias", 0, "SQBRSQL016")], + [RulesIntegrationTestCase("cast type is not a self alias", 0, "SQLCLEAN016")], ids=lambda case: case.description, ) def test_given_cast_and_relation_aliases_when_running_self_alias_rule_then_no_finding_is_reported( @@ -2472,7 +2474,7 @@ def test_given_cast_and_relation_aliases_when_running_self_alias_rule_then_no_fi @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("upstream graph selector scopes an ignore", 1, "SQBRSQL021")], + [RulesIntegrationTestCase("upstream graph selector scopes an ignore", 1, "SQLCLEAN021")], ids=lambda case: case.description, ) def test_given_graph_scoped_rule_ignore_when_running_rules_then_only_matched_resources_are_ignored( @@ -2483,7 +2485,7 @@ def test_given_graph_scoped_rule_ignore_when_running_rules_then_only_matched_res (tmp_path / "sqlbuild_project.toml").write_text( 'name = "orders"\nadapter = "duckdb"\n\n' "[[rules.rule_ignores]]\n" - 'rules = ["SQBRSQL021"]\n' + 'rules = ["SQLCLEAN021"]\n' 'selectors = ["+orders*"]\n' 'reason = "Reviewed upstream interface boundary"\n', encoding="utf-8", @@ -2570,7 +2572,7 @@ def forbidden_name(*, model: Model, ctx: RuleContext) -> list[Finding]: @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("SQL rule exception suppresses finding", 0, "SQBRSQL004")], + [RulesIntegrationTestCase("SQL rule exception suppresses finding", 0, "SQLCLEAN004")], ids=lambda case: case.description, ) def test_given_sql_rule_exception_when_compiling_then_unified_suppression_is_applied( @@ -2580,8 +2582,8 @@ def test_given_sql_rule_exception_when_compiling_then_unified_suppression_is_app ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( 'name = "orders"\nadapter = "duckdb"\n\n' - '[rules]\nselect = ["SQBRSQL004"]\n\n' - '[[rules.rule_exceptions]]\nrule = "SQBRSQL004"\n' + '[rules]\nselect = ["SQLCLEAN004"]\n\n' + '[[rules.rule_exceptions]]\nrule = "SQLCLEAN004"\n' 'path = "models/orders.sql"\nreason = "The fixture intentionally selects one row."\n', encoding="utf-8", ) @@ -2701,7 +2703,7 @@ def two_models(*, project: Project, ctx: RuleContext) -> list[Finding]: @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("default compile evaluates audit SQL", 1, "SQBRSQL004")], + [RulesIntegrationTestCase("default compile evaluates audit SQL", 1, "SQLCLEAN004")], ids=lambda case: case.description, ) def test_given_non_model_sql_violation_when_compiling_then_compile_is_authoritative( @@ -2710,7 +2712,7 @@ def test_given_non_model_sql_violation_when_compiling_then_compile_is_authoritat capsys: pytest.CaptureFixture[str], ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( - 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL004"]\n', + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN004"]\n', encoding="utf-8", ) model: Path = tmp_path / "models" / "orders.sql" @@ -2747,8 +2749,8 @@ def test_given_exception_for_other_model_when_compiling_selection_then_exception capsys: pytest.CaptureFixture[str], ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( - 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL004"]\n\n' - '[[rules.rule_exceptions]]\nrule = "SQBRSQL004"\npath = "models/customers.sql"\n' + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN004"]\n\n' + '[[rules.rule_exceptions]]\nrule = "SQLCLEAN004"\npath = "models/customers.sql"\n' 'reason = "The fixture intentionally selects one row."\n', encoding="utf-8", ) @@ -2888,7 +2890,11 @@ def test_given_subject_before_layer_when_running_folder_rule_then_canonical_move @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("relative project path retains cached SQL finding", 1, "SQBRSQL004")], + [ + RulesIntegrationTestCase( + "relative project path retains cached SQL finding", 1, "SQLCLEAN004" + ) + ], ids=lambda case: case.description, ) def test_given_relative_project_path_when_compiling_twice_then_sql_finding_remains_cached( @@ -2900,7 +2906,7 @@ def test_given_relative_project_path_when_compiling_twice_then_sql_finding_remai project_dir: Path = tmp_path / "project" project_dir.mkdir() (project_dir / "sqlbuild_project.toml").write_text( - 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL004"]\n', + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN004"]\n', encoding="utf-8", ) model: Path = project_dir / "models" / "orders.sql" @@ -2923,7 +2929,7 @@ def test_given_relative_project_path_when_compiling_twice_then_sql_finding_remai @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("inline directive without a finding fails", 1, "SQBRSQL000")], + [RulesIntegrationTestCase("inline directive without a finding fails", 1, "SQBRSUPPRESSION000")], ids=lambda case: case.description, ) def test_given_inline_directive_without_finding_when_compiling_then_directive_is_reported( @@ -2932,13 +2938,13 @@ def test_given_inline_directive_without_finding_when_compiling_then_directive_is capsys: pytest.CaptureFixture[str], ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( - 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL004"]\n', + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQLCLEAN004"]\n', encoding="utf-8", ) model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() model.write_text( - "MODEL (description 'Test model orders.');\n-- sqb: ignore SQBRSQL004 because this fixture checks stale directives\n" + "MODEL (description 'Test model orders.');\n-- sqb: ignore SQLCLEAN004 because this fixture checks stale directives\n" "SELECT 1 AS order_id\n", encoding="utf-8", ) @@ -3031,23 +3037,23 @@ def required_prefix(*, model: Model, ctx: RuleContext) -> list[Finding]: ), EmptyRuleSelectionTestCase( "missing literal ignore path is rejected", - '[[rules.rule_ignores]]\nrules = ["SQBRSQL"]\npaths = ["models/orders.sql", "models/gone.sql"]\n' + '[[rules.rule_ignores]]\nrules = ["SQLCLEAN"]\npaths = ["models/orders.sql", "models/gone.sql"]\n' 'reason = "Legacy."\n', 1, ("rule ignore path does not exist: models/gone.sql",), ), EmptyRuleSelectionTestCase( "ignore paths naming a folder or a glob that matches nothing compile", - '[[rules.rule_ignores]]\nrules = ["SQBRSQL"]\npaths = ["models", "models/legacy/**"]\n' + '[[rules.rule_ignores]]\nrules = ["SQLCLEAN"]\npaths = ["models", "models/legacy/**"]\n' 'reason = "Legacy."\n', 0, (), ), EmptyRuleSelectionTestCase( "unknown built-in rule option is rejected", - "[rules.rule_options.SQBRSQL004]\nnope = 3\n", + "[rules.rule_options.SQLCLEAN004]\nnope = 3\n", 1, - ("rule SQBRSQL004 has unknown options: nope",), + ("rule SQLCLEAN004 has unknown options: nope",), ), ], ids=lambda case: case.description, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules_concurrent.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules_concurrent.py index 649f7d8b07..287dc9703e 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules_concurrent.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules_concurrent.py @@ -26,9 +26,9 @@ def test_given_sql_and_custom_exceptions_when_compiling_cold_and_warm_then_both_ ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( 'name = "orders"\nadapter = "duckdb"\n' - '[rules]\nselect = ["SQBRSQL004", "XSQBRARCH001"]\n' + '[rules]\nselect = ["SQLCLEAN004", "XSQBRARCH001"]\n' "[rules.thresholds]\nmin_custom_rule_test_cases = 0\n" - '[[rules.rule_exceptions]]\nrule = "SQBRSQL004"\n' + '[[rules.rule_exceptions]]\nrule = "SQLCLEAN004"\n' 'path = "models/orders.sql"\nreason = "A single synthetic row is intentional."\n' '[[rules.rule_exceptions]]\nrule = "XSQBRARCH001"\n' 'path = "models/orders.sql"\nreason = "The synthetic model name is intentional."\n' diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_sql_readability.py b/tests/integration/src/sqlbuild/cli/commands/main/test_sql_readability.py index 6dc7c9f357..6f39f592e9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_sql_readability.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_sql_readability.py @@ -11,6 +11,8 @@ from sqlbuild._native import run_cli as main from tests.integration.src.sqlbuild.cli.commands.main._test_types import SqlReadabilityTestCase +PROJECT_CONFIG: str = 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nignore = ["SQLCLEAN"]\n' + @pytest.mark.parametrize( "test_case", @@ -60,7 +62,7 @@ def test_given_join_predicate_when_compiling_and_running_rules_then_expression_policy_applies( test_case: SqlReadabilityTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: - (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + (tmp_path / "sqlbuild_project.toml").write_text(PROJECT_CONFIG) model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() model.write_text( @@ -71,11 +73,11 @@ def test_given_join_predicate_when_compiling_and_running_rules_then_expression_p ) assert main(["--project-dir", str(tmp_path), "compile", "--no-cache"]) == 0 capsys.readouterr() - result: int = main(["--project-dir", str(tmp_path), "rules", "--json", "run", "SQBRSQL040"]) + result: int = main(["--project-dir", str(tmp_path), "rules", "--json", "run", "SQLCLEAN040"]) payload: dict[str, object] = json.loads(capsys.readouterr().out) assert result == int(test_case.expected_fault) assert bool(payload["findings"]) == test_case.expected_fault - assert ("SQBRSQL040" in json.dumps(payload["findings"])) == test_case.expected_fault + assert ("SQLCLEAN040" in json.dumps(payload["findings"])) == test_case.expected_fault @pytest.mark.parametrize( @@ -114,37 +116,37 @@ def test_given_join_predicate_when_compiling_and_running_rules_then_expression_p "wrapped plain terminal", "((WITH final AS (SELECT 1 AS order_id) SELECT * FROM final))", False, - "SQBRSQL035", + "SQLCLEAN035", ), SqlReadabilityTestCase( "wrapped computed terminal", "((WITH final AS (SELECT 1 AS order_id) SELECT order_id + 1 AS order_id FROM final))", True, - "SQBRSQL035", + "SQLCLEAN035", ), SqlReadabilityTestCase( "wrapped terminal union shape", "(WITH final AS (SELECT 1 AS order_id) SELECT * FROM final UNION ALL SELECT * FROM final)", True, - "SQBRSQL035", + "SQLCLEAN035", ), SqlReadabilityTestCase( "wrapped CTE union terminal shape", "((WITH final AS (SELECT 1 AS order_id UNION ALL SELECT 2 AS order_id) SELECT * FROM final))", False, - "SQBRSQL035", + "SQLCLEAN035", ), SqlReadabilityTestCase( "wrapped root is not nested", "((WITH final AS (SELECT 1 AS order_id) SELECT * FROM final))", False, - "SQBRSQL036", + "SQLCLEAN036", ), SqlReadabilityTestCase( "scalar CTE is nested", "SELECT (WITH orders AS (SELECT 1 AS order_id) SELECT order_id FROM orders) AS order_id", True, - "SQBRSQL036", + "SQLCLEAN036", ), SqlReadabilityTestCase( "final", "WITH final AS (SELECT 1 AS order_id) SELECT * FROM final", False @@ -181,7 +183,7 @@ def test_given_join_predicate_when_compiling_and_running_rules_then_expression_p def test_given_cte_names_when_compiling_and_running_rules_then_final_name_policy_applies( test_case: SqlReadabilityTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: - (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + (tmp_path / "sqlbuild_project.toml").write_text(PROJECT_CONFIG) model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() model.write_text(f'MODEL (description "Orders");\n{test_case.sql}\n') @@ -198,7 +200,7 @@ def test_given_cte_names_when_compiling_and_running_rules_then_final_name_policy @pytest.mark.parametrize( "test_case", - [SqlReadabilityTestCase("family enables both rules", "SQBRSQL", True)], + [SqlReadabilityTestCase("family enables both rules", "SQLCLEAN", True)], ids=lambda case: case.description, ) def test_given_family_selection_when_compiling_then_new_rules_are_enforced( @@ -219,14 +221,14 @@ def test_given_family_selection_when_compiling_then_new_rules_are_enforced( test_case.expected_fault ) output: CaptureResult[str] = capsys.readouterr() - assert "SQBRSQL040" in output.out + output.err - assert "SQBRSQL041" in output.out + output.err + assert "SQLCLEAN040" in output.out + output.err + assert "SQLCLEAN041" in output.out + output.err assert main(["--project-dir", str(tmp_path), "rules", "--json", "run", test_case.sql]) == int( test_case.expected_fault ) payload: dict[str, object] = json.loads(capsys.readouterr().out) - assert "SQBRSQL040" in json.dumps(payload["findings"]) - assert "SQBRSQL041" in json.dumps(payload["findings"]) + assert "SQLCLEAN040" in json.dumps(payload["findings"]) + assert "SQLCLEAN041" in json.dumps(payload["findings"]) @pytest.mark.parametrize( @@ -251,10 +253,10 @@ def test_given_lateral_join_when_running_expression_rule_then_literals_pass_and_ "SELECT o.order_id FROM raw_orders AS o LEFT JOIN LATERAL FLATTEN(input => o.items) AS f " f"ON {test_case.sql}\n" ) - result: int = main(["--project-dir", str(tmp_path), "rules", "--json", "run", "SQBRSQL040"]) + result: int = main(["--project-dir", str(tmp_path), "rules", "--json", "run", "SQLCLEAN040"]) payload: dict[str, object] = json.loads(capsys.readouterr().out) assert result == int(test_case.expected_fault) - assert ("SQBRSQL040" in json.dumps(payload["findings"])) == test_case.expected_fault + assert ("SQLCLEAN040" in json.dumps(payload["findings"])) == test_case.expected_fault if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_unicode_lint.py b/tests/integration/src/sqlbuild/cli/commands/main/test_unicode_lint.py index f3c373b6d8..16d3aeb446 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_unicode_lint.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_unicode_lint.py @@ -26,58 +26,58 @@ ( UnicodeRuleLocationTestCase( "uncontrolled star", - "SQBRSQL021", + "SQLCLEAN021", "SELECT 'LABEL' AS label, * FROM orders", ("* FROM",), ), UnicodeRuleLocationTestCase( - "controlled projection", "SQBRSQL021", "SELECT 'LABEL' AS label", () + "controlled projection", "SQLCLEAN021", "SELECT 'LABEL' AS label", () ), UnicodeRuleLocationTestCase( "computed terminal", - "SQBRSQL035", + "SQLCLEAN035", "WITH final AS (SELECT 'LABEL' AS label) SELECT label || 'x' AS label FROM final", ("SELECT label ||",), ), UnicodeRuleLocationTestCase( "plain terminal", - "SQBRSQL035", + "SQLCLEAN035", "((WITH final AS (SELECT 'LABEL' AS label) SELECT label FROM final))", (), ), UnicodeRuleLocationTestCase( "nested CTE", - "SQBRSQL036", + "SQLCLEAN036", "SELECT 'LABEL' AS label, (WITH nested_orders AS (SELECT 1 AS order_id) SELECT order_id FROM nested_orders) AS order_id", ("WITH nested_orders",), ), UnicodeRuleLocationTestCase( "root wrapper", - "SQBRSQL036", + "SQLCLEAN036", "((WITH final AS (SELECT 'LABEL' AS label) SELECT label FROM final))", (), ), UnicodeRuleLocationTestCase( "computed join", - "SQBRSQL040", + "SQLCLEAN040", "WITH orders AS (SELECT 'LABEL' AS label) SELECT o.label FROM orders AS o JOIN orders AS c ON LOWER(o.label) = c.label", ("ON LOWER",), ), UnicodeRuleLocationTestCase( "plain join", - "SQBRSQL040", + "SQLCLEAN040", "WITH orders AS (SELECT 'LABEL' AS label) SELECT o.label FROM orders AS o JOIN orders AS c ON o.label = c.label", (), ), UnicodeRuleLocationTestCase( "wrong final name", - "SQBRSQL041", + "SQLCLEAN041", "WITH marker AS (SELECT 'LABEL' AS label), orders AS (SELECT label FROM marker) SELECT label FROM orders", ("orders AS",), ), UnicodeRuleLocationTestCase( "valid final name", - "SQBRSQL041", + "SQLCLEAN041", "((WITH marker AS (SELECT 'LABEL' AS label), final AS (SELECT label FROM marker) SELECT label FROM final))", (), ), diff --git a/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py b/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py index 747ab04902..cc85d0fd85 100644 --- a/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py +++ b/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py @@ -197,14 +197,14 @@ ), PythonFreeTestCase( description="rules show cross join", - args=("rules", "show", "SQBRSQL038"), + args=("rules", "show", "SQLCLEAN038"), prepare=prepare_project, expected_exit_code=0, expected_captures=1, ), PythonFreeTestCase( description="rules show inline directive rule", - args=("rules", "show", "SQBRSQL000"), + args=("rules", "show", "SQBRSUPPRESSION000"), prepare=prepare_project, expected_exit_code=2, expected_captures=1, diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py index de0da599de..a1c9043988 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py @@ -27,8 +27,8 @@ CUSTOMERS_MODEL: str = ( 'MODEL (\n description "One row per customer",\n materialized table,\n' " columns (email (audits [not_null(severity warn)])),\n);\n\n" - "SELECT * FROM (VALUES (1, 'ada@example.com'), (2, NULL), (3, NULL)) " - "AS customers(customer_id, email)\n" + "WITH customers(customer_id, email) AS (VALUES (1, 'ada@example.com'), (2, NULL), (3, NULL))\n" + "SELECT customers.customer_id, customers.email FROM customers\n" ) COSTED_ADAPTER: str = ( diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index e5f8e00ff5..c089c2e535 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -50,7 +50,7 @@ _REACQUIRE_DELAY_SECONDS: float = 0.05 _INTERRUPT_TIMEOUT_SECONDS: float = 30.0 _INTERRUPT_DELAY_SECONDS: float = 0.15 -_SEMANTIC_BINDING_PROJECT_TOML: str = 'name = "semantic_binding"\nadapter = "duckdb"\n' +SEMANTIC_BINDING_PROJECT_TOML: str = 'name = "semantic_binding"\nadapter = "duckdb"\n' _AUDIT_FACTORY_ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( value_renderer=DuckDbAdapter(), collection_rendering=CollectionRendering.VALUE_LIST, @@ -97,6 +97,7 @@ def run_compile_pipeline_for_project( exclude: tuple[str, ...] = (), on_progress: Callable[[str], None] | None = None, resolve_python_run_selectors: bool = False, + no_sql_validation: bool = True, ) -> CompilePipelineResult: """Discover a project and run the real compile pipeline.""" @@ -105,7 +106,7 @@ def run_compile_pipeline_for_project( discovered_inputs=discovered_inputs, adapter=adapter, options=CompilePipelineOptions( - no_sql_validation=True, + no_sql_validation=no_sql_validation, defer_to=defer_to, select=select, exclude=exclude, @@ -147,11 +148,13 @@ def validate_manifest_against_dbt_schema(manifest: dict[str, object]) -> None: def write_semantic_binding_project( - *, project_dir: Path, upstream_sql: str, downstream_sql: str + *, + project_dir: Path, + upstream_sql: str, + downstream_sql: str, + project_toml: str = SEMANTIC_BINDING_PROJECT_TOML, ) -> None: - _ = (project_dir / "sqlbuild_project.toml").write_text( - _SEMANTIC_BINDING_PROJECT_TOML, encoding="utf-8" - ) + _ = (project_dir / "sqlbuild_project.toml").write_text(project_toml, encoding="utf-8") models_dir: Path = project_dir / "models" models_dir.mkdir() _ = (models_dir / "upstream.sql").write_text(upstream_sql, encoding="utf-8") diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index 06c90ea81f..f88a27c3f4 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -120,7 +120,7 @@ def test_given_typed_cte_chain_when_compiling_contract_then_type_remains_proven( [ RulesPipelineIntegrationTestCase( description="configured Rules block planning before a warehouse connection", - expected_error_pattern="SQBRSQL004", + expected_error_pattern="SQLCLEAN004", expected_connection_calls=0, ) ], @@ -132,12 +132,13 @@ def test_given_rule_finding_when_running_shared_compile_pipeline_then_planning_i monkeypatch: pytest.MonkeyPatch, ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( - f'{_PROJECT_TOML}\n[rules]\nselect = ["SQBRSQL004"]\n', encoding="utf-8" + f'{_PROJECT_TOML}\n[rules]\nselect = ["SQLCLEAN004"]\n', encoding="utf-8" ) model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() model.write_text( - 'MODEL (description "Orders");\nSELECT order_id FROM orders LIMIT 1\n', + 'MODEL (description "Orders");\n' + "WITH orders(order_id) AS (VALUES (1), (2))\nSELECT order_id FROM orders LIMIT 1\n", encoding="utf-8", ) adapter: DuckDbAdapter = DuckDbAdapter() @@ -145,7 +146,9 @@ def test_given_rule_finding_when_running_shared_compile_pipeline_then_planning_i monkeypatch.setattr(adapter, "connect", connect) with pytest.raises(RulesError, match=test_case.expected_error_pattern): - _ = run_compile_pipeline_for_project(project_dir=tmp_path, adapter=adapter) + _ = run_compile_pipeline_for_project( + project_dir=tmp_path, adapter=adapter, no_sql_validation=False + ) assert connect.call_count == test_case.expected_connection_calls diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_binding.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_binding.py index bc01ad1c82..4cca144777 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_binding.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_binding.py @@ -14,7 +14,10 @@ SemanticBindingIntegrationTestCase, SourceContractDefaultIntegrationTestCase, ) -from tests.integration.src.sqlbuild.compiler.pipeline.helpers import write_semantic_binding_project +from tests.integration.src.sqlbuild.compiler.pipeline.helpers import ( + SEMANTIC_BINDING_PROJECT_TOML, + write_semantic_binding_project, +) _AUTHORITATIVE_MODEL: str = """MODEL (description "Test model.", materialized table @@ -28,6 +31,10 @@ """ +# Some binding cases rank by a column with no proven key, which SQLCLEAN018 reports. +_UNPROVEN_RANKING_RULES: str = '\n[rules]\nignore = ["SQLCLEAN018"]\n' + + @pytest.mark.parametrize( "test_case", [ @@ -497,6 +504,7 @@ def test_given_valid_using_and_qualify_alias_when_binding_then_compile_succeeds( 'FROM __ref("upstream") l JOIN __ref("upstream") r USING (id) ' "QUALIFY rn = 1\n" ), + project_toml=SEMANTIC_BINDING_PROJECT_TOML + _UNPROVEN_RANKING_RULES, ) exit_code: int = main(["--no-color", "--project-dir", str(tmp_path), "compile", "--no-cache"]) @@ -600,7 +608,8 @@ def test_given_untyped_source_when_cast_flows_through_ctes_then_declared_type_ma capsys: pytest.CaptureFixture[str], ) -> None: _ = (tmp_path / "sqlbuild_project.toml").write_text( - 'name = "semantic_binding"\nadapter = "snowflake"\n', encoding="utf-8" + 'name = "semantic_binding"\nadapter = "snowflake"\n' + _UNPROVEN_RANKING_RULES, + encoding="utf-8", ) sources_dir: Path = tmp_path / "sources" models_dir: Path = tmp_path / "models" diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_scopes.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_scopes.py index e4de212a21..a5274acf80 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_scopes.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_scopes.py @@ -11,7 +11,10 @@ from sqlbuild._native import run_cli as main from tests.integration.src.sqlbuild.compiler.pipeline._test_types import SemanticCompileCase -from tests.integration.src.sqlbuild.compiler.pipeline.helpers import write_semantic_binding_project +from tests.integration.src.sqlbuild.compiler.pipeline.helpers import ( + SEMANTIC_BINDING_PROJECT_TOML, + write_semantic_binding_project, +) _UPSTREAM: str = "SELECT 1 AS id, 'placed' AS status" @@ -119,6 +122,7 @@ def test_given_semantic_scope_when_compiling_then_closed_shapes_are_checked( project_dir=tmp_path, upstream_sql="MODEL (description 'Test model.', materialized view);\n" + test_case.upstream, downstream_sql=test_case.header + test_case.downstream, + project_toml=SEMANTIC_BINDING_PROJECT_TOML + '\n[rules]\nignore = ["SQLCLEAN039"]\n', ) (tmp_path / "sources").mkdir(exist_ok=True) (tmp_path / "sources" / "orders.yml").write_text( diff --git a/tests/integration/src/sqlbuild/executor/build/test_build_execution.py b/tests/integration/src/sqlbuild/executor/build/test_build_execution.py index 7a21d6e6e7..e3829b48c4 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_build_execution.py +++ b/tests/integration/src/sqlbuild/executor/build/test_build_execution.py @@ -112,8 +112,9 @@ ), "models/validated_orders.sql": ( "MODEL (description 'Test model validated_orders.', materialized table);\n\n" - 'SELECT value, __udf("is_positive_int")(value) AS is_positive ' - "FROM (VALUES ('123'), ('abc')) AS input(value)" + "WITH input(value) AS (VALUES ('123'), ('abc'))\n" + 'SELECT input.value, __udf("is_positive_int")(input.value) AS is_positive ' + "FROM input" ), }, expected_status=BuildStatus.SUCCESS, @@ -147,8 +148,9 @@ ), "models/validated_orders.sql": ( "MODEL (description 'Test model validated_orders.', materialized table);\n\n" - 'SELECT value, __udf("is_positive_int")(value) AS is_positive ' - "FROM (VALUES ('123'), ('abc')) AS input(value)" + "WITH input(value) AS (VALUES ('123'), ('abc'))\n" + 'SELECT input.value, __udf("is_positive_int")(input.value) AS is_positive ' + "FROM input" ), }, expected_status=BuildStatus.SUCCESS, @@ -185,8 +187,9 @@ ), "models/validated_orders.sql": ( "MODEL (description 'Test model validated_orders.', materialized table);\n\n" - 'SELECT value, __udf("is_positive_int")(value) AS is_positive ' - "FROM (VALUES ('123'), ('abc')) AS input(value)" + "WITH input(value) AS (VALUES ('123'), ('abc'))\n" + 'SELECT input.value, __udf("is_positive_int")(input.value) AS is_positive ' + "FROM input" ), }, expected_status=BuildStatus.SUCCESS, @@ -310,7 +313,7 @@ "sqlbuild_project.toml": _PROJECT_YML, "models/orders.sql": ( "MODEL (description 'Test model orders.', materialized table, columns (id (audits [unique])));\n\n" - "SELECT * FROM (VALUES (1), (NULL), (NULL)) AS input(id)" + "WITH input(id) AS (VALUES (1), (NULL), (NULL))\nSELECT input.id FROM input" ), }, expected_status=BuildStatus.SUCCESS, @@ -327,7 +330,7 @@ "models/orders.sql": ( "MODEL (description 'Test model orders.', materialized table, columns (status (audits [accepted_values " '(values ["placed", "completed"])])));\n\n' - "SELECT * FROM (VALUES ('placed'), ('completed'), (NULL)) AS input(status)" + "WITH input(status) AS (VALUES ('placed'), ('completed'), (NULL))\nSELECT input.status FROM input" ), }, expected_status=BuildStatus.SUCCESS, @@ -859,7 +862,7 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( "sqlbuild_project.toml": _PROJECT_YML, "models/orders.sql": ( "MODEL (description 'Test model orders.', materialized table, columns (id (audits [unique])));\n\n" - "SELECT * FROM (VALUES (1), (1), (NULL)) AS input(id)" + "WITH input(id) AS (VALUES (1), (1), (NULL))\nSELECT input.id FROM input" ), }, expected_status=BuildStatus.FAILED, diff --git a/tests/integration/src/sqlbuild/executor/build/test_measurement_audit_execution.py b/tests/integration/src/sqlbuild/executor/build/test_measurement_audit_execution.py index 8efa445fc4..657dee4e29 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_measurement_audit_execution.py +++ b/tests/integration/src/sqlbuild/executor/build/test_measurement_audit_execution.py @@ -67,7 +67,8 @@ def test_given_measurement_outcomes_when_building_then_gates_and_persists_canoni ) ] ); - SELECT * FROM (VALUES (1), (2), (3)) AS orders(order_id) + WITH orders(order_id) AS (VALUES (1), (2), (3)) + SELECT orders.order_id FROM orders """ audit_sql = """ AUDIT ( diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py index 00059e1f38..1316664de7 100644 --- a/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py @@ -8,7 +8,7 @@ from sqlbuild._native import run_cli as main RULES_FIXTURE: Path = Path("tests/e2e/fixtures/waffle_shop") -RULES_CONFIG: str = '\n[rules]\nselect = ["SQBR"]\n' +RULES_CONFIG: str = '\n[rules]\nselect = ["SQBR", "SQLCLEAN"]\n' EDITED_MODEL: str = "models/staging/stg_orders.sql" type FindingKey = tuple[str, str, int | None, int | None, str] diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py index ed977f959d..5d5cfb2244 100644 --- a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py @@ -63,7 +63,7 @@ def test_given_increasing_findings_when_publishing_cache_then_findings_are_store project_dir=project_dir, project=project, model_paths=frozenset(paths), - selected=(SimpleNamespace(code="SQBRSQL004", custom=False),), + selected=(SimpleNamespace(code="SQLCLEAN004", custom=False),), effective_config=SimpleNamespace(thresholds={}, cache=SimpleNamespace(enabled=True)), ) result: SqlRulesEvaluation = native.run_planned_sql_rules( diff --git a/tests/unit/scripts/compiler_differential/_helpers/running/test_options.py b/tests/unit/scripts/compiler_differential/_helpers/running/test_options.py index f94ffa60e5..ccee4a3cc0 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/running/test_options.py +++ b/tests/unit/scripts/compiler_differential/_helpers/running/test_options.py @@ -27,8 +27,8 @@ ), ExpectedOutcomeTestCase( description="rule_code", - value="failure:SQBRSQL005", - expected_outcome=ExpectedOutcome(error_code="SQBRSQL005"), + value="failure:SQLCLEAN005", + expected_outcome=ExpectedOutcome(error_code="SQLCLEAN005"), ), ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index 1d901e9c4d..7921c088c7 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -287,7 +287,6 @@ def test_given_python_project_dag_flag_when_running_compile_then_writes_python_d ");\n\n" "SELECT\n" " CAST('1' AS VARCHAR) AS amount_cents\n" - "FROM (SELECT 1) AS source\n" ), ), CompileCommandTestCase( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index c9186d4175..52f80dc426 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -272,7 +272,7 @@ def run_dbt_plan( "rules", "--json", "run", - "SQBRSQL001", + "SQLCLEAN001", "--select", "commerce__mart__prices", "--exclude", @@ -303,7 +303,7 @@ def run_rules(request: RulesCommandRequest) -> int: project_dir=Path("/tmp/rules-project"), json_output=True, action="run", - rule_selector="SQBRSQL001", + rule_selector="SQLCLEAN001", skills_check=False, select=("commerce__mart__prices",), exclude=("legacy_model",), diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_expansion.py b/tests/unit/src/sqlbuild/lint/_helpers/test_expansion.py index 7333dffc6f..927a91f511 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_expansion.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_expansion.py @@ -46,7 +46,7 @@ "tests/unit/test_orders.sql": _ORDERS_TEST, "audits/generic/not_null.sql": _NOT_NULL_AUDIT, }, - expected_codes=("SQBRSQL001",), + expected_codes=("SQLCLEAN001",), ) ], ids=lambda case: case.description, @@ -91,7 +91,7 @@ def test_given_unexpandable_files_when_linting_then_first_in_file_order_is_raise UnexpandableSkipTestCase( description="skipped unexpandable files are reported and the rest is linted", expected_unexpandable=("models/aa_broken.sql", "models/zz_broken.sql"), - expected_codes=("SQBRSQL001",), + expected_codes=("SQLCLEAN001",), ) ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_native_sql.py b/tests/unit/src/sqlbuild/lint/_helpers/test_native_sql.py index f3585609cc..05063b8b0c 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_native_sql.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_native_sql.py @@ -65,7 +65,7 @@ def test_given_reserved_harness_cte_when_linting_then_framework_input_is_not_rep "ELSE FALSE END AS actionable " "FROM support_tickets" ), - expected_code="SQBRSQL030", + expected_code="SQLCLEAN030", ) ], ids=lambda case: case.description, @@ -104,7 +104,7 @@ def test_given_multi_branch_boolean_case_when_linting_then_partial_fix_is_withhe NativeSqlFixTestCase( description="equal-length expansion wholly inside an unused CTE", sql="WITH unused AS (SELECT 42 AS order_id) SELECT 1 AS order_id", - expected_code="SQBRSQL005", + expected_code="SQLCLEAN005", ) ], ids=lambda case: case.description, @@ -155,7 +155,7 @@ def test_given_generated_interior_with_authored_boundaries_when_linting_then_fix "CROSS JOIN processing_cutoff AS cutoff " "WHERE orders.created_at < cutoff.created_at" ), - selected_rule="SQBRSQL032", + selected_rule="SQLCLEAN032", expected_violation_count=0, ), NativeSqlFindingTestCase( @@ -166,7 +166,7 @@ def test_given_generated_interior_with_authored_boundaries_when_linting_then_fix "CROSS JOIN LATERAL UNNEST(orders.product_ids) AS item " "INNER JOIN products ON (products.product_id = item.product_id)" ), - selected_rule="SQBRSQL032", + selected_rule="SQLCLEAN032", expected_violation_count=0, ), ], @@ -207,7 +207,7 @@ def test_given_cross_join_used_downstream_when_linting_then_no_unused_join_is_re prefix="-- orders header\n-- second header line\n\n", unused_cte_count=40, file_count=3, - expected_code="SQBRSQL005", + expected_code="SQLCLEAN005", ), ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py b/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py index a10f43ea59..ecc62b6416 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py @@ -76,61 +76,61 @@ def test_given_unicode_before_intrinsics_when_preparing_then_lint_preserves_sema NeutralizeInterpolationTestCase( description="bare audit parameter becomes a sentinel", body="SELECT * FROM t WHERE NOT (@expression)", - expected_neutralized="SELECT * FROM t WHERE NOT (__sqb_lint_0__)", + expected_neutralized="SELECT * FROM t WHERE NOT (__sqlclean_interpolation_0__)", expected_original_texts=("@expression",), ), NeutralizeInterpolationTestCase( description="macro call including arguments becomes one sentinel", body="SELECT @py.cents_to_dollars(amount_cents) AS amount", - expected_neutralized="SELECT __sqb_lint_0__ AS amount", + expected_neutralized="SELECT __sqlclean_interpolation_0__ AS amount", expected_original_texts=("@py.cents_to_dollars(amount_cents)",), ), NeutralizeInterpolationTestCase( description="macro call with python valued arguments becomes one sentinel", body="SELECT @py.pick(columns=['a', 'b']) AS chosen", - expected_neutralized="SELECT __sqb_lint_0__ AS chosen", + expected_neutralized="SELECT __sqlclean_interpolation_0__ AS chosen", expected_original_texts=("@py.pick(columns=['a', 'b'])",), ), NeutralizeInterpolationTestCase( description="nested macro parentheses are matched", body="SELECT @py.outer(@py.inner(x), y) AS v", - expected_neutralized="SELECT __sqb_lint_0__ AS v", + expected_neutralized="SELECT __sqlclean_interpolation_0__ AS v", expected_original_texts=("@py.outer(@py.inner(x), y)",), ), NeutralizeInterpolationTestCase( description="quoted audit parameter becomes a sentinel", body="SELECT @'column_name' AS c", - expected_neutralized="SELECT __sqb_lint_0__ AS c", + expected_neutralized="SELECT __sqlclean_interpolation_0__ AS c", expected_original_texts=("@'column_name'",), ), NeutralizeInterpolationTestCase( description="project variable interpolation becomes a sentinel", body="SELECT * FROM @@source_table", - expected_neutralized="SELECT * FROM __sqb_lint_0__", + expected_neutralized="SELECT * FROM __sqlclean_interpolation_0__", expected_original_texts=("@@source_table",), ), NeutralizeInterpolationTestCase( description="environment interpolation becomes a sentinel", body="SELECT * FROM @@ENV:TARGET_TABLE", - expected_neutralized="SELECT * FROM __sqb_lint_0__", + expected_neutralized="SELECT * FROM __sqlclean_interpolation_0__", expected_original_texts=("@@ENV:TARGET_TABLE",), ), NeutralizeInterpolationTestCase( description="context interpolation becomes a sentinel", body="SELECT @@CTX:run_id AS run", - expected_neutralized="SELECT __sqb_lint_0__ AS run", + expected_neutralized="SELECT __sqlclean_interpolation_0__ AS run", expected_original_texts=("@@CTX:run_id",), ), NeutralizeInterpolationTestCase( description="template expression becomes a sentinel", body="SELECT ${if(eq(a, b), x, y)} AS v", - expected_neutralized="SELECT __sqb_lint_0__ AS v", + expected_neutralized="SELECT __sqlclean_interpolation_0__ AS v", expected_original_texts=("${if(eq(a, b), x, y)}",), ), NeutralizeInterpolationTestCase( description="each site receives a distinct sentinel", body="SELECT @a, @a FROM t", - expected_neutralized="SELECT __sqb_lint_0__, __sqb_lint_1__ FROM t", + expected_neutralized="SELECT __sqlclean_interpolation_0__, __sqlclean_interpolation_1__ FROM t", expected_original_texts=("@a", "@a"), ), NeutralizeInterpolationTestCase( @@ -142,7 +142,7 @@ def test_given_unicode_before_intrinsics_when_preparing_then_lint_preserves_sema NeutralizeInterpolationTestCase( description="SQLBuild reference call becomes one sentinel", body='SELECT * FROM __ref("@model")', - expected_neutralized="SELECT * FROM __sqb_lint_0__", + expected_neutralized="SELECT * FROM __sqlclean_interpolation_0__", expected_original_texts=('__ref("@model")',), ), NeutralizeInterpolationTestCase( @@ -151,14 +151,14 @@ def test_given_unicode_before_intrinsics_when_preparing_then_lint_preserves_sema 'SELECT * FROM __table_fn("events")((SELECT ARRAY_AGG(event_id) FROM event_ids))' ), expected_neutralized=( - "SELECT * FROM __sqb_lint_0__((SELECT ARRAY_AGG(event_id) FROM event_ids))" + "SELECT * FROM __sqlclean_interpolation_0__((SELECT ARRAY_AGG(event_id) FROM event_ids))" ), expected_original_texts=('__table_fn("events")',), ), NeutralizeInterpolationTestCase( description="escaped SQL quote does not hide a later table function", body="SELECT 'can''t'; SELECT * FROM __table_fn(\"events\")((SELECT 1))", - expected_neutralized="SELECT 'can''t'; SELECT * FROM __sqb_lint_0__((SELECT 1))", + expected_neutralized="SELECT 'can''t'; SELECT * FROM __sqlclean_interpolation_0__((SELECT 1))", expected_original_texts=('__table_fn("events")',), ), NeutralizeInterpolationTestCase( @@ -167,7 +167,7 @@ def test_given_unicode_before_intrinsics_when_preparing_then_lint_preserves_sema '/* Snowflake\'s table function */\nSELECT * FROM __table_fn("events")((SELECT 1))' ), expected_neutralized=( - "/* Snowflake's table function */\nSELECT * FROM __sqb_lint_0__((SELECT 1))" + "/* Snowflake's table function */\nSELECT * FROM __sqlclean_interpolation_0__((SELECT 1))" ), expected_original_texts=('__table_fn("events")',), ), @@ -234,28 +234,28 @@ def test_given_body_when_neutralizing_then_sentinels_replace_interpolation( description="databricks backtick keeps a close paren inside a macro call", dialect="databricks", body="SELECT @py.m(`a)b` x) AS y", - expected_neutralized="SELECT __sqb_lint_0__ AS y", + expected_neutralized="SELECT __sqlclean_interpolation_0__ AS y", expected_original_texts=("@py.m(`a)b` x)",), ), DialectNeutralizeInterpolationTestCase( description="bigquery backtick apostrophe does not hide a later macro", dialect="bigquery", body='SELECT `customer\'s id` AS customer_id, @py.label("order_id") AS order_id', - expected_neutralized="SELECT `customer's id` AS customer_id, __sqb_lint_0__ AS order_id", + expected_neutralized="SELECT `customer's id` AS customer_id, __sqlclean_interpolation_0__ AS order_id", expected_original_texts=('@py.label("order_id")',), ), DialectNeutralizeInterpolationTestCase( description="postgres backtick remains ordinary code", dialect="postgres", body="SELECT @py.m(`a)b` x) AS y", - expected_neutralized="SELECT __sqb_lint_0__b` x) AS y", + expected_neutralized="SELECT __sqlclean_interpolation_0__b` x) AS y", expected_original_texts=("@py.m(`a)",), ), DialectNeutralizeInterpolationTestCase( description="sqlserver backtick remains ordinary code", dialect="tsql", body="SELECT @py.m(`a)b` x) AS y", - expected_neutralized="SELECT __sqb_lint_0__b` x) AS y", + expected_neutralized="SELECT __sqlclean_interpolation_0__b` x) AS y", expected_original_texts=("@py.m(`a)",), ), ], @@ -291,7 +291,7 @@ def test_given_dialect_quoting_when_neutralizing_then_python_and_native_sites_ag description="sentinels are unique across sites", body="SELECT @a, @b, @@c, ${d} FROM t", expected_neutralized=( - "SELECT __sqb_lint_0__, __sqb_lint_1__, __sqb_lint_2__, __sqb_lint_3__ FROM t" + "SELECT __sqlclean_interpolation_0__, __sqlclean_interpolation_1__, __sqlclean_interpolation_2__, __sqlclean_interpolation_3__ FROM t" ), expected_original_texts=("@a", "@b", "@@c", "${d}"), ), @@ -315,13 +315,13 @@ def test_given_many_sites_when_neutralizing_then_every_sentinel_is_distinct( RestoreInterpolationTestCase( description="unchanged neutralized text round-trips to the original body", body="SELECT @py.cents_to_dollars(amount) AS a FROM t", - fixed_neutralized="SELECT __sqb_lint_0__ AS a FROM t", + fixed_neutralized="SELECT __sqlclean_interpolation_0__ AS a FROM t", expected_restored="SELECT @py.cents_to_dollars(amount) AS a FROM t", ), RestoreInterpolationTestCase( description="reformatted layout keeps interpolation intact", body="select @a as x,\n @b from t", - fixed_neutralized="SELECT\n __sqb_lint_0__ AS x,\n __sqb_lint_1__\nFROM t", + fixed_neutralized="SELECT\n __sqlclean_interpolation_0__ AS x,\n __sqlclean_interpolation_1__\nFROM t", expected_restored="SELECT\n @a AS x,\n @b\nFROM t", ), RestoreInterpolationTestCase( @@ -333,13 +333,13 @@ def test_given_many_sites_when_neutralizing_then_every_sentinel_is_distinct( RestoreInterpolationTestCase( description="repeated identical interpolation restores both sites", body="SELECT @a, @a FROM t", - fixed_neutralized="SELECT __sqb_lint_0__, __sqb_lint_1__ FROM t", + fixed_neutralized="SELECT __sqlclean_interpolation_0__, __sqlclean_interpolation_1__ FROM t", expected_restored="SELECT @a, @a FROM t", ), RestoreInterpolationTestCase( description="real identifier matching the interpolation name is untouched", body="SELECT expression, @expression FROM t", - fixed_neutralized="SELECT expression, __sqb_lint_0__ FROM t", + fixed_neutralized="SELECT expression, __sqlclean_interpolation_0__ FROM t", expected_restored="SELECT expression, @expression FROM t", ), RestoreInterpolationTestCase( @@ -350,7 +350,7 @@ def test_given_many_sites_when_neutralizing_then_every_sentinel_is_distinct( ), fixed_neutralized=( "SELECT expanded.order_id FROM orders CROSS JOIN " - "__SQB_LINT_0__(orders.order_id) AS expanded" + "__SQLCLEAN_INTERPOLATION_0__(orders.order_id) AS expanded" ), expected_restored=( "SELECT expanded.order_id FROM orders CROSS JOIN " @@ -382,7 +382,7 @@ def test_given_fixed_text_when_restoring_then_original_interpolation_returns( RestoreFailureTestCase( description="duplicated sentinel fails instead of duplicating interpolation", body="SELECT @a FROM t", - fixed_neutralized="SELECT __sqb_lint_0__, __sqb_lint_0__ FROM t", + fixed_neutralized="SELECT __sqlclean_interpolation_0__, __sqlclean_interpolation_0__ FROM t", expected_message_fragment="contains 2 occurrences", ), ], diff --git a/tests/unit/src/sqlbuild/lint/helpers.py b/tests/unit/src/sqlbuild/lint/helpers.py index 70ab1cac2c..27e5e38643 100644 --- a/tests/unit/src/sqlbuild/lint/helpers.py +++ b/tests/unit/src/sqlbuild/lint/helpers.py @@ -2,7 +2,6 @@ import itertools import json -import re from pathlib import Path from typing import cast @@ -50,14 +49,33 @@ def write_fixture_format_project( return test_file -_RESERVED_KEYWORDS_DIR: Path = ( - Path(__file__).resolve().parents[5] - / "crates" - / "sqlbuild-analysis" - / "src" - / "sql_tokens" - / "reserved_keywords" -) +_RESERVED_NAMES: dict[str, frozenset[str]] = { + "duckdb": frozenset({"any", "only", "order", "pivot", "some"}), + "postgres": frozenset({"any", "only", "order", "some"}), + "bigquery": frozenset( + { + "any", + "exclude", + "ignore", + "left", + "nulls", + "order", + "preceding", + "range", + "recursive", + "rows", + "some", + "tablesample", + "within", + } + ), + "snowflake": frozenset({"any", "order", "row", "rows", "some", "tablesample"}), + "tsql": frozenset( + {"any", "index", "key", "left", "order", "pivot", "some", "tablesample", "view"} + ), + "databricks": frozenset(), +} +_FOLDED_KEYWORDS: tuple[str, ...] = ("select", "from", "where", "union", "order") _DIALECTS: tuple[str, ...] = ("duckdb", "postgres", "bigquery", "snowflake", "tsql", "databricks") _KEYWORD_NAMED_IDENTIFIERS: tuple[str, ...] = ( "Left", @@ -102,10 +120,9 @@ def write_fixture_format_project( def reserved_keywords(dialect: str) -> frozenset[str]: - """Return the lower-case reserved keywords of one dialect from the native data files.""" + """Return which of the keyword-named identifiers SQLClean reserves in one dialect.""" - text: str = (_RESERVED_KEYWORDS_DIR / f"{dialect}.txt").read_text(encoding="utf-8") - return frozenset(re.findall(r"^[a-z_]+$", text, flags=re.MULTILINE)) + return _RESERVED_NAMES[dialect] def keyword_name_positions() -> tuple[tuple[str, str, str, str], ...]: @@ -128,12 +145,14 @@ def keyword_name_positions() -> tuple[tuple[str, str, str, str], ...]: def reserved_keyword_pairs() -> tuple[tuple[str, str], ...]: - """Return every (dialect, reserved keyword) pair from the native data files.""" + """Return (dialect, keyword) pairs for core keywords every dialect but Databricks reserves.""" - pairs: list[tuple[str, str]] = [] - for dialect in _DIALECTS: - pairs.extend((dialect, keyword) for keyword in sorted(reserved_keywords(dialect))) - return tuple(pairs) + return tuple( + (dialect, keyword) + for dialect in _DIALECTS + if _RESERVED_NAMES[dialect] + for keyword in _FOLDED_KEYWORDS + ) def format_project_json( diff --git a/tests/unit/src/sqlbuild/lint/test_cli_format.py b/tests/unit/src/sqlbuild/lint/test_cli_format.py index 3afec6422f..e0146adeba 100644 --- a/tests/unit/src/sqlbuild/lint/test_cli_format.py +++ b/tests/unit/src/sqlbuild/lint/test_cli_format.py @@ -58,7 +58,7 @@ def test_given_legacy_lint_configuration_when_formatting_then_it_is_rejected( capsys: pytest.CaptureFixture[str], ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( - f'{PROJECT_TOML}\n[lint]\nselect = ["SQBRSQL"]\n', encoding="utf-8" + f'{PROJECT_TOML}\n[lint]\nselect = ["SQLCLEAN"]\n', encoding="utf-8" ) model: Path = tmp_path / "models" / "orders.sql" model.parent.mkdir() @@ -72,7 +72,7 @@ def test_given_legacy_lint_configuration_when_formatting_then_it_is_rejected( @pytest.mark.parametrize( "test_case", - [FormatCliTestCase("format does not enforce unselected Rules", (), 0, "SQBRSQL004")], + [FormatCliTestCase("format does not enforce unselected Rules", (), 0, "SQLCLEAN004")], ids=lambda case: case.description, ) def test_given_unselected_sql_rule_finding_when_formatting_then_rule_is_not_enforced( diff --git a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py index 6334604c9d..8f798ab4a9 100644 --- a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py @@ -266,7 +266,7 @@ def test_given_generic_audit_argument_when_mapping_then_generated_region_is_not_ context=context, dialect="duckdb", ) - sentinel_offset: int = body.lint_text.index("__sqlbuild_audit_parameter_0__") + sentinel_offset: int = body.lint_text.index("__sqlclean_relation_parameter_0__") mapped: MappedOffset = map_expanded_offset(offset=sentinel_offset, passes=body.passes) assert mapped.generated is test_case.expected_value diff --git a/tests/unit/src/sqlbuild/lint/test_run_lint.py b/tests/unit/src/sqlbuild/lint/test_run_lint.py index e93b623929..c19fcc0cd1 100644 --- a/tests/unit/src/sqlbuild/lint/test_run_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_run_lint.py @@ -289,7 +289,7 @@ def test_given_non_ascii_sql_when_native_linting_then_span_maps_to_authored_colu config=LintConfig(dialect="duckdb"), ) - assert tuple(item.code for item in result.violations) == ("SQBRSQL001",) + assert tuple(item.code for item in result.violations) == ("SQLCLEAN001",) assert (result.violations[0].line, result.violations[0].column) == test_case.expected_value assert (result.violations[0].end_line, result.violations[0].end_column) == (2, 45) assert result.violations[0].remediation == ("Use IS NULL or IS NOT NULL when testing for NULL.") @@ -414,12 +414,12 @@ def test_given_fixture_ceremony_variants_when_formatting_and_linting_then_forms_ InlineDirectiveTestCase( description="matching directive keeps the finding and fails", query="SELECT value FROM items LIMIT 1\n", - expected_violations=(("SQBRSQL000", 2, "fault"), ("SQBRSQL004", 3, "warning")), + expected_violations=(("SQBRSUPPRESSION000", 2, "fault"), ("SQLCLEAN004", 3, "warning")), ), InlineDirectiveTestCase( description="directive without a finding still fails", query="SELECT value FROM items ORDER BY value LIMIT 1\n", - expected_violations=(("SQBRSQL000", 2, "fault"),), + expected_violations=(("SQBRSUPPRESSION000", 2, "fault"),), ), ], ids=lambda case: case.description, @@ -433,7 +433,7 @@ def test_given_inline_directive_when_linting_then_it_is_a_fault_printing_the_exc target.parent.mkdir() _ = target.write_text( 'MODEL (description "ok");\n' - "-- sqb: ignore SQBRSQL004 because this fixture intentionally samples one row\n" + "-- sqb: ignore SQLCLEAN004 because this fixture intentionally samples one row\n" f"{test_case.query}", encoding="utf-8", ) @@ -450,7 +450,7 @@ def test_given_inline_directive_when_linting_then_it_is_a_fault_printing_the_exc remediation: str = result.violations[0].remediation or "" assert ( "[[rules.rule_exceptions]]\n" - ' rule = "SQBRSQL004"\n' + ' rule = "SQLCLEAN004"\n' ' path = "models/sample.sql"\n' ' reason = "this fixture intentionally samples one row"' ) in remediation @@ -461,7 +461,7 @@ def test_given_inline_directive_when_linting_then_it_is_a_fault_printing_the_exc [ LintBehaviorTestCase( description="mandatory fault directive", - expected_value={"SQBRSQL000", "description-present"}, + expected_value={"SQBRSUPPRESSION000", "description-present"}, ) ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py index ef45160c21..8f3a5056a9 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py @@ -54,7 +54,7 @@ class RuleSignatureTestCase: @dataclass(frozen=True) class PolicySelectionTestCase: description: str - select: tuple[str, ...] + select: tuple[str, ...] | None ignore: tuple[str, ...] expected_codes: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py index c7558ff763..803a5d1ad3 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py @@ -68,8 +68,12 @@ "SQBRTEST203", "SQBRTEST301", ) -SQL_CODES: tuple[str, ...] = tuple(f"SQBRSQL{number:03d}" for number in range(1, 46)) -BUILT_IN_CODES: tuple[str, ...] = tuple(sorted((*RULES_BUILT_IN_CODES, *SQL_CODES))) +SQL_CODES: tuple[str, ...] = tuple(f"SQLCLEAN{number:03d}" for number in range(1, 54)) +SQL_DEFAULT_CODES: tuple[str, ...] = tuple( + f"SQLCLEAN{number:03d}" + for number in (1, 2, 3, 4, 5, 6, 7, 9, 10, 18, 19, 20, 39, 46, 47, 48, 51, 52, 53) +) +BUILT_IN_CODES: tuple[str, ...] = tuple(sorted(RULES_BUILT_IN_CODES)) STRUCTURE_CODES: tuple[str, ...] = ( "SQBRMODEL101", "SQBRMODEL102", @@ -119,7 +123,6 @@ "SQBRPROJECT203", "SQBRPROJECT204", *SQL_TEST_CODES, - *SQL_CODES, ) ) ) @@ -129,7 +132,7 @@ "test_case", ( PolicySelectionTestCase( - description="built-in namespace activates every built-in", + description="SQBR namespace activates every SQLBuild-owned built-in", select=("SQBR",), ignore=(), expected_codes=BUILT_IN_CODES, @@ -152,12 +155,36 @@ ignore=(), expected_codes=SQL_TEST_CODES, ), + PolicySelectionTestCase( + description="no selection activates SQLClean's default SQL Rules only", + select=None, + ignore=(), + expected_codes=SQL_DEFAULT_CODES, + ), PolicySelectionTestCase( description="empty selection activates no rules", select=(), ignore=(), expected_codes=(), ), + PolicySelectionTestCase( + description="SQLCLEAN family activates every SQL Rule, opt-in ones too", + select=("SQLCLEAN",), + ignore=(), + expected_codes=SQL_CODES, + ), + PolicySelectionTestCase( + description="exact opt-in SQL code activates only that Rule", + select=("SQLCLEAN050",), + ignore=(), + expected_codes=("SQLCLEAN050",), + ), + PolicySelectionTestCase( + description="ignoring the SQLCLEAN family without a selection activates nothing", + select=None, + ignore=("SQLCLEAN",), + expected_codes=(), + ), PolicySelectionTestCase( description="ignored family is removed from selected namespace", select=("SQBR",), diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py index 8a4696fe6b..e5ff522e95 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py @@ -21,11 +21,11 @@ ) _EXCEPTION_ENTRY: str = ( - '[[rules.rule_exceptions]]\nrule = "SQBRSQL004"\npath = "models/orders.sql"\n' + '[[rules.rule_exceptions]]\nrule = "SQLCLEAN004"\npath = "models/orders.sql"\n' 'reason = "A reviewed sample"\n' ) _IGNORE_ENTRY: str = ( - '[[rules.rule_ignores]]\nrules = ["SQBRSQL"]\npaths = ["models/examples/**"]\n' + '[[rules.rule_ignores]]\nrules = ["SQLCLEAN"]\npaths = ["models/examples/**"]\n' 'reason = "Examples keep minimal SQL"\n' ) @@ -168,15 +168,15 @@ RulesConfigErrorTestCase( description="relation for a Rule that reports no relation", source=( - '[[rules.rule_exceptions]]\nrule = "SQBRSQL004"\npath = "models/orders.sql"\n' + '[[rules.rule_exceptions]]\nrule = "SQLCLEAN004"\npath = "models/orders.sql"\n' 'relation = "customers"\nreason = "A reviewed sample"\n' ), - expected_error_pattern="sets relation, but only SQBRSQL038 report a joined relation", + expected_error_pattern="sets relation, but only SQLCLEAN038 report a joined relation", ), RulesConfigErrorTestCase( description="cte without relation", source=( - '[[rules.rule_exceptions]]\nrule = "SQBRSQL038"\npath = "models/orders.sql"\n' + '[[rules.rule_exceptions]]\nrule = "SQLCLEAN038"\npath = "models/orders.sql"\n' 'cte = "checks"\nreason = "Deliberate"\n' ), expected_error_pattern="sets cte without relation", @@ -323,7 +323,7 @@ def test_given_removed_allow_exceptions_when_loading_then_raises_error_explainin RelationExceptionConfigTestCase( description="relation-level exception narrowed to a CTE", source=( - '[[rules.rule_exceptions]]\nrule = "SQBRSQL038"\npath = "models/orders.sql"\n' + '[[rules.rule_exceptions]]\nrule = "SQLCLEAN038"\npath = "models/orders.sql"\n' 'relation = "order_policy"\ncte = "order_checks"\n' 'reason = "Every order is checked against every policy"\n' ), @@ -375,7 +375,7 @@ def test_given_pipeline_directory_when_loading_then_returns_typed_relative_confi RuleIgnoreConfigTestCase( description="graph selectors and paths can share one scoped ignore", source=( - '[[rules.rule_ignores]]\nrules = ["SQBRSQL021"]\n' + '[[rules.rule_ignores]]\nrules = ["SQLCLEAN021"]\n' 'paths = ["tests/**"]\nselectors = ["+orders", "tag:interface+"]\n' 'reason = "Reviewed interface resources"\n' ), diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_incremental_native_rules.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_incremental_native_rules.py index 5e03cc34d8..991de44938 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_incremental_native_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_incremental_native_rules.py @@ -92,8 +92,8 @@ IncrementalRulesModeTestCase( description="suppressions deferred with initial findings", defer_suppressions=True, - expected_observed_codes=("SQBRSQL034", "SQBRGRAPH102", "SQBRCONTRACT101"), - initial_codes=("SQBRSQL034",), + expected_observed_codes=("SQLCLEAN034", "SQBRGRAPH102", "SQBRCONTRACT101"), + initial_codes=("SQLCLEAN034",), ), ), ids=lambda case: case.description, diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index 86a02c0d07..d96b256882 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -97,7 +97,7 @@ A model with Rule errors starts with a header that marks it not buildable and li ```sql -- sqb: compile failed for this model; not buildable --- SQBRSQL028 at models/open_orders.sql:6:20: Single-relation query mixes qualified and unqualified columns +-- SQLCLEAN028 at models/open_orders.sql:6:20: Single-relation query mixes qualified and unqualified columns SELECT o.order_id, status FROM __ref("orders") AS o ``` diff --git a/website/src/content/docs/docs/cli/format.mdx b/website/src/content/docs/docs/cli/format.mdx index 0b0ad606e9..13963130ee 100644 --- a/website/src/content/docs/docs/cli/format.mdx +++ b/website/src/content/docs/docs/cli/format.mdx @@ -163,18 +163,18 @@ reason, and the run continues. A cycle or the 128-pass limit fails without writi | Rule | Decision | Reason | | --- | --- | --- | -| SQBRSQL001 NULL comparison | Refused | Replacing `= NULL` with `IS NULL` changes three-valued logic. | -| SQBRSQL002 implicit cartesian join | Applied | A comma-separated product becomes an explicit `CROSS JOIN`; mixed join precedence is excluded. | -| SQBRSQL003 conditionless join | Applied in Snowflake | Snowflake defines an unqualified `JOIN` without `ON` or `USING` as a cartesian product. Other dialects are refused because the original syntax may be invalid. | -| SQBRSQL005 unused CTE | Applied | Unreferenced `SELECT` CTEs contribute no rows. Mutation statements and commented CTEs are excluded. | -| SQBRSQL006 redundant DISTINCT | Applied | The projected and grouping expressions are proved to coincide. | -| SQBRSQL008 bare UNION | Applied | `UNION DISTINCT` states the existing duplicate elimination, in dialects that support it. | -| SQBRSQL013 parenthesized DISTINCT | Refused | Removing parentheses can change operator precedence. | -| SQBRSQL017 constant row count | Applied | `COUNT(1)` and `COUNT(*)` both count every row. | -| SQBRSQL023 unused table alias | Refused | Not yet reviewed as meaning-preserving. | -| SQBRSQL025 implicit inner join | Applied | A bare `JOIN ... ON` is an inner join. | -| SQBRSQL030 boolean CASE simplification | Refused | The `COALESCE` rewrite is not proved portable to every supported dialect. | -| SQBRSQL042 unused CTE output | Applied when row-neutral | Only a column reference or deterministic scalar expression is removed, from a CTE without `DISTINCT`, `DISTINCT ON`, `GROUP BY ALL`, `ORDER BY ALL`, an ungrouped aggregate, or positional `GROUP BY`/`ORDER BY`. Other removals could change the rows and are refused with the reason. | +| SQLCLEAN001 NULL comparison | Refused | Replacing `= NULL` with `IS NULL` changes three-valued logic. | +| SQLCLEAN002 implicit cartesian join | Applied | A comma-separated product becomes an explicit `CROSS JOIN`; mixed join precedence is excluded. | +| SQLCLEAN003 conditionless join | Applied in Snowflake | Snowflake defines an unqualified `JOIN` without `ON` or `USING` as a cartesian product. Other dialects are refused because the original syntax may be invalid. | +| SQLCLEAN005 unused CTE | Applied | Unreferenced `SELECT` CTEs contribute no rows. Mutation statements and commented CTEs are excluded. | +| SQLCLEAN006 redundant DISTINCT | Applied | The projected and grouping expressions are proved to coincide. | +| SQLCLEAN008 bare UNION | Applied | `UNION DISTINCT` states the existing duplicate elimination, in dialects that support it. | +| SQLCLEAN013 parenthesized DISTINCT | Refused | Removing parentheses can change operator precedence. | +| SQLCLEAN017 constant row count | Applied | `COUNT(1)` and `COUNT(*)` both count every row. | +| SQLCLEAN023 unused table alias | Refused | Not yet reviewed as meaning-preserving. | +| SQLCLEAN025 implicit inner join | Applied | A bare `JOIN ... ON` is an inner join. | +| SQLCLEAN030 boolean CASE simplification | Refused | The `COALESCE` rewrite is not proved portable to every supported dialect. | +| SQLCLEAN042 unused CTE output | Applied when row-neutral | Only a column reference or deterministic scalar expression is removed, from a CTE without `DISTINCT`, `DISTINCT ON`, `GROUP BY ALL`, `ORDER BY ALL`, an ungrouped aggregate, or positional `GROUP BY`/`ORDER BY`. Other removals could change the rows and are refused with the reason. | `--fix` also rewrites a header description that escapes double quotes as `\"` or doubles them as `""`, which descriptions do not support, as a `"""..."""` string holding real quotes, and reports diff --git a/website/src/content/docs/docs/cli/rules.mdx b/website/src/content/docs/docs/cli/rules.mdx index e41bc4f703..98088c8cc7 100644 --- a/website/src/content/docs/docs/cli/rules.mdx +++ b/website/src/content/docs/docs/cli/rules.mdx @@ -21,7 +21,7 @@ Add `--json` after `rules` for machine-readable list or run output: ```bash sqb rules --json list -sqb rules --json run SQBRSQL +sqb rules --json run SQLCLEAN ``` Lifecycle progress is written to stderr so stdout remains valid JSON. diff --git a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx index 921565ae55..857ea50498 100644 --- a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx +++ b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx @@ -301,7 +301,7 @@ fully typed. - Every read inside `USING (...)` stands unqualified, as `USING` needs, including `USING (@weekly.week)` for a column the type does not declare. - In a column-list macro, `o.@weekly.keys` outputs the group's columns under their own names. -- Rules judge a read as you wrote it. `SQBRSQL028`, which rejects a query mixing qualified and +- Rules judge a read as you wrote it. `SQLCLEAN028`, which rejects a query mixing qualified and unqualified columns, judges a query the macro writes whole, such as a CTE of a CTE-list macro, once, in the macro file. There every plain read counts as unqualified and every `alias.@weekly...` read as qualified. A step written only in plain reads, such as diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 6f428fd607..b7935a0301 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -685,7 +685,7 @@ the same checks: ```toml [rules] -select = ["SQBRSQL", "SQBRGRAPH", "XSQBRARCH"] +select = ["SQLCLEAN", "SQBRGRAPH", "XSQBRARCH"] ``` Rules are opt-in. Exact codes activate individual checks and prefixes activate a family. Built-in @@ -695,10 +695,10 @@ codes begin with `SQBR`; repository-defined codes begin with `XSQBR`. See | Key | Description | | --- | --- | | `ignore` | Rule codes or families turned off for the whole project. | -| `[[rules.rule_exceptions]]` | One reviewed exception: `rule`, `path` and `reason`, plus `relation` and optional `cte` for relation-level Rules such as `SQBRSQL038`. Stale entries fail compilation. See [Findings and exceptions](/docs/concepts/rules/findings-and-exceptions). | +| `[[rules.rule_exceptions]]` | One reviewed exception: `rule`, `path` and `reason`, plus `relation` and optional `cte` for relation-level Rules such as `SQLCLEAN038`. Stale entries fail compilation. See [Findings and exceptions](/docs/concepts/rules/findings-and-exceptions). | | `[[rules.rule_ignores]]` | Rules turned off for `paths` or `selectors`, with a `reason`. | -Rule exceptions live only in this file. Inline `-- sqb: ignore` comments are `SQBRSQL000` errors +Rule exceptions live only in this file. Inline `-- sqb: ignore` comments are `SQBRSUPPRESSION000` errors that print the entry to add. ## Scopes diff --git a/website/src/content/docs/docs/concepts/rules.mdx b/website/src/content/docs/docs/concepts/rules.mdx index c51cbf7234..297c6bd2a5 100644 --- a/website/src/content/docs/docs/concepts/rules.mdx +++ b/website/src/content/docs/docs/concepts/rules.mdx @@ -42,31 +42,31 @@ model. Use `sqb rules show` to understand a Rule before enabling it: ```bash -sqb rules show SQBRSQL004 +sqb rules show SQLCLEAN004 ``` ```text -SQBRSQL004: Row selection is nondeterministic -Family: SQBRSQL +SQLCLEAN004: Row selection is nondeterministic +Family: SQLCLEAN Slug: unordered-limit Subject: model Enabled by default: yes Remediation: Add ORDER BY with a deterministic tie-breaker before LIMIT or OFFSET. ``` -`Enabled by default` means the Rule is included when a matching family is selected. Configurable -Rules do not run unless `select` contains an exact code or matching family. +`Enabled by default` means the Rule runs when `select` is unset. Without `select`, SQLBuild runs +the default SQL Rules and no other configurable Rules. Select Rules in `sqlbuild_project.toml`. This example enables the complete built-in SQL family: ```toml [rules] -select = ["SQBRSQL"] +select = ["SQLCLEAN"] ``` -An exact code such as `SQBRSQL004` selects one Rule. A family code such as `SQBRSQL` selects every -Rule in that family. An empty `select` disables configurable Rules, but mandatory compiler -correctness still applies. +An exact code such as `SQLCLEAN004` selects one Rule. A family code such as `SQLCLEAN` selects every +Rule in that family, including Rules that are not enabled by default. To run no SQL Rules, add +`SQLCLEAN` to `ignore`; mandatory compiler correctness still applies. Now consider a model that returns an arbitrary row because its `LIMIT` has no ordering: @@ -84,7 +84,7 @@ sqb compile The selected Rule reports the source location, problem, and remediation: ```text -error[SQBRSQL004]: Row selection is nondeterministic +error[SQLCLEAN004]: Row selection is nondeterministic --> models/orders.sql:2:36 | 2 | SELECT order_id FROM orders LIMIT 1 @@ -103,17 +103,20 @@ Use focused commands when discovering or adopting Rules: ```bash sqb rules list -sqb rules run SQBRSQL004 +sqb rules run SQLCLEAN004 ``` A focused run checks only the requested Rule or family. Run `sqb compile` before treating the whole project as valid. -## Where SQL linting went +## SQL Rules -SQL linting is part of the Rules system rather than a separate command. +SQL linting is part of the Rules system rather than a separate command. The SQL Rules come from +[SQLClean](https://sqlclean.io), the SQL linter and formatter SQLBuild runs on every SQL body. +They have the same codes, defaults, findings and fixes as in the `sqlclean` command; see the +[SQLClean Rule reference](https://sqlclean.io/docs/rules/overview/). -The `SQBRSQL` built-in family covers deterministic SQL diagnostics such as: +The `SQLCLEAN` family covers deterministic SQL diagnostics such as: - unsafe `NULL` comparisons; - implicit cartesian joins; @@ -134,7 +137,7 @@ SQL responsibility: | Responsibility | Owner | Example | |---|---|---| | Correctness required to understand the project | Compiler | Invalid syntax, unresolved references, incompatible output shape | -| Configurable, deterministic requirements | `SQBRSQL` Rules (executed by the compiler) | Unordered `LIMIT`, unused CTE, implicit cartesian join | +| Configurable, deterministic requirements | `SQLCLEAN` Rules (executed by the compiler) | Unordered `LIMIT`, unused CTE, implicit cartesian join | | Canonical presentation | [`sqb format`](/docs/cli/format) (separate from the compiler) | Capitalization, indentation, spacing, comma and clause layout | Rules never rewrite source. `sqb format` rewrites source and does not decide whether a project @@ -142,8 +145,9 @@ passes its configured Rules. This keeps diagnostics and source mutation separate ## Built-in and custom Rules -SQLBuild maintains native built-in Rules for reusable requirements. Their codes begin with `SQBR`, -for example `SQBRSQL004` and `SQBRGRAPH101`. +SQLBuild maintains native built-in Rules for reusable requirements. SQL Rule codes begin with +`SQLCLEAN`, for example `SQLCLEAN004`. Rules that read the project, such as `SQBRGRAPH101`, begin +with `SQBR`. Projects can define custom Python Rules under `rules/**/*.py`. Their codes begin with `XSQBR`, for example `XSQBRARCH001`. Custom Rules use the same selection, findings, exceptions, execution order, @@ -152,7 +156,7 @@ and cache system as built-ins. ```toml [rules] select = [ - "SQBRSQL", + "SQLCLEAN", "SQBRGRAPH", "XSQBRARCH", ] diff --git a/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx b/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx index fdd98992e3..4c2e7b76ac 100644 --- a/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx +++ b/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx @@ -9,28 +9,34 @@ Configure built-in and custom Rules together: ```toml [rules] -select = ["SQBRSQL", "SQBRGRAPH", "XSQBRARCH"] -ignore = ["SQBRSQL004"] +select = ["SQLCLEAN", "SQBRGRAPH", "XSQBRARCH"] +ignore = ["SQLCLEAN004"] ``` -An empty `select` disables configurable Rules. It does not disable mandatory compiler correctness. +Without `select`, SQLBuild runs the SQL Rules that are enabled by default and no other +configurable Rules. `select = []` runs no configurable Rules. Mandatory compiler correctness +always applies. + +SQL Rules read the SQL that SQLBuild analyses, so they skip a model with `sql_analysis false` and +every model in a run with `--no-sql-analysis`. ## Codes and families -Built-in codes use `SQBR`, such as `SQBRSQL004`, `SQBRMODEL101`, and -`SQBRGRAPH101`. Custom codes use `XSQBR`, such as -`XSQBRARCH001`. +SQL Rule codes use `SQLCLEAN`, such as `SQLCLEAN004`. Other built-in codes use +`SQBR`, such as `SQBRMODEL101` and `SQBRGRAPH101`. Custom codes use +`XSQBR`, such as `XSQBRARCH001`. A family is always the code with its final three digits removed: ```text -SQBRSQL004 → SQBRSQL +SQLCLEAN004 → SQLCLEAN +SQBRGRAPH101 → SQBRGRAPH XSQBRARCH001 → XSQBRARCH XSQBR001 → XSQBR ``` -Exact codes select one Rule. Prefixes select every matching Rule. `ignore` uses the same matching -semantics and takes precedence over `select`. +Exact codes select one Rule. Families select every matching Rule, whether or not it is enabled by +default. `ignore` uses the same matching semantics and takes precedence over `select`. A selector that matches no Rule fails with `C000`. `SQBRTEST105` (scenario description) was retired when descriptions became the compile error `P010` @@ -41,7 +47,7 @@ Use the catalogue to inspect available codes and metadata: ```bash sqb rules list -sqb rules show SQBRSQL004 +sqb rules show SQLCLEAN004 ``` ## Rule options @@ -73,8 +79,8 @@ Unknown Rule codes, option names, and invalid option values fail configuration. Run one exact Rule or family while developing or adopting it: ```bash -sqb rules run SQBRSQL004 -sqb rules run SQBRSQL +sqb rules run SQLCLEAN004 +sqb rules run SQLCLEAN sqb rules run XSQBRARCH --select customer_orders ``` @@ -87,7 +93,7 @@ the complete configured enforcement performed by `sqb compile`. ## Join keys and final CTE names -`SQBRSQL040` checks every `JOIN ... ON` in the SQL covered by SQL Rules, including joins inside CTEs +`SQLCLEAN040` checks every `JOIN ... ON` in the SQL covered by SQL Rules, including joins inside CTEs and subqueries. `USING (...)` is accepted. Allowed predicates are: - Plain column comparisons, including equality, inequality, range and null-safe comparisons. @@ -106,16 +112,16 @@ projects `LOWER(category) AS category_key`, then join `ON o.category_key = c.cat variant path access such as `ON a.data:key = b.id` is also computed: project the value as a named column before joining. -`SQBRSQL041` checks CTE naming: the last top-level CTE must be named `final` (compared +`SQLCLEAN041` checks CTE naming: the last top-level CTE must be named `final` (compared case-insensitively, including quoted identifiers), and no other CTE, including a nested one, may use -that name. Queries without CTEs are unaffected, and SQL-test fixture CTEs are exempt. `SQBRSQL035` +that name. Queries without CTEs are unaffected, and SQL-test fixture CTEs are exempt. `SQLCLEAN035` owns the terminal `SELECT`'s plain projection from that last CTE, so select both, normally through -the `SQBRSQL` family, for the complete convention. Parentheses around the entire query do not create +the `SQLCLEAN` family, for the complete convention. Parentheses around the entire query do not create a nested CTE scope. ## Repeated JSON parsing -`SQBRSQL045` reports a JSON parse function applied to the same column more than once in one +`SQLCLEAN045` reports a JSON parse function applied to the same column more than once in one `SELECT`. Parse the payload once in the CTE that reads it, then read every field from that column: ```sql @@ -173,9 +179,9 @@ max_literal_length = 100 # default max_ranking_order_by = 5 # default ``` -- `SQBRSQL042`: every output column of a non-final CTE must be read later in the same model, +- `SQLCLEAN042`: every output column of a non-final CTE must be read later in the same model, in any clause of a later CTE or the final query, including through a later `SELECT *` - pass-through. The last CTE (the model output), unreachable CTEs (reported by `SQBRSQL005`), + pass-through. The last CTE (the model output), unreachable CTEs (reported by `SQLCLEAN005`), CTEs with a column list, recursive queries and SQL-test fixtures are exempt. A later step that uses an `@` interpolation in an expression may read any column, so it keeps every column, and interpolated outputs are never reported. A later step that reads the CTE as a whole row (the @@ -190,14 +196,14 @@ max_ranking_order_by = 5 # default `SELECT DISTINCT`, `DISTINCT ON`, `GROUP BY ALL`, `ORDER BY ALL`, an aggregate without `GROUP BY`, or positional `GROUP BY`/`ORDER BY`. Other findings are reported without a fix and name the reason. The fix repeats until nothing changes. -- `SQBRSQL043`: `ROW_NUMBER`, `RANK`, `DENSE_RANK`, `FIRST_VALUE` and `LAST_VALUE` windows may +- `SQLCLEAN043`: `ROW_NUMBER`, `RANK`, `DENSE_RANK`, `FIRST_VALUE` and `LAST_VALUE` windows may sort by at most `max_ranking_order_by` expressions. Order by the one or two columns that decide which row wins, then a unique row key as the final tie-breaker. If the rows have no unique column, build one in the upstream model from the columns that identify a row (for example `HASH(source_partition, source_offset, line_index) AS row_key`) and declare it with a `unique` - audit, so `SQBRSQL018` can prove the order. Raise the project limit only when the order + audit, so `SQLCLEAN018` can prove the order. Raise the project limit only when the order genuinely needs more keys. -- `SQBRSQL044`: a string literal longer than `max_literal_length` characters is an error. Define +- `SQLCLEAN044`: a string literal longer than `max_literal_length` characters is an error. Define the value once as a [constant](/docs/concepts/constants) in the narrowest [scope](/docs/concepts/declaration-scopes) that covers every use, and reference it with `@const`: a model-private `constants (_status_pattern '...')` entry in the `MODEL` header when @@ -207,7 +213,7 @@ max_ranking_order_by = 5 # default Otherwise shorten the literal. When the identical literal is reported in other files, the help lists them (up to five) and names the nearest common folder and its constants directory. Literals produced by `@const` or a macro are not reported, and there is no automatic fix. -- `SQBRSQL018` requires every `ROW_NUMBER`, `FIRST_VALUE` and `LAST_VALUE` window to be proven +- `SQLCLEAN018` requires every `ROW_NUMBER`, `FIRST_VALUE` and `LAST_VALUE` window to be proven deterministic: `PARTITION BY` and `ORDER BY` together must cover a unique key of the ranked rows. Keys come from `unique` column audits and model-level `unique` audits naming a column, when the column is also non-null (a `not_null` audit or `nullable false`), and from the `unique_key` of @@ -226,7 +232,6 @@ max_ranking_order_by = 5 # default A finding that `sqb format --fix` can fix says so in text output and carries `"fixable": true` in `sqb rules --json run` output; `sqb compile` adds the same note to the diagnostic. -Family selectors such as `select = ["SQBRSQL", "SQBRMODEL", "SQBRGRAPH"]` pick up new built-in Rules -on upgrade, so a release that adds `SQBRSQL040` to `SQBRSQL045` can expose computed join keys, -differently named final CTEs, unused CTE columns, long literals, ranking sorts or repeated JSON -parses that were accepted before. +Family selectors such as `select = ["SQLCLEAN", "SQBRMODEL", "SQBRGRAPH"]` pick up new built-in Rules +on upgrade, and an unset `select` picks up new default SQL Rules. A new Rule can report SQL that +compiled under an earlier release. diff --git a/website/src/content/docs/docs/concepts/rules/findings-and-exceptions.mdx b/website/src/content/docs/docs/concepts/rules/findings-and-exceptions.mdx index 3b063f1b7c..d28e8f89f6 100644 --- a/website/src/content/docs/docs/concepts/rules/findings-and-exceptions.mdx +++ b/website/src/content/docs/docs/concepts/rules/findings-and-exceptions.mdx @@ -16,7 +16,7 @@ Use an exact exception for one known finding. Exceptions require a reason and ar ```toml [[rules.rule_exceptions]] -rule = "SQBRSQL004" +rule = "SQLCLEAN004" path = "models/examples/sample_orders.sql" reason = "This example intentionally demonstrates one sampled row." ``` @@ -31,12 +31,12 @@ paragraph (see [Folded descriptions](/docs/concepts/models/configuration#folded- ## Relation-level exceptions A file-wide exception for a deliberate cartesian product would also hide any accidental cross join -added to the same file later. `SQBRSQL038` findings name the joined relation and, inside a CTE, +added to the same file later. `SQLCLEAN038` findings name the joined relation and, inside a CTE, that CTE, so an exception can cover exactly one join: ```toml [[rules.rule_exceptions]] -rule = "SQBRSQL038" +rule = "SQLCLEAN038" path = "models/orders/order_checks.sql" relation = "order_policy" cte = "policy_checks" # optional: only the join inside this top-level CTE @@ -46,14 +46,14 @@ reason = "Every order is evaluated against every policy check." `relation` is the joined relation as written, with `__ref`, `__seed` and `__source` calls resolved to the referenced name. Without `cte`, the exception covers that relation anywhere in the file. Relation-level exceptions are stale-checked like exact exceptions: an entry that matches no finding -fails compilation. `relation` is accepted only for `SQBRSQL038`, and a file-wide exception cannot be +fails compilation. `relation` is accepted only for `SQLCLEAN038`, and a file-wide exception cannot be combined with relation-level exceptions for the same code and path. A join that reads no named relation, such as `CROSS JOIN UNNEST(...)` or `CROSS JOIN LATERAL`, has no relation to name. Its finding says so and prints the file-wide exception (rule and path) that is the only way to cover it. -`SQBRSQL038` does not report a cross join when either side provably holds at most one row, so most +`SQLCLEAN038` does not report a cross join when either side provably holds at most one row, so most intentional cross joins need no exception at all. SQLBuild infers this from the query, never from a declaration: a seed whose CSV has one data row (adding a second row brings the finding back), an aggregate without `GROUP BY`, a one-row `VALUES`, `LIMIT 1`, or `QUALIFY ROW_NUMBER() OVER (...) = 1` @@ -66,7 +66,7 @@ Use a path ignore when a documented project area intentionally follows a differe ```toml [[rules.rule_ignores]] -rules = ["SQBRSQL"] +rules = ["SQLCLEAN"] paths = ["models/examples/**"] reason = "Examples retain intentionally minimal SQL." ``` @@ -80,7 +80,7 @@ Use `selectors` when the exception follows graph-resource identities or lineage ```toml [[rules.rule_ignores]] -rules = ["SQBRSQL021"] +rules = ["SQLCLEAN021"] selectors = ["intermediate_*"] reason = "These intermediate interfaces intentionally preserve upstream columns." ``` @@ -91,7 +91,7 @@ matching, including SQL tests and audits that are not graph resources: ```toml [[rules.rule_ignores]] -rules = ["SQBRSQL021"] +rules = ["SQLCLEAN021"] paths = ["models/**/intermediate_*.sql"] reason = "These intermediate SQL files intentionally preserve upstream columns." ``` @@ -103,15 +103,15 @@ To turn a Rule off for the whole project, add its code to `[rules] ignore`. ## No inline suppressions SQLBuild has no inline suppression comment. An `-- sqb: ignore ...` or `/* sqb: ignore ... */` -comment is a `SQBRSQL000` error that suppresses nothing, so the finding it was meant to hide is reported too. The error prints the +comment is a `SQBRSUPPRESSION000` error that suppresses nothing, so the finding it was meant to hide is reported too. The error prints the exact entry to add instead: ```text -error[SQBRSQL000]: Inline Rule suppressions are not supported; declare the exception in sqlbuild_project.toml +error[SQBRSUPPRESSION000]: Inline Rule suppressions are not supported; declare the exception in sqlbuild_project.toml --> models/orders/order_checks.sql:4:1 - = help: Remove this comment, then fix the SQBRSQL038 finding, or record it as a reviewed exception in sqlbuild_project.toml: + = help: Remove this comment, then fix the SQLCLEAN038 finding, or record it as a reviewed exception in sqlbuild_project.toml: [[rules.rule_exceptions]] - rule = "SQBRSQL038" + rule = "SQLCLEAN038" path = "models/orders/order_checks.sql" relation = "order_policy" reason = "every order meets every policy" @@ -139,7 +139,7 @@ Use JSON when another tool consumes findings: ```bash sqb compile --json -sqb rules --json run SQBRSQL +sqb rules --json run SQLCLEAN ``` Machine-readable output remains on stdout. Lifecycle progress and terminal status are written to From bfc804bc1a8e972ed62c26cb5284b5ef3efce666 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:24:52 +0000 Subject: [PATCH 2/8] feat(rules): finish the SQLClean engine integration Pin sqlclean to v0.4.1, lint with the compile target and vars, skip audit and SQL test files that opt out of SQL analysis, skip the row-rule fact build for SQL-only selections, align fixtures and DuckShop with the new findings, and update the docs. --- Cargo.lock | 4 +- Cargo.toml | 2 +- benchmarks/duckshop-10k/checksum.txt | 2 +- .../duckshop-10k/duckshop10k/__init__.py | 2 +- .../support/int_call_ivr_paths.sql | 2 - .../compile_attachment/effective_config.rs | 2 +- .../sql_expansion_context.rs | 48 ++++++--- .../_helpers/compile_driver/analysis.rs | 15 +-- .../_helpers/compile_driver/plan_pipeline.rs | 34 +++--- .../bindings/_helpers/records/construct.rs | 20 ++++ .../_helpers/rules/early_builtin_rules.rs | 4 +- .../bindings/_helpers/rules/early_sql_lint.rs | 100 +++++++++++++----- .../bindings/_helpers/rules/fix_session.rs | 1 + .../bindings/_helpers/rules/lint_expansion.rs | 17 ++- .../bindings/_helpers/rules/rule_dispatch.rs | 19 +++- .../src/bindings/_helpers/rules/rule_plan.rs | 13 +++ .../src/bindings/_helpers/rules/run_lint.rs | 1 + .../src/bindings/_helpers/rules/sql_rules.rs | 49 +++++++-- .../src/engine/tests/test_sql_scanning.rs | 2 +- .../src/lint_text/_helpers/mod.rs | 1 - .../src/lint_text/_helpers/preparation.rs | 95 ----------------- .../src/lint_text/_helpers/sites.rs | 80 +++++++++++++- .../src/lint_text/_helpers/sql_diagnostics.rs | 2 +- .../sqlbuild-rules/src/lint_text/constants.rs | 3 + .../lint_text/main/backtick_identifiers.rs | 4 - .../src/lint_text/main/defined_cte_names.rs | 2 +- .../sqlbuild-rules/src/lint_text/main/mod.rs | 1 - .../main/neutralize_interpolation.rs | 2 +- .../src/lint_text/main/preparation.rs | 7 +- crates/sqlbuild-rules/src/models.rs | 15 ++- .../src/rules/_helpers/catalogue.rs | 15 ++- .../src/rules/main/selected_sql_codes.rs | 10 +- fensu.toml | 8 +- .../classes/project_builder.py | 2 + scripts/compiler_differential/constants.py | 7 ++ .../_helpers/steps/mutations.py | 5 +- .../sqlbuild/references/docs/cli/compile.md | 2 +- .../sqlbuild/references/docs/cli/format.md | 29 ++--- .../sqlbuild/references/docs/cli/rules.md | 2 +- .../docs/concepts/macros/sql-macros.md | 2 +- .../docs/concepts/project-configuration.md | 11 +- .../references/docs/concepts/rules.md | 41 +++---- .../rules/configuration-and-selection.md | 60 ++++++----- .../concepts/rules/findings-and-exceptions.md | 26 ++--- src/sqlbuild/_native.pyi | 1 + .../cli/commands/main/audit/helpers.py | 6 +- .../cli/commands/main/build/helpers.py | 5 +- .../cli/commands/main/build/test_build.py | 2 +- .../build/test_capped_microbatch_build.py | 3 +- .../main/build/test_model_config_build.py | 4 +- .../main/build/test_unbuilt_dependencies.py | 4 +- .../cli/commands/main/clone/test_clone.py | 3 +- .../cli/commands/main/compile/_test_types.py | 2 + .../compile/fixtures/semantic/corpus.json | 10 +- .../cli/commands/main/compile/helpers.py | 33 +++++- .../main/compile/sql_macros/helpers.py | 16 ++- .../sql_macros/test_sql_macro_build.py | 6 +- .../sql_macros/test_sql_macro_cte_names.py | 29 +++-- .../test_sql_macro_relation_slots.py | 7 ++ .../test_sql_macro_typing_matrix.py | 6 ++ .../main/compile/test_dialect_syntax.py | 12 ++- .../test_duckshop_benchmark_compile.py | 2 +- .../compile/test_function_name_validation.py | 12 ++- .../compile/test_star_beside_cast_constant.py | 4 +- .../main/format/test_token_preservation.py | 6 +- .../cli/commands/main/sqlite/helpers.py | 13 ++- .../cli/commands/main/test/helpers.py | 15 ++- .../commands/main/test_rules_concurrent.py | 4 +- .../test_sql_rule_cache_publication.py | 5 +- tests/unit/src/sqlbuild/lint/helpers.py | 8 +- website/src/content/docs/docs/cli/format.mdx | 11 +- .../docs/concepts/project-configuration.mdx | 5 +- .../src/content/docs/docs/concepts/rules.mdx | 5 +- .../rules/configuration-and-selection.mdx | 5 +- 74 files changed, 641 insertions(+), 372 deletions(-) delete mode 100644 crates/sqlbuild-rules/src/lint_text/_helpers/preparation.rs delete mode 100644 crates/sqlbuild-rules/src/lint_text/main/backtick_identifiers.rs diff --git a/Cargo.lock b/Cargo.lock index f833ee8a04..005477f929 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1500,8 +1500,8 @@ dependencies = [ [[package]] name = "sqlclean" -version = "0.4.0" -source = "git+https://github.com/sqlstack-io/sqlclean.git?tag=v0.4.0#58511b8ad9750e3275d80c2f22bccee932f19d23" +version = "0.4.1" +source = "git+https://github.com/sqlstack-io/sqlclean.git?rev=65eb3f32683a6a6fc84ca3b8637d2cabda750425#65eb3f32683a6a6fc84ca3b8637d2cabda750425" dependencies = [ "polyglot-sql", "polyglot-sql-function-catalogs", diff --git a/Cargo.toml b/Cargo.toml index f51c001f33..27c2dfe453 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -140,7 +140,7 @@ sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } sqlbuild-state = { path = "crates/sqlbuild-state" } # SQL lint Rules, quality Rules, the formatter and the canonical token stream. -sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", tag = "v0.4.0" } +sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", rev = "65eb3f32683a6a6fc84ca3b8637d2cabda750425" } stacker = "0.1" syn = { version = "2", features = ["full", "visit", "extra-traits"] } tempfile = "3" diff --git a/benchmarks/duckshop-10k/checksum.txt b/benchmarks/duckshop-10k/checksum.txt index fb97b67c11..a49cc2c0f7 100644 --- a/benchmarks/duckshop-10k/checksum.txt +++ b/benchmarks/duckshop-10k/checksum.txt @@ -1 +1 @@ -generator 2 files 10874 sha256 7dd651a573bce90f9816889cdeece75b9f837137813a033125daafa28b7b70eb +generator 3 files 10874 sha256 5f3d30b82ebf52f2eb869494f52f77d1699f9a08743febd6ee9ca0cb22ed0611 diff --git a/benchmarks/duckshop-10k/duckshop10k/__init__.py b/benchmarks/duckshop-10k/duckshop10k/__init__.py index 0a8894d664..504bea12ab 100644 --- a/benchmarks/duckshop-10k/duckshop10k/__init__.py +++ b/benchmarks/duckshop-10k/duckshop10k/__init__.py @@ -4,6 +4,6 @@ regenerate checksum.txt, whenever a change alters any generated file. """ -GENERATOR_VERSION = "2" +GENERATOR_VERSION = "3" SEED = 20261024 PROJECT_NAME = "duckshop_10k" diff --git a/benchmarks/duckshop/project/models/intermediate/support/int_call_ivr_paths.sql b/benchmarks/duckshop/project/models/intermediate/support/int_call_ivr_paths.sql index e5e48e3663..82987a87bb 100644 --- a/benchmarks/duckshop/project/models/intermediate/support/int_call_ivr_paths.sql +++ b/benchmarks/duckshop/project/models/intermediate/support/int_call_ivr_paths.sql @@ -77,8 +77,6 @@ final AS ( THEN 'APAC' WHEN '965' THEN 'MENA' - WHEN NULL - THEN NULL ELSE 'EU' END::VARCHAR AS caller_region, CASE diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs index 80ad38ac57..382aa2f4bc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs @@ -108,7 +108,7 @@ pub(crate) fn build_effective_vars<'py>( } /// The selected target name of a discovered project and its effective config, if any. -fn target_selection<'py>( +pub(crate) fn target_selection<'py>( discovered: &Bound<'py, PyAny>, selected: Option>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs index 3e0e8df43d..056b4d827d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs @@ -10,7 +10,9 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compile_attachment::declaration_scope::build_declaration_scope; use crate::bindings::_helpers::compile_attachment::declaration_scope::rebind_declaration_scope; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_vars; +use crate::bindings::_helpers::compile_attachment::effective_config::{ + build_effective_vars, target_selection, +}; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_collection_rendering; @@ -86,12 +88,20 @@ fn local_declarations<'py>(discovered: &Bound<'py, PyAny>) -> PyResult { pub(crate) project_dir: Bound<'py, PyAny>, pub(crate) value_renderer: Bound<'py, PyAny>, - pub(crate) cli_vars: Option>, + pub(crate) vars: ExpansionVars<'py>, pub(crate) discovered_inputs: Option>, pub(crate) declaration_scope: Option>, pub(crate) static_declaration_scope: Option>, } +/// Where expansion takes its target and vars from. +pub(crate) enum ExpansionVars<'py> { + /// The default target's effective vars, overridden by these CLI vars when given. + DefaultTarget(Option>), + /// The target and vars of the compiling command's macro context. + Compile(Bound<'py, PyAny>), +} + /// Assemble SQL expansion inputs; a static scope lends only its index to private macros. pub(crate) fn sql_expansion_inputs(sources: ExpansionSources<'_>) -> PyResult { let py: Python<'_> = sources.project_dir.py(); @@ -114,16 +124,23 @@ pub(crate) fn sql_expansion_inputs(sources: ExpansionSources<'_>) -> PyResult = native_field(&discovered, "project_config")?; let local_config: Bound<'_, PyAny> = native_field(&discovered, "local_config")?; - let cli_vars: Bound<'_, PyAny> = sources - .cli_vars - .unwrap_or_else(|| PyDict::new(py).into_any()); - let effective_vars: Bound<'_, PyAny> = build_effective_vars( - &project_config.clone(), - &local_config.clone(), - &py.None().into_bound(py), - &cli_vars, - ) - .and_then(|value| value.into_bound_py_any(py))?; + let (target_name, effective_vars): (Bound<'_, PyAny>, Bound<'_, PyAny>) = match sources.vars { + ExpansionVars::Compile(context) => ( + native_field(&context, "target_name")?, + native_field(&context, "vars")?, + ), + ExpansionVars::DefaultTarget(cli_vars) => { + let cli_vars: Bound<'_, PyAny> = cli_vars.unwrap_or_else(|| PyDict::new(py).into_any()); + let (target_name, target_config) = target_selection(&discovered, None)?; + let vars: Bound<'_, PyAny> = build_effective_vars( + &project_config.clone(), + &local_config.clone(), + &target_config, + &cli_vars, + )?; + (target_name, vars) + } + }; let scope: Bound<'_, PyAny> = match sources.declaration_scope { Some(scope) => scope, None => declaration_scope( @@ -154,10 +171,7 @@ pub(crate) fn sql_expansion_inputs(sources: ExpansionSources<'_>) -> PyResult( sql_expansion_inputs(ExpansionSources { project_dir: arguments.value("project_dir")?, value_renderer: arguments.value("value_renderer")?, - cli_vars: present(arguments.value("cli_vars")?), + vars: ExpansionVars::DefaultTarget(present(arguments.value("cli_vars")?)), discovered_inputs: present(arguments.value("discovered_inputs")?), declaration_scope: present(arguments.value("declaration_scope")?), static_declaration_scope: present(arguments.value("static_declaration_scope")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index cf8472a4b1..eaabced465 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -382,16 +382,11 @@ impl<'py> Driver<'_, 'py> { ), ], )?; - let on_inputs_ready: Bound<'py, PyAny> = imported(py, "functools", "partial")?.call( - (self.early_lint.getattr("start")?,), - Some(&keywords( - py, - &[ - ("dialect", lint_dialect.clone()), - ("value_renderer", adapter.clone()), - ], - )?), - )?; + let on_inputs_ready: Bound<'py, PyAny> = EarlySqlLint::inputs_ready_hook( + self.early_lint, + &lint_dialect.extract::()?, + adapter, + ); let analysis_selection: Bound<'py, PyAny> = replaced( &options.analysis_selection.clone(), &[("on_inputs_ready", on_inputs_ready)], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs index 537fe036dc..59da81f65f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs @@ -6,7 +6,7 @@ use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_rules::rule_results::main::default_rules_notice::default_rules_notice; -use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, imported, keywords}; +use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, keywords}; use crate::bindings::_helpers::compile_driver::phases::monotonic; use crate::bindings::_helpers::compile_driver::pipeline::{ compiled_phase, progress, timing_tracker, @@ -20,8 +20,8 @@ use crate::bindings::_helpers::compile_driver::selection_phases::{ }; use crate::bindings::_helpers::frontier::stage_capture::write_plan_capture; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::new_record_with; +use crate::bindings::_helpers::records::construct::{error_class, new_annotated_error}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; @@ -59,17 +59,8 @@ pub(crate) fn run_compile_pipeline<'py>( let on_progress: Bound<'py, PyAny> = native_field(&hooks, "on_progress")?; let expansion_reuse: Bound<'py, EarlySqlLint> = Bound::new(py, EarlySqlLint::new(false, false))?; - let on_inputs_ready: Bound<'py, PyAny> = imported(py, "functools", "partial")?.call( - (expansion_reuse.getattr("start")?,), - Some(&keywords( - py, - &[ - // Only expansion reuse is kept here; early lint is off, so the dialect is unused. - ("dialect", GENERIC_DIALECT.into_pyobject(py)?.into_any()), - ("value_renderer", adapter.clone()), - ], - )?), - )?; + let on_inputs_ready: Bound<'py, PyAny> = + EarlySqlLint::inputs_ready_hook(&expansion_reuse, GENERIC_DIALECT, &adapter); let compiled: Bound<'py, PyAny> = compiled_phase( (&discovered, &adapter), &options, @@ -170,13 +161,18 @@ fn run_rules_gate<'py>( .try_iter()? .map(|finding| native_field(&finding?, "code")?.extract()) .collect::>()?; - let error: Bound<'py, PyAny> = - rules_error.call1((format!("configured compiler rules failed:\n{rendered}"),))?; let select_omitted: bool = native_field(&config, "select")?.is_none(); - if let Some(notice) = default_rules_notice(select_omitted, codes.iter().map(String::as_str)) { - error.setattr(DEFAULT_RULES_NOTICE_ATTR, notice)?; - } - Err(PyErr::from_value(error)) + let notice: Vec<(&str, String)> = + default_rules_notice(select_omitted, codes.iter().map(String::as_str)) + .map(|notice| (DEFAULT_RULES_NOTICE_ATTR, notice)) + .into_iter() + .collect(); + Err(new_annotated_error( + py, + ErrorId::Rules, + format!("configured compiler rules failed:\n{rendered}"), + ¬ice, + )) } /// Open the planning connection, plan, and record the compile and planning times. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/construct.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/construct.rs index f03a9a7840..d21f1b0ada 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/construct.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/construct.rs @@ -54,6 +54,26 @@ pub(crate) fn new_error(py: Python<'_>, id: ErrorId, message: String, code: &str } } +/// A raisable native `id` error carrying `message`, with each `(name, value)` attribute set on it. +pub(crate) fn new_annotated_error( + py: Python<'_>, + id: ErrorId, + message: String, + attributes: &[(&str, String)], +) -> PyErr { + let raised = (|| { + let error: Bound<'_, PyAny> = error_class(py, id)?.call1((message,))?; + for (name, value) in attributes { + error.setattr(*name, value)?; + } + Ok::<_, PyErr>(error) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + /// A raisable native `id` error carrying `message`, the diagnostic `code` and optional `help`. pub(crate) fn new_helped_error( py: Python<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs index 0445d7e3a1..3da4f38787 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::compiled_views::records::{ use crate::bindings::_helpers::records::fields::field; use crate::bindings::_helpers::rules::catalogue::references_custom_rules; use crate::bindings::_helpers::rules::rule_dispatch::speculative_rules_request; -use crate::bindings::_helpers::rules::rule_plan::rule_run_plan; +use crate::bindings::_helpers::rules::rule_plan::{rule_run_plan, selects_row_rules}; use crate::bindings::_helpers::rules::rules_config::load_rules_config_with_cache; use crate::bindings::types::CompilerDetach; @@ -199,7 +199,7 @@ fn early_request( None, settings.no_sql_analysis, )?; - if field(&plan, "selected")?.len()? == 0 || field(&plan, "custom_payloads")?.is_truthy()? { + if !selects_row_rules(&plan)? || field(&plan, "custom_payloads")?.is_truthy()? { return Ok(None); } let facts: Bound<'_, NativeCompiledProject> = Bound::new( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index e7e331c2d4..25220ba870 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -211,7 +211,7 @@ fn same_codes(left: &[String], right: &[String]) -> bool { /// Start early SQL lint on a native thread when the project selects SQL rules without a cache. fn start_early_lint( (inputs, dialect): (&Bound<'_, PyAny>, &str), - value_renderer: &Bound<'_, PyAny>, + reuse: ExpansionReuse, no_cache: bool, stop: &Arc, ) -> PyResult> { @@ -227,9 +227,7 @@ fn start_early_lint( let work: EarlyLintWork = EarlyLintWork { project_dir: plan.project_dir.clone(), settings: plan.settings, - discovered: record_field(inputs, "discovered_inputs")?.unbind(), - static_scope: record_field(inputs, "declaration_scope")?.unbind(), - value_renderer: value_renderer.clone().unbind(), + reuse, compiled, stop: Arc::clone(stop), }; @@ -250,9 +248,7 @@ fn start_early_lint( struct EarlyLintWork { project_dir: PathBuf, settings: LintSettings, - discovered: Py, - static_scope: Py, - value_renderer: Py, + reuse: ExpansionReuse, compiled: HashMap, stop: Arc, } @@ -262,10 +258,11 @@ fn early_lint(py: Python<'_>, work: &EarlyLintWork) -> PyResult let project_dir: &Path = &work.project_dir; let context: SqlExpansionInputs = lint_expansion_inputs(LintExpansionSources { project_dir: project_dir.into_pyobject(py)?.into_any(), - value_renderer: Some(work.value_renderer.bind(py).clone()), - discovered_inputs: Some(work.discovered.bind(py).clone()), + value_renderer: Some(work.reuse.value_renderer(py)), + compile: Some(&work.reuse), + discovered_inputs: Some(work.reuse.discovered_inputs(py)?), declaration_scope: None, - static_declaration_scope: present(work.static_scope.bind(py).clone()), + static_declaration_scope: present(work.reuse.declaration_scope(py)?), })?; let files: Vec = lint_sources(project_dir, None, None)?; let outcome: LintOutcome = lint( @@ -284,11 +281,39 @@ fn early_lint(py: Python<'_>, work: &EarlyLintWork) -> PyResult Ok(EarlyLintResult { outcome, context }) } -/// The compiler's discovery, declaration scope and adapter, which lint expansion reuses for the same discovery. +/// The compiler's inputs and adapter, which lint expansion reuses for the same discovery. pub(crate) struct ExpansionReuse { - pub(crate) discovered_inputs: Py, - pub(crate) declaration_scope: Py, - pub(crate) value_renderer: Py, + inputs: Py, + value_renderer: Py, +} + +impl ExpansionReuse { + fn clone_ref(&self, py: Python<'_>) -> Self { + Self { + inputs: self.inputs.clone_ref(py), + value_renderer: self.value_renderer.clone_ref(py), + } + } + + /// The discovery the compile ran on. + pub(crate) fn discovered_inputs<'py>(&self, py: Python<'py>) -> PyResult> { + record_field(self.inputs.bind(py), "discovered_inputs") + } + + /// The compile's declaration scope. + pub(crate) fn declaration_scope<'py>(&self, py: Python<'py>) -> PyResult> { + record_field(self.inputs.bind(py), "declaration_scope") + } + + /// The compile's macro context, carrying its target and vars. + pub(crate) fn macro_context<'py>(&self, py: Python<'py>) -> PyResult> { + record_field(self.inputs.bind(py), "macro_context") + } + + /// The adapter that renders declared values. + pub(crate) fn value_renderer<'py>(&self, py: Python<'py>) -> Bound<'py, PyAny> { + self.value_renderer.bind(py).clone() + } } /// Own speculative SQL checks for one compilation invocation. @@ -300,6 +325,8 @@ pub(crate) struct EarlySqlLint { no_cache: bool, preparation: Option>, expansion_reuse: Option, + /// The dialect and adapter the `on_inputs_ready` call passes to `start`. + ready_arguments: Option<(String, Py)>, stop_requested: Arc, } @@ -311,6 +338,7 @@ impl EarlySqlLint { no_cache, preparation: None, expansion_reuse: None, + ready_arguments: None, stop_requested: Arc::new(AtomicBool::new(false)), } } @@ -325,6 +353,17 @@ impl EarlySqlLint { self.expansion_reuse.as_ref() } + /// This manager as the `on_inputs_ready` hook, which calls `start` with `dialect` and `value_renderer`. + pub(crate) fn inputs_ready_hook<'py>( + slf: &Bound<'py, Self>, + dialect: &str, + value_renderer: &Bound<'py, PyAny>, + ) -> Bound<'py, PyAny> { + slf.borrow_mut().ready_arguments = + Some((dialect.to_owned(), value_renderer.clone().unbind())); + slf.clone().into_any() + } + /// Abandon speculative work, return the milliseconds spent joining it, and raise Ctrl+C caught while waiting. pub(crate) fn stop(&self, py: Python<'_>) -> PyResult { let started: Instant = Instant::now(); @@ -369,26 +408,37 @@ impl EarlySqlLint { value_renderer: &Bound<'_, PyAny>, ) -> PyResult<()> { let py: Python<'_> = inputs.py(); + let reuse: ExpansionReuse = ExpansionReuse { + inputs: inputs.clone().unbind(), + value_renderer: value_renderer.clone().unbind(), + }; if slf.enabled { let stop: Arc = Arc::clone(&slf.stop_requested); slf.preparation = - start_early_lint((inputs, dialect), value_renderer, slf.no_cache, &stop)? + start_early_lint((inputs, dialect), reuse.clone_ref(py), slf.no_cache, &stop)? .map(|preparation| Py::new(py, preparation)) .transpose()?; } - let declaration_scope: Bound<'_, PyAny> = record_field(inputs, "declaration_scope")?; - slf.expansion_reuse = if declaration_scope.is_none() { - None - } else { - Some(ExpansionReuse { - discovered_inputs: record_field(inputs, "discovered_inputs")?.unbind(), - declaration_scope: declaration_scope.unbind(), - value_renderer: value_renderer.clone().unbind(), - }) - }; + slf.expansion_reuse = (!reuse.declaration_scope(py)?.is_none()).then_some(reuse); Ok(()) } + /// Start with the arguments `inputs_ready_hook` stored, as the compile's `on_inputs_ready` hook. + #[pyo3(signature = (*, inputs))] + fn __call__(slf: PyRefMut<'_, Self>, inputs: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = inputs.py(); + let Some((dialect, value_renderer)) = slf + .ready_arguments + .as_ref() + .map(|(dialect, renderer)| (dialect.clone(), renderer.clone_ref(py))) + else { + return Err(PyRuntimeError::new_err( + "EarlySqlLint was called before inputs_ready_hook armed it", + )); + }; + Self::start(slf, inputs, &dialect, value_renderer.bind(py)) + } + /// Abandon unconsumed speculative work and return the milliseconds spent joining it. #[pyo3(name = "stop")] fn py_stop(&self, py: Python<'_>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index 378561660d..c0dfb8c809 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -172,6 +172,7 @@ impl<'py> FixSession<'py> { let sources: Vec = lint_sources(&project_dir, Some(&selected), given)?; let context: Option = if settings.native_enabled { Some(lint_expansion_inputs(LintExpansionSources { + compile: None, project_dir: self.project_dir.clone(), value_renderer: self.adapter.is_truthy()?.then(|| self.adapter.clone()), discovered_inputs: Some(updated_inputs(&self.inputs, files)?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs index d84b4a5c3b..1046192ffd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs @@ -10,7 +10,7 @@ use sqlbuild_host::entries::models::AdapterHost; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::compile_attachment::sql_expansion_context::{ - ExpansionSources, present, sql_expansion_inputs, + ExpansionSources, ExpansionVars, present, sql_expansion_inputs, }; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; @@ -19,6 +19,7 @@ use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::render::authored_sql::SqlExpansionInputs; +use crate::bindings::_helpers::rules::early_sql_lint::ExpansionReuse; const COMPILE_FIRST_MESSAGE: &str = "compiler-integrated Rules check the SQL your project \ actually produces, so the project must compile first"; @@ -46,6 +47,7 @@ pub(crate) fn build_lint_expansion_context<'py>( )?; let renderer: Bound<'py, PyAny> = arguments.value("value_renderer")?; lint_expansion_inputs(LintExpansionSources { + compile: None, project_dir: arguments.value("project_dir")?, value_renderer: renderer.is_truthy()?.then_some(renderer), discovered_inputs: present(arguments.value("discovered_inputs")?), @@ -56,9 +58,11 @@ pub(crate) fn build_lint_expansion_context<'py>( } /// Where lint's expansion inputs come from; a missing renderer is the project's adapter. -pub(crate) struct LintExpansionSources<'py> { +pub(crate) struct LintExpansionSources<'a, 'py> { pub(crate) project_dir: Bound<'py, PyAny>, pub(crate) value_renderer: Option>, + /// The compile whose macro context lint expands with, for its target and vars. + pub(crate) compile: Option<&'a ExpansionReuse>, pub(crate) discovered_inputs: Option>, pub(crate) declaration_scope: Option>, pub(crate) static_declaration_scope: Option>, @@ -66,13 +70,13 @@ pub(crate) struct LintExpansionSources<'py> { /// Lint's expansion inputs, reporting compile failures as `ProjectCompileError`. pub(crate) fn lint_expansion_inputs( - sources: LintExpansionSources<'_>, + sources: LintExpansionSources<'_, '_>, ) -> PyResult { let py: Python<'_> = sources.project_dir.py(); expansion_inputs(sources).or_else(|error| Err(compile_first_error(py, error)?)) } -fn expansion_inputs(sources: LintExpansionSources<'_>) -> PyResult { +fn expansion_inputs(sources: LintExpansionSources<'_, '_>) -> PyResult { let py: Python<'_> = sources.project_dir.py(); let project_dir: Bound<'_, PyAny> = sources.project_dir; let discovered: Bound<'_, PyAny> = match sources.discovered_inputs { @@ -111,7 +115,10 @@ fn expansion_inputs(sources: LintExpansionSources<'_>) -> PyResult ExpansionVars::Compile(compile.macro_context(py)?), + None => ExpansionVars::DefaultTarget(None), + }, discovered_inputs: Some(discovered), declaration_scope: sources.declaration_scope, static_declaration_scope: sources.static_declaration_scope, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs index ef2dec1e10..75005f7853 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs @@ -45,7 +45,9 @@ use crate::bindings::_helpers::rules::rows::{ NativeRulesRequest, build_rules_request, evaluate_rules_request, evaluate_rules_request_with, rules_request_over, selectionless_rules_request, }; -use crate::bindings::_helpers::rules::rule_plan::{plan_rule_run, whole_project_codes}; +use crate::bindings::_helpers::rules::rule_plan::{ + plan_rule_run, selects_row_rules, whole_project_codes, +}; use crate::bindings::_helpers::rules::rule_run::finish_rule_run; use crate::bindings::_helpers::rules::selection::{rule_selected, selected_rule_codes}; use crate::bindings::_helpers::rules::sql_rules::run_planned_sql_rules; @@ -294,7 +296,7 @@ pub(crate) fn evaluate_rules<'py>( ]; let native_dialect: String = run.dialect.clone(); let native_cancel: HostCancel = run.cancel.clone(); - let selectionless: bool = custom.is_none() && field(&run.config, "select")?.len()? == 0; + let selectionless: bool = custom.is_none() && !selects_row_rules(&plan)?; let early: Option = take_early_rules(); let early_report: Option> = early.as_ref().map(EarlyBuiltinRules::report); let native_thread: Option = if selectionless { @@ -454,7 +456,7 @@ fn evaluate_native<'py>( rules_result(run, defer_suppressions, evaluated, custom) } -/// The built-in rules result of a run that selects no rules, without building its models. +/// The built-in rules result of a run that selects no rows-engine rules, without building its models. fn selectionless_result<'py>( run: &NativeRun<'py>, payloads: &Bound<'py, PyAny>, @@ -465,11 +467,20 @@ fn selectionless_result<'py>( selectionless_rules_request(&run.config, &run.project_dir, &run.dialect, payloads)?, )?; let evaluated: PyResult> = evaluate_rules_request(py, &request); - rules_result( + let result: Bound<'py, PyAny> = rules_result( run, true, evaluated, outcome(py, (Vec::new(), 0, 0), (0, 0), &[]), + )?; + let evaluated_models: usize = if field(&run.config, "select")?.len()? == 0 { + 0 + } else { + project_models(&run.project)?.len() + }; + replaced( + &result, + &[("evaluated_models", evaluated_models.into_bound_py_any(py)?)], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs index db398f032d..274b9c8261 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs @@ -154,6 +154,19 @@ pub(crate) fn rule_run_plan<'py>( ) } +/// Whether the plan selects any built-in or custom rule beyond the SQL Rules, which need the rows engine's facts. +pub(crate) fn selects_row_rules(plan: &Bound<'_, PyAny>) -> PyResult { + for rule in field(plan, "selected")?.try_iter()? { + if !field(&rule?, "code")? + .extract::()? + .starts_with(SQL_RULE_PREFIX) + { + return Ok(true); + } + } + Ok(false) +} + /// The selected non-SQL-lint codes a model selection evaluates over the whole project. pub(crate) fn whole_project_codes(plan: &Bound<'_, PyAny>) -> PyResult> { let mut codes: Vec = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs index 0eddfda678..f1931dede7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs @@ -125,6 +125,7 @@ fn run_lint<'py>( } else if settings.native_enabled { let renderer: Bound<'py, PyAny> = arguments.value("value_renderer")?; Some(lint_expansion_inputs(LintExpansionSources { + compile: None, project_dir: project_dir.clone(), value_renderer: renderer.is_truthy()?.then_some(renderer), discovered_inputs: present(arguments.value("discovered_inputs")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index 3e0a63232b..5b65648e48 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -28,6 +28,7 @@ use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::rules::constants::SQL_RULE_PREFIX; use crate::bindings::_helpers::rules::constants::UNSAFE_RULE_FIX_CODES; @@ -278,6 +279,35 @@ fn model_posix_path(model: &Bound<'_, PyAny>) -> PyResult { .extract() } +/// Audit and SQL test files whose every block sets `sql_analysis false` in its header. +fn opted_out_resource_paths( + project: &Bound<'_, PyAny>, + project_dir: &Path, +) -> PyResult> { + let excluded: Bound<'_, PyAny> = false.into_bound_py_any(project.py())?; + let mut opted_out: HashMap = HashMap::new(); + for (collection, file_field, block_field) in [ + ("audits", "audit_file", "audit_block"), + ("sql_tests", "test_file", "test_block"), + ] { + for resource in project_attribute(project, collection)?.try_iter()? { + let resource: Bound<'_, PyAny> = resource?; + let relative: String = + native_field(&native_field(&resource, file_field)?, "relative_path")? + .str()? + .extract()?; + let values: Bound<'_, PyAny> = + native_field(&native_field(&resource, block_field)?, "header_values")?; + let opts_out: bool = mapping_get(&values, "sql_analysis")?.is(&excluded); + *opted_out.entry(project_dir.join(relative)).or_insert(true) &= opts_out; + } + } + Ok(opted_out + .into_iter() + .filter_map(|(path, opts_out)| opts_out.then_some(path)) + .collect()) +} + /// Whether the model's config values set `sql_analysis` to `False`. fn python_excluded(model: &Bound<'_, PyAny>) -> PyResult { let excluded: Bound<'_, PyAny> = false.into_bound_py_any(model.py())?; @@ -408,14 +438,15 @@ fn evaluated_sql_rules( let mut project_files: Option> = None; let compiled: HashMap = project_sql_expansions(&run.project)?; if !run.model_selection { - let mut model_paths: HashSet = HashSet::new(); + let mut skipped_paths: HashSet = + opted_out_resource_paths(&run.project, project_dir)?; for model in project_attribute(&run.project, "models")?.try_iter()? { - model_paths.insert(project_dir.join(model_posix_path(&model?)?)); + skipped_paths.insert(project_dir.join(model_posix_path(&model?)?)); } let files: Vec = linted_files(run, prepared_files)?; let mut file_keys: Vec<(PathBuf, String, Option)> = Vec::new(); for file in &files { - if model_paths.contains(&file.path) { + if skipped_paths.contains(&file.path) { continue; } let Ok(relative) = file.path.strip_prefix(project_dir) else { @@ -616,17 +647,21 @@ fn lint_expansion_context( run: &SqlRun<'_, '_>, ) -> PyResult { let py: Python<'_> = run.project.py(); - let reuse: Option<&ExpansionReuse> = run + let reuse: Option<&ExpansionReuse> = match run .early_lint .as_ref() .and_then(|early| early.expansion_reuse()) - .filter(|reuse| reuse.discovered_inputs.bind(py).is(&run.discovered_inputs)); + { + Some(reuse) if reuse.discovered_inputs(py)?.is(&run.discovered_inputs) => Some(reuse), + _ => None, + }; lint_expansion_inputs(LintExpansionSources { project_dir: run.project_dir.clone().into_pyobject(py)?.into_any(), - value_renderer: reuse.map(|reuse| reuse.value_renderer.bind(py).clone()), + value_renderer: reuse.map(|reuse| reuse.value_renderer(py)), + compile: reuse, discovered_inputs: (!run.discovered_inputs.is_none()) .then(|| run.discovered_inputs.clone()), - declaration_scope: reuse.map(|reuse| reuse.declaration_scope.bind(py).clone()), + declaration_scope: reuse.map(|reuse| reuse.declaration_scope(py)).transpose()?, static_declaration_scope: None, }) } diff --git a/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs b/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs index 8c32966ca2..0140ad49cb 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs @@ -1,7 +1,7 @@ use crate::engine::tests::test_types::{ DialectCallsTestCase, DollarQuoteSiteTestCase, LintDialectSiteTestCase, SqlScannerTestCase, }; -use crate::lint_text::_helpers::preparation::prepare; +use crate::lint_text::main::preparation::prepare; use crate::query_facts::main::model_shape::model_shape; use crate::rules::_helpers::numeric_decisions::compact_sql; use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren; diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/mod.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/mod.rs index f96b7b7983..a666bb0e05 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/mod.rs @@ -3,7 +3,6 @@ pub(crate) mod fixture_formatting; pub(crate) mod fixtures; pub(crate) mod format_bodies; pub(crate) mod lines; -pub(crate) mod preparation; pub(crate) mod settings; pub(crate) mod sites; pub(crate) mod sql_diagnostics; diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/preparation.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/preparation.rs deleted file mode 100644 index dc41a4cdcd..0000000000 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/preparation.rs +++ /dev/null @@ -1,95 +0,0 @@ -//! One native lexical pass for interpolation provenance and CTE dependency evidence. - -use std::collections::HashSet; -use std::sync::LazyLock; - -use regex::Regex; - -use crate::lint_text::main::neutralize_interpolation::neutralize_interpolation; -use crate::lint_text::models::Neutralized; -use crate::lint_text::types::PreparedSql; -use sqlbuild_sqltext::sql_scan::main::quote_policy::quote_policy; - -static CTES: LazyLock> = LazyLock::new(|| { - Regex::new( - r#"(?is)(?:\bWITH(?:\s+RECURSIVE)?|,)\s*["`\[]?(?P[A-Za-z_][A-Za-z0-9_]*)["`\]]?(?:\s*\([^)]*\))?\s+AS\s*\("#, - ) -}); -static WORDS: LazyLock> = - LazyLock::new(|| Regex::new(r"\b[A-Za-z_][A-Za-z0-9_]*\b")); -static OPAQUE_CTE: LazyLock> = - LazyLock::new(|| Regex::new(r"(?i)\b[A-Za-z_][A-Za-z0-9_]*\s+AS\s*\(\s*(?:--[^\n]*\n\s*)?$")); - -/// Return whether SQL lint treats backticks as identifier quotes for `dialect`, as rules do. -pub(crate) fn backtick_identifiers(dialect: &str) -> bool { - quote_policy(dialect).backtick_identifiers -} - -pub(crate) fn prepare( - expanded: &str, - before_expansion: &str, - prior_sites: &[usize], - dialect: &str, -) -> Result, String> { - let cte_pattern = CTES.as_ref().map_err(std::string::ToString::to_string)?; - let word_pattern = WORDS.as_ref().map_err(std::string::ToString::to_string)?; - let opaque_pattern = OPAQUE_CTE - .as_ref() - .map_err(std::string::ToString::to_string)?; - let neutralized: Neutralized = - neutralize_interpolation(expanded, dialect, false).map_err(|error| error.to_string())?; - let names: HashSet = cte_pattern - .captures_iter(expanded) - .map(|capture| capture["name"].to_ascii_lowercase()) - .collect(); - let mut referenced: HashSet = HashSet::new(); - for site in &neutralized.sites { - referenced.extend( - word_pattern - .find_iter(&site.original_text) - .map(|word| word.as_str().to_ascii_lowercase()) - .filter(|word| names.contains(word)), - ); - } - for &start in prior_sites { - let Some(prefix) = byte_offset(before_expansion, start) - .and_then(|byte_start| before_expansion.get(..byte_start)) - else { - return Ok(None); - }; - if opaque_pattern.is_match(prefix) { - referenced.extend( - cte_pattern - .captures_iter(prefix) - .map(|capture| capture["name"].to_ascii_lowercase()) - .filter(|name| names.contains(name)), - ); - } - } - let mut referenced: Vec = referenced.into_iter().collect(); - referenced.sort(); - Ok(Some((neutralized.text, neutralized.sites, referenced))) -} - -/// Every CTE name `sql` defines, lowercased, in order and without repeats. -pub(crate) fn defined_cte_names(sql: &str) -> Vec { - let Ok(cte_pattern) = CTES.as_ref() else { - return Vec::new(); - }; - let mut names: Vec = Vec::new(); - for capture in cte_pattern.captures_iter(sql) { - let name: String = capture["name"].to_ascii_lowercase(); - if !names.contains(&name) { - names.push(name); - } - } - names -} - -/// The byte offset of character position `position`, or `None` past the end. -fn byte_offset(text: &str, position: usize) -> Option { - text.char_indices() - .map(|(index, _)| index) - .chain(std::iter::once(text.len())) - .nth(position) -} diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sites.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sites.rs index f4f342eaba..e569987b83 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sites.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sites.rs @@ -1,5 +1,6 @@ //! Interpolation site scanning and sentinel assembly over byte offsets. +use std::collections::HashSet; use std::sync::LazyLock; use regex::Regex; @@ -7,19 +8,29 @@ use sqlbuild_core::text::main::python_isalnum::python_isalnum; use sqlbuild_core::text::main::python_isspace::python_isspace; use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren; use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; +use sqlbuild_sqltext::sql_scan::main::quote_policy::quote_policy; use sqlbuild_sqltext::sql_scan::models::QuotePolicy; use crate::lint_text::constants::{ - FORMAT_SITE_PATTERN, FUNCTION_NAMES, LINT_SITE_PATTERN, SENTINEL_SUFFIX, SITE_START_BYTES, + CTE_DEFINITION_PATTERN, FORMAT_SITE_PATTERN, FUNCTION_NAMES, LINT_SITE_PATTERN, + SENTINEL_SUFFIX, SITE_START_BYTES, }; use crate::lint_text::errors::LintTextError; +use crate::lint_text::main::neutralize_interpolation::neutralize_interpolation; use crate::lint_text::models::{InterpolationSite, Neutralized}; +use crate::lint_text::types::PreparedSql; pub(crate) static DEPENDENCY_SITE_PATTERN: LazyLock> = LazyLock::new(|| Regex::new(r"(?i)^__(?:ref|source)\s*\(")); pub(crate) static KEYED_RELATION_SITE_PATTERN: LazyLock> = LazyLock::new(|| Regex::new(r#"(?i)^__(ref|source|seed)\s*\(\s*['"]([^'"]+)['"]\s*\)$"#)); +static CTES: LazyLock> = + LazyLock::new(|| Regex::new(CTE_DEFINITION_PATTERN)); +static WORDS: LazyLock> = + LazyLock::new(|| Regex::new(r"\b[A-Za-z_][A-Za-z0-9_]*\b")); +static OPAQUE_CTE: LazyLock> = + LazyLock::new(|| Regex::new(r"(?i)\b[A-Za-z_][A-Za-z0-9_]*\s+AS\s*\(\s*(?:--[^\n]*\n\s*)?$")); static LINT_SITE: LazyLock> = LazyLock::new(|| Regex::new(LINT_SITE_PATTERN)); static FORMAT_SITE: LazyLock> = @@ -246,3 +257,70 @@ pub(crate) fn opens_call(body: &str, end: usize) -> bool { .find(|character| !python_isspace(*character)) == Some('(') } + +/// Return whether SQL lint treats backticks as identifier quotes for `dialect`, as rules do. +pub(crate) fn backtick_identifiers(dialect: &str) -> bool { + quote_policy(dialect).backtick_identifiers +} + +/// The neutralized SQL, its sites and the CTEs interpolation references, or `None` when a prior site is out of range. +pub(crate) fn prepare( + expanded: &str, + before_expansion: &str, + prior_sites: &[usize], + dialect: &str, +) -> Result, String> { + let cte_pattern = CTES.as_ref().map_err(std::string::ToString::to_string)?; + let word_pattern = WORDS.as_ref().map_err(std::string::ToString::to_string)?; + let opaque_pattern = OPAQUE_CTE + .as_ref() + .map_err(std::string::ToString::to_string)?; + let neutralized: Neutralized = + neutralize_interpolation(expanded, dialect, false).map_err(|error| error.to_string())?; + let names: HashSet = cte_pattern + .captures_iter(expanded) + .map(|capture| capture["name"].to_ascii_lowercase()) + .collect(); + let mut referenced: HashSet = HashSet::new(); + for site in &neutralized.sites { + referenced.extend( + word_pattern + .find_iter(&site.original_text) + .map(|word| word.as_str().to_ascii_lowercase()) + .filter(|word| names.contains(word)), + ); + } + for &start in prior_sites { + let Some(prefix) = byte_offset(before_expansion, start) + .and_then(|byte_start| before_expansion.get(..byte_start)) + else { + return Ok(None); + }; + if opaque_pattern.is_match(prefix) { + referenced.extend( + cte_pattern + .captures_iter(prefix) + .map(|capture| capture["name"].to_ascii_lowercase()) + .filter(|name| names.contains(name)), + ); + } + } + let mut referenced: Vec = referenced.into_iter().collect(); + referenced.sort(); + Ok(Some((neutralized.text, neutralized.sites, referenced))) +} + +/// Every CTE name `sql` defines, lowercased, in order and without repeats. +pub(crate) fn defined_cte_names(sql: &str) -> Vec { + let Ok(cte_pattern) = CTES.as_ref() else { + return Vec::new(); + }; + let mut names: Vec = Vec::new(); + for capture in cte_pattern.captures_iter(sql) { + let name: String = capture["name"].to_ascii_lowercase(); + if !names.contains(&name) { + names.push(name); + } + } + names +} diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs index 577a1e5c81..d7ddf43c3c 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs @@ -6,6 +6,7 @@ use std::sync::LazyLock; use regex::Regex; use serde_json::{Map, Value}; +use crate::lint_text::_helpers::sites::backtick_identifiers; use crate::lint_text::_helpers::sites::{byte_offset, compiled, lint_policy, site_end}; use crate::lint_text::constants::{ GENERATED_FIX, GENERATED_SQL_SUFFIX, HARNESS_CTE_PREFIXES, INVALID_DIAGNOSTIC, @@ -15,7 +16,6 @@ use crate::lint_text::constants::{ PARSE_ERROR_POSITION_PATTERN, PARSE_FAILURE_PREFIX, PARSE_FAILURE_REMEDIATION, SQL_MACRO_PARAMETER_SENTINEL_PREFIX, UNSUPPORTED_VERSION, UNUSED_CTE_CODE, }; -use crate::lint_text::main::backtick_identifiers::backtick_identifiers; use crate::lint_text::models::{AuthoredDiagnostic, AuthoredEdit, ExpansionSpan, LintSqlBody}; use sqlbuild_analysis::sql_macros::constants::SQL_CALL_PREFIX; use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs index 39092eed11..ea4e21a0bf 100644 --- a/crates/sqlbuild-rules/src/lint_text/constants.rs +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -205,3 +205,6 @@ pub(crate) const EXPRESSION_SELECT_PREFIX: &str = "SELECT "; pub(crate) const LINTED_EXPRESSION_ALIAS: &str = "__sqlbuild_macro_expression__"; /// The characters that open a quoted string or identifier in formatted SQL. pub(crate) const STRING_QUOTES: [char; 3] = ['\'', '"', '`']; + +/// A CTE definition, capturing its `name`. +pub(crate) const CTE_DEFINITION_PATTERN: &str = r#"(?is)(?:\bWITH(?:\s+RECURSIVE)?|,)\s*["`\[]?(?P[A-Za-z_][A-Za-z0-9_]*)["`\]]?(?:\s*\([^)]*\))?\s+AS\s*\("#; diff --git a/crates/sqlbuild-rules/src/lint_text/main/backtick_identifiers.rs b/crates/sqlbuild-rules/src/lint_text/main/backtick_identifiers.rs deleted file mode 100644 index c3006c0bf2..0000000000 --- a/crates/sqlbuild-rules/src/lint_text/main/backtick_identifiers.rs +++ /dev/null @@ -1,4 +0,0 @@ -/// Return whether SQL lint treats backticks as identifier quotes for `dialect`. -pub fn backtick_identifiers(dialect: &str) -> bool { - crate::lint_text::_helpers::preparation::backtick_identifiers(dialect) -} diff --git a/crates/sqlbuild-rules/src/lint_text/main/defined_cte_names.rs b/crates/sqlbuild-rules/src/lint_text/main/defined_cte_names.rs index a3a26cc297..4b1a3aeb3f 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/defined_cte_names.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/defined_cte_names.rs @@ -3,5 +3,5 @@ /// Every CTE name `sql` defines, lowercased, in order and without repeats. #[must_use] pub fn defined_cte_names(sql: &str) -> Vec { - crate::lint_text::_helpers::preparation::defined_cte_names(sql) + crate::lint_text::_helpers::sites::defined_cte_names(sql) } diff --git a/crates/sqlbuild-rules/src/lint_text/main/mod.rs b/crates/sqlbuild-rules/src/lint_text/main/mod.rs index ae5b947776..3343ef6e90 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/mod.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/mod.rs @@ -1,4 +1,3 @@ -pub mod backtick_identifiers; pub mod defined_cte_names; pub mod dependency_sites; pub mod first_header_only; diff --git a/crates/sqlbuild-rules/src/lint_text/main/neutralize_interpolation.rs b/crates/sqlbuild-rules/src/lint_text/main/neutralize_interpolation.rs index 4bc32e3492..c055d8d93b 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/neutralize_interpolation.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/neutralize_interpolation.rs @@ -2,12 +2,12 @@ use regex::Regex; +use crate::lint_text::_helpers::sites::backtick_identifiers; use crate::lint_text::_helpers::sites::{ ByteSite, assemble, lint_policy, sentinel, site_end, site_pattern, site_starts, }; use crate::lint_text::constants::LINT_SENTINEL_PREFIX; use crate::lint_text::errors::LintTextError; -use crate::lint_text::main::backtick_identifiers::backtick_identifiers; use crate::lint_text::models::Neutralized; /// Neutralize references for lint, or every intrinsic spelling in any case for formatting. diff --git a/crates/sqlbuild-rules/src/lint_text/main/preparation.rs b/crates/sqlbuild-rules/src/lint_text/main/preparation.rs index 680974de92..a33be24f44 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/preparation.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/preparation.rs @@ -8,10 +8,5 @@ pub fn prepare( prior_sites: &[usize], dialect: &str, ) -> Result, String> { - crate::lint_text::_helpers::preparation::prepare( - expanded, - before_expansion, - prior_sites, - dialect, - ) + crate::lint_text::_helpers::sites::prepare(expanded, before_expansion, prior_sites, dialect) } diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index 8e5981896b..cf91da8c3a 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -44,17 +44,16 @@ impl fensu_policy::policy::types::RuleCodeGrammar for RulesCodeGrammar { impl RulesCodeGrammar { fn namespace_and_remainder(value: &str) -> Option<(&'static str, &str)> { - [ + for namespace in [ CUSTOM_RULE_NAMESPACE, SQL_RULE_PREFIX, BUILT_IN_RULE_NAMESPACE, - ] - .into_iter() - .find_map(|namespace| { - value - .strip_prefix(namespace) - .map(|remainder| (namespace, remainder)) - }) + ] { + if let Some(remainder) = value.strip_prefix(namespace) { + return Some((namespace, remainder)); + } + } + None } } diff --git a/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs b/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs index b0bd8f7873..23d3f563f1 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs @@ -496,14 +496,23 @@ pub(crate) fn sql_lint_selection( .filter(|rule| rule.code.starts_with(SQL_RULE_PREFIX)) .filter(|rule| match selected { None => rule.enabled_by_default, - Some(selectors) => selectors - .iter() - .any(|selector| grammar.code_matches_selector(&rule.code, selector)), + Some(selectors) => matches_any_selector(grammar, &rule.code, selectors), }) .map(|rule| rule.code.clone()) .collect() } +/// Whether any of `selectors` matches `code`. +pub(crate) fn matches_any_selector( + grammar: &RulesCodeGrammar, + code: &str, + selectors: &[String], +) -> bool { + selectors + .iter() + .any(|selector| grammar.code_matches_selector(code, selector)) +} + fn activate_explicit_custom_matches( catalogue: &[RuleMetadata], selected: &[String], diff --git a/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs b/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs index 9bf552da84..fe88b7198c 100644 --- a/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs +++ b/crates/sqlbuild-rules/src/rules/main/selected_sql_codes.rs @@ -1,9 +1,7 @@ //! `selected_sql_codes`: the built-in SQL Rule codes a `select`/`ignore` pair enables. -use fensu_policy::policy::types::RuleCodeGrammar; - use crate::models::RulesCodeGrammar; -use crate::rules::_helpers::catalogue::sql_lint_selection; +use crate::rules::_helpers::catalogue::{matches_any_selector, sql_lint_selection}; use crate::rules::main::catalogue::catalogue; /// SQL Rule codes `select` enables (SQLClean's defaults when it is `None`) that no `ignore` selector matches, in catalogue order. @@ -12,10 +10,6 @@ pub fn selected_sql_codes(select: Option<&[String]>, ignore: &[String]) -> Vec None: diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 085040bf50..4626809a05 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -143,6 +143,13 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "header_syntax": "D002", "over_broad_constant": "S024", } +GENERATOR_SQL_RULE_IGNORES: tuple[str, ...] = ( + "", + "[[rules.rule_ignores]]", + 'rules = ["SQLCLEAN"]', + 'paths = ["**"]', + 'reason = "Generated SQL picks shapes at random to cover compiler paths."', +) GENERATOR_HALF: float = 0.5 GENERATOR_CROSS_DOMAIN_SHARE: float = 0.4 GENERATOR_NON_ASCII_DESCRIPTION_SHARE: float = 0.3 diff --git a/scripts/planner_differential/_helpers/steps/mutations.py b/scripts/planner_differential/_helpers/steps/mutations.py index eeee8094c7..30b47522a2 100644 --- a/scripts/planner_differential/_helpers/steps/mutations.py +++ b/scripts/planner_differential/_helpers/steps/mutations.py @@ -99,7 +99,7 @@ def _edit_query(*, context: StepContext, target: ModelFacts) -> MutationResult: target=target, rewrite=lambda model_file: ModelFile( header=model_file.header, - query=f"SELECT *\nFROM (\n{model_file.query}\n) AS {_EDIT_ALIAS}", + query=f"WITH {_EDIT_ALIAS} AS (\n{model_file.query}\n)\nSELECT *\nFROM {_EDIT_ALIAS}", ), ) @@ -166,8 +166,9 @@ def rewrite(model_file: ModelFile) -> ModelFile: return ModelFile( header=header, query=( + f"WITH {_EDIT_ALIAS} AS (\n{model_file.query}\n)\n" f"SELECT {', '.join(columns)}, CAST(1 AS INTEGER) AS {ADDED_COLUMN_NAME}\n" - f"FROM (\n{model_file.query}\n) AS {_EDIT_ALIAS}" + f"FROM {_EDIT_ALIAS}" ), ) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index d3988debd7..24e26a8b1b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -113,7 +113,7 @@ A model with Rule errors starts with a header that marks it not buildable and li ```sql -- sqb: compile failed for this model; not buildable --- SQBRSQL028 at models/open_orders.sql:6:20: Single-relation query mixes qualified and unqualified columns +-- SQLCLEAN028 at models/open_orders.sql:6:20: Single-relation query mixes qualified and unqualified columns SELECT o.order_id, status FROM __ref("orders") AS o ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md index 442d2f734d..34ba87480d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md @@ -170,20 +170,25 @@ still fixed. When a difference cannot be attributed, each file's edits are check file whose SQL cannot be expanded is skipped with a `format-fix-skipped` warning that gives the reason, and the run continues. A cycle or the 128-pass limit fails without writing any edit. +`--fix` attempts the fixes SQLClean marks safe and never the unsafe ones; every edit is still +checked by the compiler as above. + | Rule | Decision | Reason | | --- | --- | --- | -| SQBRSQL001 NULL comparison | Refused | Replacing `= NULL` with `IS NULL` changes three-valued logic. | -| SQBRSQL002 implicit cartesian join | Applied | A comma-separated product becomes an explicit `CROSS JOIN`; mixed join precedence is excluded. | -| SQBRSQL003 conditionless join | Applied in Snowflake | Snowflake defines an unqualified `JOIN` without `ON` or `USING` as a cartesian product. Other dialects are refused because the original syntax may be invalid. | -| SQBRSQL005 unused CTE | Applied | Unreferenced `SELECT` CTEs contribute no rows. Mutation statements and commented CTEs are excluded. | -| SQBRSQL006 redundant DISTINCT | Applied | The projected and grouping expressions are proved to coincide. | -| SQBRSQL008 bare UNION | Applied | `UNION DISTINCT` states the existing duplicate elimination, in dialects that support it. | -| SQBRSQL013 parenthesized DISTINCT | Refused | Removing parentheses can change operator precedence. | -| SQBRSQL017 constant row count | Applied | `COUNT(1)` and `COUNT(*)` both count every row. | -| SQBRSQL023 unused table alias | Refused | Not yet reviewed as meaning-preserving. | -| SQBRSQL025 implicit inner join | Applied | A bare `JOIN ... ON` is an inner join. | -| SQBRSQL030 boolean CASE simplification | Refused | The `COALESCE` rewrite is not proved portable to every supported dialect. | -| SQBRSQL042 unused CTE output | Applied when row-neutral | Only a column reference or deterministic scalar expression is removed, from a CTE without `DISTINCT`, `DISTINCT ON`, `GROUP BY ALL`, `ORDER BY ALL`, an ungrouped aggregate, or positional `GROUP BY`/`ORDER BY`. Other removals could change the rows and are refused with the reason. | +| SQLCLEAN001 NULL comparison | Refused | Replacing `= NULL` with `IS NULL` changes three-valued logic. | +| SQLCLEAN002 implicit cartesian join | Applied | A comma-separated product becomes an explicit `CROSS JOIN`; mixed join precedence is excluded. | +| SQLCLEAN003 conditionless join | Applied in Snowflake | Snowflake defines an unqualified `JOIN` without `ON` or `USING` as a cartesian product. Other dialects are refused because the original syntax may be invalid. | +| SQLCLEAN005 unused CTE | Applied | Unreferenced `SELECT` CTEs contribute no rows. Mutation statements and commented CTEs are excluded. | +| SQLCLEAN006 redundant DISTINCT | Applied | The projected and grouping expressions are proved to coincide. | +| SQLCLEAN008 bare UNION | Applied | `UNION DISTINCT` states the existing duplicate elimination, in dialects that support it. | +| SQLCLEAN012 redundant ELSE NULL | Applied | A `CASE` without `ELSE` already yields `NULL`. | +| SQLCLEAN013 parenthesized DISTINCT | Refused when unproved | Removing parentheses can change operator precedence, so such an edit is refused with the reason. | +| SQLCLEAN015 empty statement | Applied | A repeated terminator adds no statement. | +| SQLCLEAN017 constant row count | Applied | `COUNT(1)` and `COUNT(*)` both count every row. | +| SQLCLEAN023 unused table alias | Applied | An alias no expression references is removed. | +| SQLCLEAN025 implicit inner join | Applied | A bare `JOIN ... ON` is an inner join. | +| SQLCLEAN030 boolean CASE simplification | Applied when proved | The `COALESCE` rewrite is kept only when recompiling shows the same output types and nullability; otherwise the file reports `format-fix-verification-failed`. | +| SQLCLEAN042 unused CTE output | Applied when row-neutral | Only a column reference or deterministic scalar expression is removed, from a CTE without `DISTINCT`, `DISTINCT ON`, `GROUP BY ALL`, `ORDER BY ALL`, an ungrouped aggregate, or positional `GROUP BY`/`ORDER BY`. Other removals could change the rows and are refused with the reason. | `--fix` also rewrites a header description that escapes double quotes as `\"` or doubles them as `""`, which descriptions do not support, as a `"""..."""` string holding real quotes, and reports diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rules.md index 6678e5438f..590199fdb6 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rules.md @@ -22,7 +22,7 @@ Add `--json` after `rules` for machine-readable list or run output: ```bash sqb rules --json list -sqb rules --json run SQBRSQL +sqb rules --json run SQLCLEAN ``` Lifecycle progress is written to stderr so stdout remains valid JSON. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md index ab7cff180e..4cf0ec9046 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md @@ -316,7 +316,7 @@ fully typed. - Every read inside `USING (...)` stands unqualified, as `USING` needs, including `USING (@weekly.week)` for a column the type does not declare. - In a column-list macro, `o.@weekly.keys` outputs the group's columns under their own names. -- Rules judge a read as you wrote it. `SQBRSQL028`, which rejects a query mixing qualified and +- Rules judge a read as you wrote it. `SQLCLEAN028`, which rejects a query mixing qualified and unqualified columns, judges a query the macro writes whole, such as a CTE of a CTE-list macro, once, in the macro file. There every plain read counts as unqualified and every `alias.@weekly...` read as qualified. A step written only in plain reads, such as diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 19740dcb30..9a67ec2de1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -710,20 +710,21 @@ the same checks: ```toml [rules] -select = ["SQBRSQL", "SQBRGRAPH", "XSQBRARCH"] +select = ["SQLCLEAN", "SQBRGRAPH", "XSQBRARCH"] ``` -Rules are opt-in. Exact codes activate individual checks and prefixes activate a family. Built-in -codes begin with `SQBR`; repository-defined codes begin with `XSQBR`. See +Without `select`, the SQL Rules enabled by default run; `select = []` runs no configurable Rules. +Exact codes activate individual checks and prefixes activate a family. SQL Rule codes begin with +`SQLCLEAN`, other built-in codes with `SQBR`, and repository-defined codes with `XSQBR`. See [Compiler-integrated Rules](rules.md) for configuration, authoring, and exceptions. | Key | Description | | --- | --- | | `ignore` | Rule codes or families turned off for the whole project. | -| `[[rules.rule_exceptions]]` | One reviewed exception: `rule`, `path` and `reason`, plus `relation` and optional `cte` for relation-level Rules such as `SQBRSQL038`. Stale entries fail compilation. See [Findings and exceptions](rules/findings-and-exceptions.md). | +| `[[rules.rule_exceptions]]` | One reviewed exception: `rule`, `path` and `reason`, plus `relation` and optional `cte` for relation-level Rules such as `SQLCLEAN038`. Stale entries fail compilation. See [Findings and exceptions](rules/findings-and-exceptions.md). | | `[[rules.rule_ignores]]` | Rules turned off for `paths` or `selectors`, with a `reason`. | -Rule exceptions live only in this file. Inline `-- sqb: ignore` comments are `SQBRSQL000` errors +Rule exceptions live only in this file. Inline `-- sqb: ignore` comments are `SQBRSUPPRESSION000` errors that print the entry to add. ## Scopes diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md index 3533e1495d..daeba27ce3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md @@ -10,7 +10,7 @@ Online: https://sqlbuild.com/docs/concepts/rules/ - What Rules are for - How Rules work -- Where SQL linting went +- SQL Rules - Built-in and custom Rules - One definition of project validity - Rules compared with tests, audits, and contracts @@ -54,30 +54,31 @@ model. Use `sqb rules show` to understand a Rule before enabling it: ```bash -sqb rules show SQBRSQL004 +sqb rules show SQLCLEAN004 ``` ```text -SQBRSQL004: Row selection is nondeterministic -Family: SQBRSQL +SQLCLEAN004: Row selection is nondeterministic +Family: SQLCLEAN Slug: unordered-limit Subject: model Enabled by default: yes Remediation: Add ORDER BY with a deterministic tie-breaker before LIMIT or OFFSET. ``` -`Enabled by default` means the Rule is included when a matching family is selected. Configurable -Rules do not run unless `select` contains an exact code or matching family. +`Enabled by default` means the Rule runs when `select` is unset. Without `select`, SQLBuild runs +the default SQL Rules and no other configurable Rules. Select Rules in `sqlbuild_project.toml`. This example enables the complete built-in SQL family: ```toml [rules] -select = ["SQBRSQL"] +select = ["SQLCLEAN"] ``` -An exact code such as `SQBRSQL004` selects one Rule. A family code such as `SQBRSQL` selects every -Rule in that family. An empty `select` disables configurable Rules, but mandatory compiler +An exact code such as `SQLCLEAN004` selects one Rule. A family code such as `SQLCLEAN` selects every +Rule in that family, including Rules that are not enabled by default. To run no configurable Rules, set +`select = []`; to turn off only the SQL Rules, add `SQLCLEAN` to `ignore`. Mandatory compiler correctness still applies. Now consider a model that returns an arbitrary row because its `LIMIT` has no ordering: @@ -96,7 +97,7 @@ sqb compile The selected Rule reports the source location, problem, and remediation: ```text -error[SQBRSQL004]: Row selection is nondeterministic +error[SQLCLEAN004]: Row selection is nondeterministic --> models/orders.sql:2:36 | 2 | SELECT order_id FROM orders LIMIT 1 @@ -115,17 +116,20 @@ Use focused commands when discovering or adopting Rules: ```bash sqb rules list -sqb rules run SQBRSQL004 +sqb rules run SQLCLEAN004 ``` A focused run checks only the requested Rule or family. Run `sqb compile` before treating the whole project as valid. -## Where SQL linting went +## SQL Rules -SQL linting is part of the Rules system rather than a separate command. +SQL linting is part of the Rules system rather than a separate command. The SQL Rules come from +[SQLClean](https://sqlclean.io), the SQL linter and formatter SQLBuild runs on every SQL body. +They have the same codes, defaults, findings and fixes as in the `sqlclean` command; see the +[SQLClean Rule reference](https://sqlclean.io/docs/rules/overview/). -The `SQBRSQL` built-in family covers deterministic SQL diagnostics such as: +The `SQLCLEAN` family covers deterministic SQL diagnostics such as: - unsafe `NULL` comparisons; - implicit cartesian joins; @@ -146,7 +150,7 @@ SQL responsibility: | Responsibility | Owner | Example | |---|---|---| | Correctness required to understand the project | Compiler | Invalid syntax, unresolved references, incompatible output shape | -| Configurable, deterministic requirements | `SQBRSQL` Rules (executed by the compiler) | Unordered `LIMIT`, unused CTE, implicit cartesian join | +| Configurable, deterministic requirements | `SQLCLEAN` Rules (executed by the compiler) | Unordered `LIMIT`, unused CTE, implicit cartesian join | | Canonical presentation | [`sqb format`](../cli/format.md) (separate from the compiler) | Capitalization, indentation, spacing, comma and clause layout | Rules never rewrite source. `sqb format` rewrites source and does not decide whether a project @@ -154,8 +158,9 @@ passes its configured Rules. This keeps diagnostics and source mutation separate ## Built-in and custom Rules -SQLBuild maintains native built-in Rules for reusable requirements. Their codes begin with `SQBR`, -for example `SQBRSQL004` and `SQBRGRAPH101`. +SQLBuild maintains native built-in Rules for reusable requirements. SQL Rule codes begin with +`SQLCLEAN`, for example `SQLCLEAN004`. Rules that read the project, such as `SQBRGRAPH101`, begin +with `SQBR`. Projects can define custom Python Rules under `rules/**/*.py`. Their codes begin with `XSQBR`, for example `XSQBRARCH001`. Custom Rules use the same selection, findings, exceptions, execution order, @@ -164,7 +169,7 @@ and cache system as built-ins. ```toml [rules] select = [ - "SQBRSQL", + "SQLCLEAN", "SQBRGRAPH", "XSQBRARCH", ] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md index 6a8d53c6da..7c746213ae 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md @@ -19,28 +19,35 @@ Configure built-in and custom Rules together: ```toml [rules] -select = ["SQBRSQL", "SQBRGRAPH", "XSQBRARCH"] -ignore = ["SQBRSQL004"] +select = ["SQLCLEAN", "SQBRGRAPH", "XSQBRARCH"] +ignore = ["SQLCLEAN004"] ``` -An empty `select` disables configurable Rules. It does not disable mandatory compiler correctness. +Without `select`, SQLBuild runs the SQL Rules that are enabled by default and no other +configurable Rules. `select = []` runs no configurable Rules. Mandatory compiler correctness +always applies. + +SQL Rules read the SQL that SQLBuild analyses, so they skip a model with `sql_analysis false`, an +audit or SQL test file whose headers all set `sql_analysis false`, and every model in a run with +`--no-sql-analysis`. ## Codes and families -Built-in codes use `SQBR`, such as `SQBRSQL004`, `SQBRMODEL101`, and -`SQBRGRAPH101`. Custom codes use `XSQBR`, such as -`XSQBRARCH001`. +SQL Rule codes use `SQLCLEAN`, such as `SQLCLEAN004`. Other built-in codes use +`SQBR`, such as `SQBRMODEL101` and `SQBRGRAPH101`. Custom codes use +`XSQBR`, such as `XSQBRARCH001`. A family is always the code with its final three digits removed: ```text -SQBRSQL004 → SQBRSQL +SQLCLEAN004 → SQLCLEAN +SQBRGRAPH101 → SQBRGRAPH XSQBRARCH001 → XSQBRARCH XSQBR001 → XSQBR ``` -Exact codes select one Rule. Prefixes select every matching Rule. `ignore` uses the same matching -semantics and takes precedence over `select`. +Exact codes select one Rule. Families select every matching Rule, whether or not it is enabled by +default. `ignore` uses the same matching semantics and takes precedence over `select`. A selector that matches no Rule fails with `C000`. `SQBRTEST105` (scenario description) was retired when descriptions became the compile error `P010` @@ -51,7 +58,7 @@ Use the catalogue to inspect available codes and metadata: ```bash sqb rules list -sqb rules show SQBRSQL004 +sqb rules show SQLCLEAN004 ``` ## Rule options @@ -83,8 +90,8 @@ Unknown Rule codes, option names, and invalid option values fail configuration. Run one exact Rule or family while developing or adopting it: ```bash -sqb rules run SQBRSQL004 -sqb rules run SQBRSQL +sqb rules run SQLCLEAN004 +sqb rules run SQLCLEAN sqb rules run XSQBRARCH --select customer_orders ``` @@ -95,7 +102,7 @@ the complete configured enforcement performed by `sqb compile`. ## Join keys and final CTE names -`SQBRSQL040` checks every `JOIN ... ON` in the SQL covered by SQL Rules, including joins inside CTEs +`SQLCLEAN040` checks every `JOIN ... ON` in the SQL covered by SQL Rules, including joins inside CTEs and subqueries. `USING (...)` is accepted. Allowed predicates are: - Plain column comparisons, including equality, inequality, range and null-safe comparisons. @@ -114,16 +121,16 @@ projects `LOWER(category) AS category_key`, then join `ON o.category_key = c.cat variant path access such as `ON a.data:key = b.id` is also computed: project the value as a named column before joining. -`SQBRSQL041` checks CTE naming: the last top-level CTE must be named `final` (compared +`SQLCLEAN041` checks CTE naming: the last top-level CTE must be named `final` (compared case-insensitively, including quoted identifiers), and no other CTE, including a nested one, may use -that name. Queries without CTEs are unaffected, and SQL-test fixture CTEs are exempt. `SQBRSQL035` +that name. Queries without CTEs are unaffected, and SQL-test fixture CTEs are exempt. `SQLCLEAN035` owns the terminal `SELECT`'s plain projection from that last CTE, so select both, normally through -the `SQBRSQL` family, for the complete convention. Parentheses around the entire query do not create +the `SQLCLEAN` family, for the complete convention. Parentheses around the entire query do not create a nested CTE scope. ## Repeated JSON parsing -`SQBRSQL045` reports a JSON parse function applied to the same column more than once in one +`SQLCLEAN045` reports a JSON parse function applied to the same column more than once in one `SELECT`. Parse the payload once in the CTE that reads it, then read every field from that column: ```sql @@ -181,9 +188,9 @@ max_literal_length = 100 # default max_ranking_order_by = 5 # default ``` -- `SQBRSQL042`: every output column of a non-final CTE must be read later in the same model, +- `SQLCLEAN042`: every output column of a non-final CTE must be read later in the same model, in any clause of a later CTE or the final query, including through a later `SELECT *` - pass-through. The last CTE (the model output), unreachable CTEs (reported by `SQBRSQL005`), + pass-through. The last CTE (the model output), unreachable CTEs (reported by `SQLCLEAN005`), CTEs with a column list, recursive queries and SQL-test fixtures are exempt. A later step that uses an `@` interpolation in an expression may read any column, so it keeps every column, and interpolated outputs are never reported. A later step that reads the CTE as a whole row (the @@ -198,14 +205,14 @@ max_ranking_order_by = 5 # default `SELECT DISTINCT`, `DISTINCT ON`, `GROUP BY ALL`, `ORDER BY ALL`, an aggregate without `GROUP BY`, or positional `GROUP BY`/`ORDER BY`. Other findings are reported without a fix and name the reason. The fix repeats until nothing changes. -- `SQBRSQL043`: `ROW_NUMBER`, `RANK`, `DENSE_RANK`, `FIRST_VALUE` and `LAST_VALUE` windows may +- `SQLCLEAN043`: `ROW_NUMBER`, `RANK`, `DENSE_RANK`, `FIRST_VALUE` and `LAST_VALUE` windows may sort by at most `max_ranking_order_by` expressions. Order by the one or two columns that decide which row wins, then a unique row key as the final tie-breaker. If the rows have no unique column, build one in the upstream model from the columns that identify a row (for example `HASH(source_partition, source_offset, line_index) AS row_key`) and declare it with a `unique` - audit, so `SQBRSQL018` can prove the order. Raise the project limit only when the order + audit, so `SQLCLEAN018` can prove the order. Raise the project limit only when the order genuinely needs more keys. -- `SQBRSQL044`: a string literal longer than `max_literal_length` characters is an error. Define +- `SQLCLEAN044`: a string literal longer than `max_literal_length` characters is an error. Define the value once as a [constant](../constants.md) in the narrowest [scope](../declaration-scopes.md) that covers every use, and reference it with `@const`: a model-private `constants (_status_pattern '...')` entry in the `MODEL` header when @@ -215,7 +222,7 @@ max_ranking_order_by = 5 # default Otherwise shorten the literal. When the identical literal is reported in other files, the help lists them (up to five) and names the nearest common folder and its constants directory. Literals produced by `@const` or a macro are not reported, and there is no automatic fix. -- `SQBRSQL018` requires every `ROW_NUMBER`, `FIRST_VALUE` and `LAST_VALUE` window to be proven +- `SQLCLEAN018` requires every `ROW_NUMBER`, `FIRST_VALUE` and `LAST_VALUE` window to be proven deterministic: `PARTITION BY` and `ORDER BY` together must cover a unique key of the ranked rows. Keys come from `unique` column audits and model-level `unique` audits naming a column, when the column is also non-null (a `not_null` audit or `nullable false`), and from the `unique_key` of @@ -234,7 +241,6 @@ max_ranking_order_by = 5 # default A finding that `sqb format --fix` can fix says so in text output and carries `"fixable": true` in `sqb rules --json run` output; `sqb compile` adds the same note to the diagnostic. -Family selectors such as `select = ["SQBRSQL", "SQBRMODEL", "SQBRGRAPH"]` pick up new built-in Rules -on upgrade, so a release that adds `SQBRSQL040` to `SQBRSQL045` can expose computed join keys, -differently named final CTEs, unused CTE columns, long literals, ranking sorts or repeated JSON -parses that were accepted before. +Family selectors such as `select = ["SQLCLEAN", "SQBRMODEL", "SQBRGRAPH"]` pick up new built-in Rules +on upgrade, and an unset `select` picks up new default SQL Rules. A new Rule can report SQL that +compiled under an earlier release. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md index 41281df46d..e9598a47a9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md @@ -29,7 +29,7 @@ Use an exact exception for one known finding. Exceptions require a reason and ar ```toml [[rules.rule_exceptions]] -rule = "SQBRSQL004" +rule = "SQLCLEAN004" path = "models/examples/sample_orders.sql" reason = "This example intentionally demonstrates one sampled row." ``` @@ -44,12 +44,12 @@ paragraph (see [Folded descriptions](../models/configuration.md#folded-descripti ## Relation-level exceptions A file-wide exception for a deliberate cartesian product would also hide any accidental cross join -added to the same file later. `SQBRSQL038` findings name the joined relation and, inside a CTE, +added to the same file later. `SQLCLEAN038` findings name the joined relation and, inside a CTE, that CTE, so an exception can cover exactly one join: ```toml [[rules.rule_exceptions]] -rule = "SQBRSQL038" +rule = "SQLCLEAN038" path = "models/orders/order_checks.sql" relation = "order_policy" cte = "policy_checks" # optional: only the join inside this top-level CTE @@ -59,14 +59,14 @@ reason = "Every order is evaluated against every policy check." `relation` is the joined relation as written, with `__ref`, `__seed` and `__source` calls resolved to the referenced name. Without `cte`, the exception covers that relation anywhere in the file. Relation-level exceptions are stale-checked like exact exceptions: an entry that matches no finding -fails compilation. `relation` is accepted only for `SQBRSQL038`, and a file-wide exception cannot be +fails compilation. `relation` is accepted only for `SQLCLEAN038`, and a file-wide exception cannot be combined with relation-level exceptions for the same code and path. A join that reads no named relation, such as `CROSS JOIN UNNEST(...)` or `CROSS JOIN LATERAL`, has no relation to name. Its finding says so and prints the file-wide exception (rule and path) that is the only way to cover it. -`SQBRSQL038` does not report a cross join when either side provably holds at most one row, so most +`SQLCLEAN038` does not report a cross join when either side provably holds at most one row, so most intentional cross joins need no exception at all. SQLBuild infers this from the query, never from a declaration: a seed whose CSV has one data row (adding a second row brings the finding back), an aggregate without `GROUP BY`, a one-row `VALUES`, `LIMIT 1`, or `QUALIFY ROW_NUMBER() OVER (...) = 1` @@ -79,7 +79,7 @@ Use a path ignore when a documented project area intentionally follows a differe ```toml [[rules.rule_ignores]] -rules = ["SQBRSQL"] +rules = ["SQLCLEAN"] paths = ["models/examples/**"] reason = "Examples retain intentionally minimal SQL." ``` @@ -93,7 +93,7 @@ Use `selectors` when the exception follows graph-resource identities or lineage ```toml [[rules.rule_ignores]] -rules = ["SQBRSQL021"] +rules = ["SQLCLEAN021"] selectors = ["intermediate_*"] reason = "These intermediate interfaces intentionally preserve upstream columns." ``` @@ -104,7 +104,7 @@ matching, including SQL tests and audits that are not graph resources: ```toml [[rules.rule_ignores]] -rules = ["SQBRSQL021"] +rules = ["SQLCLEAN021"] paths = ["models/**/intermediate_*.sql"] reason = "These intermediate SQL files intentionally preserve upstream columns." ``` @@ -116,15 +116,15 @@ To turn a Rule off for the whole project, add its code to `[rules] ignore`. ## No inline suppressions SQLBuild has no inline suppression comment. An `-- sqb: ignore ...` or `/* sqb: ignore ... */` -comment is a `SQBRSQL000` error that suppresses nothing, so the finding it was meant to hide is reported too. The error prints the +comment is a `SQBRSUPPRESSION000` error that suppresses nothing, so the finding it was meant to hide is reported too. The error prints the exact entry to add instead: ```text -error[SQBRSQL000]: Inline Rule suppressions are not supported; declare the exception in sqlbuild_project.toml +error[SQBRSUPPRESSION000]: Inline Rule suppressions are not supported; declare the exception in sqlbuild_project.toml --> models/orders/order_checks.sql:4:1 - = help: Remove this comment, then fix the SQBRSQL038 finding, or record it as a reviewed exception in sqlbuild_project.toml: + = help: Remove this comment, then fix the SQLCLEAN038 finding, or record it as a reviewed exception in sqlbuild_project.toml: [[rules.rule_exceptions]] - rule = "SQBRSQL038" + rule = "SQLCLEAN038" path = "models/orders/order_checks.sql" relation = "order_policy" reason = "every order meets every policy" @@ -152,7 +152,7 @@ Use JSON when another tool consumes findings: ```bash sqb compile --json -sqb rules --json run SQBRSQL +sqb rules --json run SQLCLEAN ``` Machine-readable output remains on stdout. Lifecycle progress and terminal status are written to diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index d12632c725..a64b35d2da 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5817,6 +5817,7 @@ class EarlySqlLint: def start( self, *, inputs: CompileInputsView, dialect: str, value_renderer: TypedSqlValueRenderer ) -> None: ... + def __call__(self, *, inputs: CompileInputsView) -> None: ... def stop(self) -> int: ... def scan_headers(*, contents: str, first_only: bool = False) -> tuple[HeaderSpan, ...]: ... diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/audit/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/audit/helpers.py index fd8e1e11d5..a9d24c3e5b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/audit/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/audit/helpers.py @@ -210,8 +210,10 @@ def order_quality(): ' expression_is_true (name "known_region", expression "region IS NOT NULL"),\n' " ],\n" ");\n\n" - "SELECT * FROM (VALUES (1, 'emea', 10), (2, 'emea', 20), (3, 'apac', 30))" - " AS orders(order_id, region, amount)\n" + "WITH orders(order_id, region, amount) AS (\n" + " VALUES (1, 'emea', 10), (2, 'emea', 20), (3, 'apac', 30)\n" + ")\n" + "SELECT * FROM orders\n" ), } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index ba5d3ed224..a71e643d6e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -557,7 +557,8 @@ def prepare_attached_audit_gate_project( "audits/generic/code_check.sql": _GATE_AUDIT_TEMPLATE.format(read=read), "models/valid_codes.sql": ( "MODEL (description 'Test model valid_codes.', " - "materialized table);\n\nSELECT code FROM (VALUES ('A'), ('B')) AS valid(code)\n" + "materialized table);\n\n" + "WITH valid(code) AS (VALUES ('A'), ('B'))\nSELECT code FROM valid\n" ), } files.update( @@ -607,7 +608,7 @@ def prepare_nested_source_gate_project(*, tmp_path: Path, raw_code: str) -> Path ), "models/valid_codes.sql": ( "MODEL (description 'Test model valid_codes.', materialized table);\n\n" - "SELECT code FROM (VALUES ('A'), ('B')) AS valid(code)\n" + "WITH valid(code) AS (VALUES ('A'), ('B'))\nSELECT code FROM valid\n" ), }, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py index 57a5c0b7bc..d788ead43c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py @@ -136,7 +136,7 @@ def test_given_changed_unselected_upstream_when_building_leaf_then_warns_and_noo "models/b.sql": "MODEL (description 'Test model b.', materialized table);\n\nselect 1 as id\n", "models/c.sql": ( "MODEL (description 'Test model c.', materialized table);\n\n" - 'select * from __ref("a") union all select * from __ref("b")\n' + 'select id from __ref("a") union all select id from __ref("b")\n' ), }, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py index d606fca874..14512d3b2e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py @@ -96,10 +96,11 @@ def test_given_monthly_producer_with_midmonth_max_when_building_capped_daily_con batch_size 1d, microbatch_limit (max_batches 7, action cap_from_end), ); + WITH meetings AS (SELECT COUNT(*) AS row_count FROM __ref("monthly_meetings")) SELECT CAST(__cursor_start() AS TIMESTAMP) AS batch_start, CAST(__cursor_end() AS TIMESTAMP) AS batch_end - FROM (SELECT COUNT(*) AS row_count FROM __ref("monthly_meetings")) + FROM meetings """ ).strip() + "\n", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_config_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_config_build.py index 3063cef9c9..ec9627fe60 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_config_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_config_build.py @@ -36,8 +36,8 @@ ), "models/staging/stg_orders.sql": ( 'MODEL (description "Staged orders.");\n\n' - "SELECT * FROM (VALUES (1, 10, 'placed'), (2, 20, 'shipped'))" - " AS t(order_id, amount, status)\n" + "WITH t(order_id, amount, status) AS (VALUES (1, 10, 'placed'), (2, 20, 'shipped'))\n" + "SELECT * FROM t\n" ), "models/marts/order_totals.sql": ( 'MODEL (\n description "Order totals for ${CTX:run.target}.",\n' diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_unbuilt_dependencies.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_unbuilt_dependencies.py index eea6167c71..3339c39b67 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_unbuilt_dependencies.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_unbuilt_dependencies.py @@ -92,8 +92,8 @@ "models/summary.sql": ( "MODEL (description 'Test model summary.');\n" "SELECT c.customer_id, f.order_id\n" - 'FROM __ref("customers") AS c,\n' - ' __table_fn("customer_orders")(c.customer_id) AS f\n' + 'FROM __ref("customers") AS c\n' + ' CROSS JOIN __table_fn("customer_orders")(c.customer_id) AS f\n' ), "tests/unit/test_summary.sql": ( "TEST();\nWITH\n__ref__customers AS (SELECT 1 AS customer_id),\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py index 7e3220b3fc..3d7dbd2ae1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py @@ -562,7 +562,8 @@ def test_given_multi_schema_functions_when_cloning_then_destination_graph_is_que """ MODEL (description "Test model orders.", materialized table, schema model_origin); - SELECT * FROM (VALUES (1, 10), (2, 20)) AS orders(order_id, amount) + WITH amounts(order_id, amount) AS (VALUES (1, 10), (2, 20)) + SELECT * FROM amounts """ ).strip() + "\n", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index c1ac4c26c4..c7d2892773 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -39,6 +39,7 @@ class SemanticCorpusCase: expected_exit_code: int expected_codes: tuple[str, ...] pending_native: bool + ignored_rules: tuple[str, ...] = () @dataclass(frozen=True) @@ -1360,6 +1361,7 @@ class DialectSyntaxCompileE2ETestCase: expected_exit_code: int expected_codes: tuple[str, ...] expected_column_counts: dict[str, int] + ignored_rules: tuple[str, ...] = () @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/corpus.json b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/corpus.json index f2e97f52db..701ac3f3c6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/corpus.json +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/corpus.json @@ -98,7 +98,10 @@ "pending_native": false, "repo_files": { "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nSELECT id FROM __source(\"raw__orders\") QUALIFY ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at) = 1\n" - } + }, + "ignored_rules": [ + "SQLCLEAN018" + ] }, { "description": "t05: AVG(TIMESTAMP)", @@ -1766,7 +1769,10 @@ "pending_native": true, "repo_files": { "models/probe.sql": "MODEL (description \"Test model probe.\", materialized table);\n\nWITH a AS (SELECT id FROM __source(\"raw__orders\")) SELECT * FROM b\n" - } + }, + "ignored_rules": [ + "SQLCLEAN005" + ] }, { "description": "x05: raw table reference", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 2262e3926c..34e0792349 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -128,6 +128,13 @@ def prepare_semantic_corpus_project( for relative_path, contents in test_case.repo_files.items(): path: Path = project / relative_path path.write_text(contents, encoding="utf-8") + ignored: str = ", ".join(f'"{code}"' for code in test_case.ignored_rules) + ignore_entry: str = ( + f'\n[[rules.rule_ignores]]\nrules = [{ignored}]\npaths = ["models/**"]\n' + 'reason = "The probe SQL exercises compiler semantics, not this Rule."\n' + ) + with (project / "sqlbuild_project.toml").open("a", encoding="utf-8") as config: + config.write({True: ignore_entry, False: ""}[bool(test_case.ignored_rules)]) return project @@ -1271,6 +1278,11 @@ def _write_compile_project_config(project_dir: Path) -> None: "[targets.dev]", 'schema = "main"', "", + "[[rules.rule_ignores]]", + 'rules = ["SQLCLEAN018"]', + 'paths = ["models/**"]', + 'reason = "The generated workload ranks by a non-unique id; keep its SQL stable."', + "", ) ), encoding="utf-8", @@ -1642,6 +1654,11 @@ def _layered_write_project_config(*, project_dir: Path) -> None: [path_defaults."mart"] materialized = "table" + +[[rules.rule_ignores]] +rules = ["SQLCLEAN039"] +paths = ["models/**"] +reason = "The workload mixes nested query shapes on purpose to cover their compile path." """, encoding="utf-8", ) @@ -2590,7 +2607,11 @@ def write_relation_stub_project( "[connection]\n" 'database = "orders.duckdb"\n\n' "[targets.dev]\n" - 'schema = "main"\n' + 'schema = "main"\n\n' + "[[rules.rule_ignores]]\n" + 'rules = ["SQLCLEAN005"]\n' + 'paths = ["models/order_status.sql"]\n' + 'reason = "The stub-named CTE is left unread on purpose."\n' ), "models/orders.sql": ( "MODEL (description 'Test model orders.', " @@ -5347,6 +5368,16 @@ def dialect_syntax_model(description: str, sql: str, config: str = "") -> str: return f"MODEL (description '{description}'{config});\n{sql}\n" +def dialect_ignored_rules_entry(codes: tuple[str, ...]) -> str: + """A scoped ignore of `codes` for cases whose SQL shape is the parser input under test.""" + + ignored: str = ", ".join(f'"{code}"' for code in codes) + return ( + f'\n[[rules.rule_ignores]]\nrules = [{ignored}]\npaths = ["models/**"]\n' + 'reason = "The case covers parsing this SQL shape."\n' + ) + + def append_to_compiled_sql_tests(project_dir: Path) -> None: """Append a comment to every compiled SQL test artifact, changing its size.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py index 8d3c8d95af..77cc1c3143 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py @@ -24,7 +24,11 @@ BASE_FILES: dict[str, str] = { "sqlbuild_project.toml": ( 'name = "shop"\nadapter = "duckdb"\n\n[connection]\ndatabase = "shop.duckdb"\n\n' - "[scopes]\nenforce_placement = false\n" + "[scopes]\nenforce_placement = false\n\n" + "[[rules.rule_ignores]]\n" + 'rules = ["SQLCLEAN018"]\n' + 'paths = ["macros/sql/**", "models/**"]\n' + 'reason = "The dedup macros rank by updated_at alone; these tests cover expansion."\n' ), "seeds/raw_orders.csv": ( "order_id,status,amount_cents,fee_cents,tax_cents,created_at,updated_at,is_deleted," @@ -197,6 +201,16 @@ def with_rules(*codes: str) -> dict[str, str]: } +def with_ignored_rules(*codes: str, reason: str) -> dict[str, str]: + """The base project with the given Rules ignored, for cases whose SQL shape is their subject.""" + + ignored: str = ", ".join(f'"{code}"' for code in codes) + return { + "sqlbuild_project.toml": BASE_FILES["sqlbuild_project.toml"] + + f'\n[[rules.rule_ignores]]\nrules = [{ignored}]\npaths = ["**"]\nreason = "{reason}"\n' + } + + def check_macro(arguments: str, body: str) -> dict[str, str]: """The SQL macro `check` returning a boolean from `body`.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_build.py index 620db772a1..16c69ca91b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_build.py @@ -27,6 +27,7 @@ orders_model, output, run_project, + with_ignored_rules, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import query_duckdb, run_sqb @@ -419,6 +420,7 @@ def test_given_any_spelling_of_list_calls_when_building_then_the_model_holds_the 'WITH o AS (SELECT order_id FROM __ref("stg_orders"))\n' 'SELECT o.order_id, @sql.audit_columns(o) FROM __ref("stg_orders") AS o\n' ), + **with_ignored_rules("SQLCLEAN005", reason="The shadowed CTE is never read."), }, command=("compile",), expected_returncode=0, @@ -581,9 +583,7 @@ def test_given_an_edited_sql_macro_when_rebuilding_then_its_callers_use_the_new_ 'MACRO (description "Net", arguments (gross BIGINT, refund BIGINT), ' "returns DOUBLE);\n\n" ), - expected_ending=( - "ROUND(\n CAST((\n @gross - COALESCE(@refund, 0)\n ) AS double)\n / 100,\n 2\n)\n" - ), + expected_ending="ROUND(CAST((@gross - COALESCE(@refund, 0)) AS double) / 100, 2)\n", ), SqlMacroFormatTestCase( description="column_list_macro_body_and_multi_line_header", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_cte_names.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_cte_names.py index e91aafcafe..b1a106cea5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_cte_names.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_cte_names.py @@ -22,6 +22,7 @@ picked_orders, run_project, text_macro, + with_ignored_rules, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import query_duckdb @@ -63,7 +64,13 @@ ), SqlMacroBuildTestCase( description="a_caller_reads_a_prefixed_intermediate_cte", - files={**CLEAN_ORDERS, **orders_model(_INTERMEDIATE)}, + files={ + **CLEAN_ORDERS, + **orders_model(_INTERMEDIATE), + **with_ignored_rules( + "SQLCLEAN005", reason="The caller reads one intermediate CTE of the macro." + ), + }, query="SELECT live_rows FROM orders", expected_rows=[(3,)], ), @@ -85,17 +92,25 @@ ), SqlMacroBuildTestCase( description="a_nested_with_shadowing_a_cte", - files=picked_orders( - "WITH live AS (SELECT * FROM @src),\nlatest AS (WITH live AS (SELECT 42 AS order_id) SELECT order_id FROM live)" - ), + files={ + **picked_orders( + "WITH live AS (SELECT * FROM @src),\nlatest AS (WITH live AS (SELECT 42 AS order_id) SELECT order_id FROM live)" + ), + **with_ignored_rules("SQLCLEAN005", reason="The shadowed outer CTE is never read."), + }, query="SELECT order_id FROM orders", expected_rows=[(42,)], ), SqlMacroBuildTestCase( description="a_cte_read_in_a_derived_table", - files=picked_orders( - "WITH live AS (SELECT * FROM @src WHERE order_id = 2),\nlatest AS (SELECT s.order_id FROM (SELECT live.order_id FROM live) AS s)" - ), + files={ + **picked_orders( + "WITH live AS (SELECT * FROM @src WHERE order_id = 2),\nlatest AS (SELECT s.order_id FROM (SELECT live.order_id FROM live) AS s)" + ), + **with_ignored_rules( + "SQLCLEAN039", reason="The case reads a CTE in a derived table." + ), + }, query="SELECT order_id FROM orders", expected_rows=[(2,)], ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_relation_slots.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_relation_slots.py index 2eea85e385..eea790a291 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_relation_slots.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_relation_slots.py @@ -36,6 +36,7 @@ refunded_weeks, run_project, slot_model, + with_ignored_rules, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import query_duckdb, run_sqb @@ -416,6 +417,9 @@ def test_given_wrong_slot_mappings_when_compiling_then_the_error_names_the_slot( "SELECT s.@weekly.keys FROM @weekly AS s),\n" "grouped AS (SELECT @weekly.keys FROM refunded GROUP BY @weekly.keys" ), + **with_ignored_rules( + "SQLCLEAN005", reason="The chain's last CTE is a slot read the caller ignores." + ), }, command=("compile",), expected_returncode=0, @@ -430,6 +434,9 @@ def test_given_wrong_slot_mappings_when_compiling_then_the_error_names_the_slot( "SELECT t.@weekly.keys FROM (SELECT @weekly.keys, week FROM @weekly) AS t\n" "WHERE t.week IN (SELECT r.week FROM @refunds AS r)" ), + **with_ignored_rules( + "SQLCLEAN039", reason="The case reads slots in a derived table." + ), }, command=("compile",), expected_returncode=0, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py index 52b91805db..3e88f700d7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py @@ -23,6 +23,7 @@ text_macro, typed_position_files, untyped_text_macro, + with_ignored_rules, with_rules, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb @@ -130,6 +131,11 @@ files={ **_MACROS, **typed_position_files(shape, sql.replace("{m}", _UNKNOWN_CALL), declared), + **with_ignored_rules( + "SQLCLEAN004", + "SQLCLEAN039", + reason="The matrix covers calls in derived tables and scalar subqueries.", + ), }, expected_returncode=0, expected_fragments=_CLEAN, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dialect_syntax.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dialect_syntax.py index 7c55560bcd..5d5e846f1c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dialect_syntax.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dialect_syntax.py @@ -12,7 +12,10 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( DialectSyntaxCompileE2ETestCase, ) -from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import dialect_syntax_model +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + dialect_ignored_rules_entry, + dialect_syntax_model, +) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb _CONNECTIONS: dict[str, str] = { @@ -52,6 +55,7 @@ expected_exit_code=0, expected_codes=(), expected_column_counts={"unaliased": 2, "set_operation": 1, "renamed": 2}, + ignored_rules=("SQLCLEAN039",), ), DialectSyntaxCompileE2ETestCase( description="star EXCLUDE drops a column before set operation arms are compared", @@ -66,6 +70,7 @@ expected_exit_code=0, expected_codes=(), expected_column_counts={"orders": 1}, + ignored_rules=("SQLCLEAN007", "SQLCLEAN039"), ), DialectSyntaxCompileE2ETestCase( description="column alias lists name table function and table columns", @@ -101,6 +106,7 @@ expected_exit_code=0, expected_codes=(), expected_column_counts={"totals": 1, "names": 1}, + ignored_rules=("SQLCLEAN039",), ), DialectSyntaxCompileE2ETestCase( description="Snowflake rejects Oracle's CONNECT_BY_ISLEAF pseudo-column", @@ -142,6 +148,7 @@ expected_exit_code=0, expected_codes=(), expected_column_counts={"codes": 1}, + ignored_rules=("SQLCLEAN039",), ), DialectSyntaxCompileE2ETestCase( description="a ClickHouse DATE_TRUNC timestamp cursor keeps its timestamp type", @@ -186,6 +193,9 @@ def test_given_dialect_syntax_when_compiling_then_analysis_parses_and_types_it( "sqlbuild_project.toml": ( f'name = "shop"\nadapter = "{test_case.adapter}"\n\n' f"{_CONNECTIONS[test_case.adapter]}\n[scopes]\nenforce_placement = false\n" + + {True: dialect_ignored_rules_entry(test_case.ignored_rules), False: ""}[ + bool(test_case.ignored_rules) + ] ), **{f"models/{name}.sql": sql for name, sql in test_case.models.items()}, }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_benchmark_compile.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_benchmark_compile.py index 184b99c169..1a545b3743 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_benchmark_compile.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_duckshop_benchmark_compile.py @@ -73,7 +73,7 @@ def test_given_benchmark_project_when_compiling_then_reports_no_errors_or_warnin expected_errors=0, expected_warnings=0, expected_compiled_checksum=( - "a36dfda0afbe642c86441adc078d026cf6eefc95921173c32df74f2783a04f87" + "1ff1d0955ea3839b3c969707fd47ca9a4d7d7216753759d23911836f6ea5879d" ), ) ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_name_validation.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_name_validation.py index de78810c9e..b0fdfb24c8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_name_validation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_function_name_validation.py @@ -24,8 +24,8 @@ "LEFT(value, LENGTH(prefix)) = prefix\n" ) -_SELECT: str = "SELECT\n" -_FROM: str = "\nFROM (SELECT 'anvil' AS product_name) AS catalog_items\n" +_SELECT: str = "WITH catalog_items AS (SELECT 'anvil' AS product_name)\nSELECT\n" +_FROM: str = "\nFROM catalog_items\n" _SNOWFLAKE_PROJECT: str = 'name = "products"\nadapter = "snowflake"\n' _DUCKDB_PROJECT: str = ( 'name = "products"\nadapter = "duckdb"\n\n[connection]\ndatabase = ":memory:"\n' @@ -44,7 +44,7 @@ ( "B101", "Unknown function 'STARTS_WITH' for dialect Snowflake; did you mean STARTSWITH?", - 4, + 5, 5, ), ), @@ -71,7 +71,9 @@ ), FunctionNameCompileTestCase( description="snowflake derived-table alias named like an alias compiles", - project_toml=_SNOWFLAKE_PROJECT, + project_toml=_SNOWFLAKE_PROJECT + + '\n[[rules.rule_ignores]]\nrules = ["SQLCLEAN039"]\npaths = ["models/**"]\n' + + 'reason = "The case names a derived table like a date part."\n', query_sql="SELECT x FROM (VALUES (1)) day_of_week (x)\n", expected_exit_code=0, expected_diagnostics=(), @@ -85,7 +87,7 @@ ( "B101", "Unknown function 'STARTSWITH' for dialect DuckDB; did you mean STARTS_WITH?", - 4, + 5, 5, ), ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_star_beside_cast_constant.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_star_beside_cast_constant.py index 311dbaece0..497a58b1d3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_star_beside_cast_constant.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_star_beside_cast_constant.py @@ -19,7 +19,9 @@ ) _PROJECT_TOML: str = ( - 'name = "orders"\nadapter = "duckdb"\n[connection]\ndatabase = "orders.duckdb"\n' + 'name = "orders"\nadapter = "duckdb"\n[connection]\ndatabase = "orders.duckdb"\n\n' + '[[rules.rule_ignores]]\nrules = ["SQLCLEAN039"]\npaths = ["models/flagged_orders.sql"]\n' + 'reason = "One case stars over a derived table on purpose."\n' ) _BASE_ORDERS_SQL: str = ( "MODEL (description 'Orders.', materialized table);\n\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/format/test_token_preservation.py b/tests/e2e/src/sqlbuild/cli/commands/main/format/test_token_preservation.py index c8cecc40ce..46c1b88268 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/format/test_token_preservation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/format/test_token_preservation.py @@ -79,14 +79,16 @@ def test_given_case_sensitive_tokens_when_formatting_then_cli_writes_them_exactl description="split literal and keyword-named aliases build unchanged", authored_sql=( 'MODEL (description "Order lines", materialized table);\n\n' + "with line as (select 1 as pos, 'waffle' as sku)\n" "select 'north'\n'east' as region, line.pos as index, line.sku as replace " - "from (select 1 as pos, 'waffle' as sku) as line\n" + "from line\n" ), expected_sql=( 'MODEL (description "Order lines", materialized table);\n\n' + "WITH line AS (\n SELECT\n 1 AS pos,\n 'waffle' AS sku\n)\n\n" "SELECT\n 'north'\n 'east' AS region,\n line.pos AS index,\n" " line.sku AS replace\n" - "FROM (\n SELECT\n 1 AS pos,\n 'waffle' AS sku\n) AS line\n" + "FROM line\n" ), expected_columns=["region", "index", "replace"], expected_rows=[("northeast", 1, "waffle")], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/helpers.py index 321a061610..8fc04a71a0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/helpers.py @@ -75,11 +75,12 @@ ), ); -SELECT order_id, customer_id, amount_cents, ordered_at, status -FROM ( +WITH ranked AS ( SELECT *, ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY event_seq DESC) AS event_rank FROM __seed("orders") -) AS ranked +) +SELECT order_id, customer_id, amount_cents, ordered_at, status +FROM ranked WHERE event_rank = 1 """, "models/marts/customer_totals.sql": """MODEL ( @@ -170,7 +171,11 @@ def build_orders_project_files(*, adapter: str) -> dict[str, str]: "[targets.prod.clone]\n" "allow_as_clone_destination = true\n\n" "[defaults]\n" - 'materialized = "table"\n' + 'materialized = "table"\n\n' + "[[rules.rule_ignores]]\n" + 'rules = ["SQLCLEAN018"]\n' + 'paths = ["models/staging/stg_orders.sql"]\n' + 'reason = "Seed event_seq is unique by construction; seeds declare no keys."\n' ), "seeds/customers.csv": _CUSTOMERS_CSV, "seeds/orders.csv": "event_seq,order_id,customer_id,amount_cents,ordered_at,status\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py index 7cd9ded090..eea08cac2f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py @@ -949,7 +949,11 @@ def build_complex_values_fixture_project_files(*, adapter_name: str) -> dict[str f'adapter = "{adapter_name}"\n\n' "[connection]\n" 'database = "complex_values_fixture.duckdb"\n' - 'schema = "main"\n' + 'schema = "main"\n\n' + "[[rules.rule_ignores]]\n" + 'rules = ["SQLCLEAN039"]\n' + 'paths = ["tests/unit/test_orders.sql"]\n' + 'reason = "The fixtures cover complex strings in a VALUES relation on every dialect."\n' ), "models/orders.sql": ( 'MODEL (description "Test model orders.", database "fixture_db", schema "main");\n\n' @@ -1237,12 +1241,13 @@ def build_diamond_chain_test_project_files(*, sql_analysis_enabled: bool) -> dic ), "models/customer_order_mix.sql": ( "MODEL (description 'Test model customer_order_mix.', materialized table);\n\n" + 'WITH customers AS (SELECT DISTINCT customer_id FROM __ref("stg_orders"))\n' "SELECT customer_id,\n" ' (SELECT COUNT(*) FROM __ref("large_orders") AS l\n' " WHERE l.customer_id = c.customer_id) AS large_count,\n" ' (SELECT COUNT(*) FROM __ref("small_orders") AS s\n' " WHERE s.customer_id = c.customer_id) AS small_count\n" - 'FROM (SELECT DISTINCT customer_id FROM __ref("stg_orders")) AS c\n' + "FROM customers AS c\n" ), "tests/unit/test_customer_order_mix.sql": ( 'TEST (name "customer_order_mix_matches");\n\n' @@ -1961,7 +1966,11 @@ def build_helper_reference_project_files( "[connection]\n" 'database = "helper_reference_demo.duckdb"\n\n' "[settings]\n" - f"sql_analysis = {str(sql_analysis).lower()}\n" + f"sql_analysis = {str(sql_analysis).lower()}\n\n" + "[[rules.rule_ignores]]\n" + 'rules = ["SQLCLEAN005"]\n' + 'paths = ["tests/unit/**"]\n' + 'reason = "These SQL tests leave helpers unread to cover reference detection."\n' ), "sources/item_returns.yml": ( "sources:\n" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules_concurrent.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules_concurrent.py index 287dc9703e..3d78bd8bce 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules_concurrent.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules_concurrent.py @@ -36,7 +36,9 @@ def test_given_sql_and_custom_exceptions_when_compiling_cold_and_warm_then_both_ models: Path = tmp_path / "models" models.mkdir() (models / "orders.sql").write_text( - "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id LIMIT 1\n" + "MODEL (description 'Test model orders.');\n" + "WITH source AS (SELECT 1 AS order_id UNION ALL SELECT 2 AS order_id)\n" + "SELECT s.order_id FROM source AS s LIMIT 1\n" ) rules: Path = tmp_path / "rules" rules.mkdir() diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py index 5d5cfb2244..e46fe043ac 100644 --- a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py @@ -34,11 +34,12 @@ def test_given_increasing_findings_when_publishing_cache_then_findings_are_store paths: list[Path] = [Path(f"models/orders_{index}.sql") for index in range(size)] for index, path in enumerate(paths[:-1]): (project_dir / path).write_text( - f'MODEL (description "Orders {index}");\nSELECT 1 AS order_id LIMIT 1\n', + f'MODEL (description "Orders {index}");\n' + "SELECT o.order_id FROM orders AS o LIMIT 1\n", encoding="utf-8", ) (project_dir / paths[-1]).write_text( - 'MODEL (description "Orders without a limit");\nSELECT 1 AS order_id\n', + 'MODEL (description "Orders without a limit");\nSELECT o.order_id FROM orders AS o\n', encoding="utf-8", ) models: list[SimpleNamespace] = [ diff --git a/tests/unit/src/sqlbuild/lint/helpers.py b/tests/unit/src/sqlbuild/lint/helpers.py index 27e5e38643..8b6c4b759d 100644 --- a/tests/unit/src/sqlbuild/lint/helpers.py +++ b/tests/unit/src/sqlbuild/lint/helpers.py @@ -147,12 +147,8 @@ def keyword_name_positions() -> tuple[tuple[str, str, str, str], ...]: def reserved_keyword_pairs() -> tuple[tuple[str, str], ...]: """Return (dialect, keyword) pairs for core keywords every dialect but Databricks reserves.""" - return tuple( - (dialect, keyword) - for dialect in _DIALECTS - if _RESERVED_NAMES[dialect] - for keyword in _FOLDED_KEYWORDS - ) + reserving: tuple[str, ...] = tuple(filter(_RESERVED_NAMES.__getitem__, _DIALECTS)) + return tuple(itertools.product(reserving, _FOLDED_KEYWORDS)) def format_project_json( diff --git a/website/src/content/docs/docs/cli/format.mdx b/website/src/content/docs/docs/cli/format.mdx index 13963130ee..66465a86d4 100644 --- a/website/src/content/docs/docs/cli/format.mdx +++ b/website/src/content/docs/docs/cli/format.mdx @@ -161,6 +161,9 @@ still fixed. When a difference cannot be attributed, each file's edits are check file whose SQL cannot be expanded is skipped with a `format-fix-skipped` warning that gives the reason, and the run continues. A cycle or the 128-pass limit fails without writing any edit. +`--fix` attempts the fixes SQLClean marks safe and never the unsafe ones; every edit is still +checked by the compiler as above. + | Rule | Decision | Reason | | --- | --- | --- | | SQLCLEAN001 NULL comparison | Refused | Replacing `= NULL` with `IS NULL` changes three-valued logic. | @@ -169,11 +172,13 @@ reason, and the run continues. A cycle or the 128-pass limit fails without writi | SQLCLEAN005 unused CTE | Applied | Unreferenced `SELECT` CTEs contribute no rows. Mutation statements and commented CTEs are excluded. | | SQLCLEAN006 redundant DISTINCT | Applied | The projected and grouping expressions are proved to coincide. | | SQLCLEAN008 bare UNION | Applied | `UNION DISTINCT` states the existing duplicate elimination, in dialects that support it. | -| SQLCLEAN013 parenthesized DISTINCT | Refused | Removing parentheses can change operator precedence. | +| SQLCLEAN012 redundant ELSE NULL | Applied | A `CASE` without `ELSE` already yields `NULL`. | +| SQLCLEAN013 parenthesized DISTINCT | Refused when unproved | Removing parentheses can change operator precedence, so such an edit is refused with the reason. | +| SQLCLEAN015 empty statement | Applied | A repeated terminator adds no statement. | | SQLCLEAN017 constant row count | Applied | `COUNT(1)` and `COUNT(*)` both count every row. | -| SQLCLEAN023 unused table alias | Refused | Not yet reviewed as meaning-preserving. | +| SQLCLEAN023 unused table alias | Applied | An alias no expression references is removed. | | SQLCLEAN025 implicit inner join | Applied | A bare `JOIN ... ON` is an inner join. | -| SQLCLEAN030 boolean CASE simplification | Refused | The `COALESCE` rewrite is not proved portable to every supported dialect. | +| SQLCLEAN030 boolean CASE simplification | Applied when proved | The `COALESCE` rewrite is kept only when recompiling shows the same output types and nullability; otherwise the file reports `format-fix-verification-failed`. | | SQLCLEAN042 unused CTE output | Applied when row-neutral | Only a column reference or deterministic scalar expression is removed, from a CTE without `DISTINCT`, `DISTINCT ON`, `GROUP BY ALL`, `ORDER BY ALL`, an ungrouped aggregate, or positional `GROUP BY`/`ORDER BY`. Other removals could change the rows and are refused with the reason. | `--fix` also rewrites a header description that escapes double quotes as `\"` or doubles them as diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index b7935a0301..4a521e186d 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -688,8 +688,9 @@ the same checks: select = ["SQLCLEAN", "SQBRGRAPH", "XSQBRARCH"] ``` -Rules are opt-in. Exact codes activate individual checks and prefixes activate a family. Built-in -codes begin with `SQBR`; repository-defined codes begin with `XSQBR`. See +Without `select`, the SQL Rules enabled by default run; `select = []` runs no configurable Rules. +Exact codes activate individual checks and prefixes activate a family. SQL Rule codes begin with +`SQLCLEAN`, other built-in codes with `SQBR`, and repository-defined codes with `XSQBR`. See [Compiler-integrated Rules](/docs/concepts/rules) for configuration, authoring, and exceptions. | Key | Description | diff --git a/website/src/content/docs/docs/concepts/rules.mdx b/website/src/content/docs/docs/concepts/rules.mdx index 297c6bd2a5..5bec227174 100644 --- a/website/src/content/docs/docs/concepts/rules.mdx +++ b/website/src/content/docs/docs/concepts/rules.mdx @@ -65,8 +65,9 @@ select = ["SQLCLEAN"] ``` An exact code such as `SQLCLEAN004` selects one Rule. A family code such as `SQLCLEAN` selects every -Rule in that family, including Rules that are not enabled by default. To run no SQL Rules, add -`SQLCLEAN` to `ignore`; mandatory compiler correctness still applies. +Rule in that family, including Rules that are not enabled by default. To run no configurable Rules, set +`select = []`; to turn off only the SQL Rules, add `SQLCLEAN` to `ignore`. Mandatory compiler +correctness still applies. Now consider a model that returns an arbitrary row because its `LIMIT` has no ordering: diff --git a/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx b/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx index 4c2e7b76ac..883f56f474 100644 --- a/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx +++ b/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx @@ -17,8 +17,9 @@ Without `select`, SQLBuild runs the SQL Rules that are enabled by default and no configurable Rules. `select = []` runs no configurable Rules. Mandatory compiler correctness always applies. -SQL Rules read the SQL that SQLBuild analyses, so they skip a model with `sql_analysis false` and -every model in a run with `--no-sql-analysis`. +SQL Rules read the SQL that SQLBuild analyses, so they skip a model with `sql_analysis false`, an +audit or SQL test file whose headers all set `sql_analysis false`, and every model in a run with +`--no-sql-analysis`. ## Codes and families From 444f1d32777a35b95523e138250c16811cbd9221 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:45:58 +0000 Subject: [PATCH 3/8] fix(rules): lint with the compiled target and vars, keep format fix eligibility SQL Rules now expand with the compiled project effective target and vars on every compile path instead of falling back to the default target when the early-lint reuse is missing. Commands without target flags use the project default explicitly. Restore the eight-code format --fix allowlist and its documented decisions, and cover --target and --vars end to end. --- .../sql_expansion_context.rs | 20 +-- .../src/bindings/_helpers/rules/constants.rs | 13 +- .../bindings/_helpers/rules/early_sql_lint.rs | 13 +- .../bindings/_helpers/rules/fix_session.rs | 7 +- .../bindings/_helpers/rules/lint_expansion.rs | 18 +- .../src/bindings/_helpers/rules/run_lint.rs | 3 +- .../src/bindings/_helpers/rules/sql_rules.rs | 12 +- .../sqlbuild/references/docs/cli/format.md | 11 +- .../cli/commands/main/rules/_test_types.py | 8 + .../main/rules/test_sql_rule_target_vars.py | 154 ++++++++++++++++++ .../sqlbuild/cli/commands/main/_test_types.py | 1 - .../cli/commands/main/test_format_fix.py | 9 +- .../rule_engine/native_rows/_test_types.py | 9 + .../test_sql_rule_cache_publication.py | 2 + .../native_rows/test_sql_rule_compile_vars.py | 78 +++++++++ website/src/content/docs/docs/cli/format.mdx | 11 +- 16 files changed, 312 insertions(+), 57 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_rule_target_vars.py create mode 100644 tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_compile_vars.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs index 056b4d827d..e45811c11c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs @@ -96,10 +96,13 @@ pub(crate) struct ExpansionSources<'py> { /// Where expansion takes its target and vars from. pub(crate) enum ExpansionVars<'py> { - /// The default target's effective vars, overridden by these CLI vars when given. - DefaultTarget(Option>), - /// The target and vars of the compiling command's macro context. - Compile(Bound<'py, PyAny>), + /// For commands without `--target`: the project's default target, with these CLI vars. + ProjectDefault(Option>), + /// The compile's effective target name and vars, exactly as the command resolved them. + Compile { + target_name: Bound<'py, PyAny>, + vars: Bound<'py, PyAny>, + }, } /// Assemble SQL expansion inputs; a static scope lends only its index to private macros. @@ -125,11 +128,8 @@ pub(crate) fn sql_expansion_inputs(sources: ExpansionSources<'_>) -> PyResult = native_field(&discovered, "project_config")?; let local_config: Bound<'_, PyAny> = native_field(&discovered, "local_config")?; let (target_name, effective_vars): (Bound<'_, PyAny>, Bound<'_, PyAny>) = match sources.vars { - ExpansionVars::Compile(context) => ( - native_field(&context, "target_name")?, - native_field(&context, "vars")?, - ), - ExpansionVars::DefaultTarget(cli_vars) => { + ExpansionVars::Compile { target_name, vars } => (target_name, vars), + ExpansionVars::ProjectDefault(cli_vars) => { let cli_vars: Bound<'_, PyAny> = cli_vars.unwrap_or_else(|| PyDict::new(py).into_any()); let (target_name, target_config) = target_selection(&discovered, None)?; let vars: Bound<'_, PyAny> = build_effective_vars( @@ -227,7 +227,7 @@ pub(crate) fn build_sql_expansion_context<'py>( sql_expansion_inputs(ExpansionSources { project_dir: arguments.value("project_dir")?, value_renderer: arguments.value("value_renderer")?, - vars: ExpansionVars::DefaultTarget(present(arguments.value("cli_vars")?)), + vars: ExpansionVars::ProjectDefault(present(arguments.value("cli_vars")?)), discovered_inputs: present(arguments.value("discovered_inputs")?), declaration_scope: present(arguments.value("declaration_scope")?), static_declaration_scope: present(arguments.value("static_declaration_scope")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs index 69dd8a1a59..56a557093d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs @@ -1,8 +1,17 @@ //! Authored SQL-test mode identifiers. pub(crate) const MODEL_TEST_MODE: &str = "model"; -/// The SQL Rules whose fixes may change a query's meaning, so `sqb format --fix` never applies them; as in SQLClean, every other SQL Rule fix is safe. -pub(crate) const UNSAFE_RULE_FIX_CODES: [&str; 1] = ["SQLCLEAN001"]; +/// The rule codes whose fixes `sqb format --fix` may apply after compiler verification. +pub(crate) const SAFE_RULE_FIX_CODES: [&str; 8] = [ + "SQLCLEAN002", + "SQLCLEAN003", + "SQLCLEAN005", + "SQLCLEAN006", + "SQLCLEAN008", + "SQLCLEAN017", + "SQLCLEAN025", + "SQLCLEAN042", +]; /// The note attached to a rule finding that has an available fix. pub(crate) const RULE_FIX_AVAILABLE_NOTE: &str = "Fix available: `sqb format --fix` applies it after compiler verification"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index 25220ba870..086f3a5241 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -21,6 +21,7 @@ use sqlbuild_rules::rule_cache::main::rules_bulk_cache_path::rules_bulk_cache_pa use sqlbuild_rules::rules::main::selected_sql_codes::selected_sql_codes; use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::compile_attachment::sql_expansion_context::ExpansionVars; use crate::bindings::_helpers::compile_attachment::sql_expansion_context::present; use crate::bindings::_helpers::compiled_views::relation_keys::model_key_row; use crate::bindings::_helpers::compiled_views::relation_keys::{ @@ -259,7 +260,7 @@ fn early_lint(py: Python<'_>, work: &EarlyLintWork) -> PyResult let context: SqlExpansionInputs = lint_expansion_inputs(LintExpansionSources { project_dir: project_dir.into_pyobject(py)?.into_any(), value_renderer: Some(work.reuse.value_renderer(py)), - compile: Some(&work.reuse), + vars: work.reuse.expansion_vars(py)?, discovered_inputs: Some(work.reuse.discovered_inputs(py)?), declaration_scope: None, static_declaration_scope: present(work.reuse.declaration_scope(py)?), @@ -305,9 +306,13 @@ impl ExpansionReuse { record_field(self.inputs.bind(py), "declaration_scope") } - /// The compile's macro context, carrying its target and vars. - pub(crate) fn macro_context<'py>(&self, py: Python<'py>) -> PyResult> { - record_field(self.inputs.bind(py), "macro_context") + /// The compile's target and vars, read from its macro context. + pub(crate) fn expansion_vars<'py>(&self, py: Python<'py>) -> PyResult> { + let context: Bound<'py, PyAny> = record_field(self.inputs.bind(py), "macro_context")?; + Ok(ExpansionVars::Compile { + target_name: record_field(&context, "target_name")?, + vars: record_field(&context, "vars")?, + }) } /// The adapter that renders declared values. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index c0dfb8c809..a99c39500b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -16,6 +16,7 @@ use sqlbuild_rules::fix_verification::main::with_edits::with_edits; use sqlbuild_rules::fix_verification::models::{FixEdit, FixEligibility}; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, keywords}; +use crate::bindings::_helpers::compile_attachment::sql_expansion_context::ExpansionVars; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::compiled_views::relation_keys::{ RelationKeys, project_relation_keys, @@ -28,8 +29,8 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::{field, replaced}; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::render::authored_sql::SqlExpansionInputs; +use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; use crate::bindings::_helpers::rules::constants::SQL_RULE_PREFIX; -use crate::bindings::_helpers::rules::constants::UNSAFE_RULE_FIX_CODES; use crate::bindings::_helpers::rules::fix_facts::compile_facts; use crate::bindings::_helpers::rules::fix_verdict::fix_verdict; use crate::bindings::_helpers::rules::lint_expansion::{ @@ -172,7 +173,7 @@ impl<'py> FixSession<'py> { let sources: Vec = lint_sources(&project_dir, Some(&selected), given)?; let context: Option = if settings.native_enabled { Some(lint_expansion_inputs(LintExpansionSources { - compile: None, + vars: ExpansionVars::ProjectDefault(None), project_dir: self.project_dir.clone(), value_renderer: self.adapter.is_truthy()?.then(|| self.adapter.clone()), discovered_inputs: Some(updated_inputs(&self.inputs, files)?), @@ -273,7 +274,7 @@ impl<'py> FixSession<'py> { let facts = FixEligibility { model_body: model_paths.contains(&path)?, has_fix, - safe_code: !UNSAFE_RULE_FIX_CODES.contains(&code.extract::()?.as_str()), + safe_code: SAFE_RULE_FIX_CODES.contains(&code.extract::()?.as_str()), conditionless_join: code.eq("SQLCLEAN003")?, snowflake: field(&self.config, "dialect")?.eq("snowflake")?, }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs index 1046192ffd..726f6889d2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs @@ -19,7 +19,6 @@ use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::render::authored_sql::SqlExpansionInputs; -use crate::bindings::_helpers::rules::early_sql_lint::ExpansionReuse; const COMPILE_FIRST_MESSAGE: &str = "compiler-integrated Rules check the SQL your project \ actually produces, so the project must compile first"; @@ -47,7 +46,7 @@ pub(crate) fn build_lint_expansion_context<'py>( )?; let renderer: Bound<'py, PyAny> = arguments.value("value_renderer")?; lint_expansion_inputs(LintExpansionSources { - compile: None, + vars: ExpansionVars::ProjectDefault(None), project_dir: arguments.value("project_dir")?, value_renderer: renderer.is_truthy()?.then_some(renderer), discovered_inputs: present(arguments.value("discovered_inputs")?), @@ -58,11 +57,11 @@ pub(crate) fn build_lint_expansion_context<'py>( } /// Where lint's expansion inputs come from; a missing renderer is the project's adapter. -pub(crate) struct LintExpansionSources<'a, 'py> { +pub(crate) struct LintExpansionSources<'py> { pub(crate) project_dir: Bound<'py, PyAny>, pub(crate) value_renderer: Option>, - /// The compile whose macro context lint expands with, for its target and vars. - pub(crate) compile: Option<&'a ExpansionReuse>, + /// The target and vars lint expands with. + pub(crate) vars: ExpansionVars<'py>, pub(crate) discovered_inputs: Option>, pub(crate) declaration_scope: Option>, pub(crate) static_declaration_scope: Option>, @@ -70,13 +69,13 @@ pub(crate) struct LintExpansionSources<'a, 'py> { /// Lint's expansion inputs, reporting compile failures as `ProjectCompileError`. pub(crate) fn lint_expansion_inputs( - sources: LintExpansionSources<'_, '_>, + sources: LintExpansionSources<'_>, ) -> PyResult { let py: Python<'_> = sources.project_dir.py(); expansion_inputs(sources).or_else(|error| Err(compile_first_error(py, error)?)) } -fn expansion_inputs(sources: LintExpansionSources<'_, '_>) -> PyResult { +fn expansion_inputs(sources: LintExpansionSources<'_>) -> PyResult { let py: Python<'_> = sources.project_dir.py(); let project_dir: Bound<'_, PyAny> = sources.project_dir; let discovered: Bound<'_, PyAny> = match sources.discovered_inputs { @@ -115,10 +114,7 @@ fn expansion_inputs(sources: LintExpansionSources<'_, '_>) -> PyResult ExpansionVars::Compile(compile.macro_context(py)?), - None => ExpansionVars::DefaultTarget(None), - }, + vars: sources.vars, discovered_inputs: Some(discovered), declaration_scope: sources.declaration_scope, static_declaration_scope: sources.static_declaration_scope, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs index f1931dede7..486dbf6d53 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs @@ -8,6 +8,7 @@ use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_attachment::sql_expansion_context::ExpansionVars; use crate::bindings::_helpers::compile_attachment::sql_expansion_context::present; use crate::bindings::_helpers::render::authored_sql::SqlExpansionInputs; use crate::bindings::_helpers::rules::lint_expansion::{ @@ -125,7 +126,7 @@ fn run_lint<'py>( } else if settings.native_enabled { let renderer: Bound<'py, PyAny> = arguments.value("value_renderer")?; Some(lint_expansion_inputs(LintExpansionSources { - compile: None, + vars: ExpansionVars::ProjectDefault(None), project_dir: project_dir.clone(), value_renderer: renderer.is_truthy()?.then_some(renderer), discovered_inputs: present(arguments.value("discovered_inputs")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index 5b65648e48..e211ebcefa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -19,6 +19,7 @@ use sqlbuild_rules::rule_results::models::RuleFinding; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::boundary::python_calls::{imported, keywords}; +use crate::bindings::_helpers::compile_attachment::sql_expansion_context::ExpansionVars; use crate::bindings::_helpers::compiled_views::compiled_model_view::{ NativeCompiledModel, SqlRuleInputs, SqlRuleKey, }; @@ -30,8 +31,8 @@ use crate::bindings::_helpers::records::fields::field; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; +use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; use crate::bindings::_helpers::rules::constants::SQL_RULE_PREFIX; -use crate::bindings::_helpers::rules::constants::UNSAFE_RULE_FIX_CODES; use crate::bindings::_helpers::rules::early_sql_lint::{ EarlyLintResult, EarlySqlLint, ExpansionReuse, PreparedSqlLint, sql_lint_settings, sql_rule_cache_path, @@ -642,7 +643,7 @@ fn path_rows(outcome: LintOutcome) -> Vec<(PathBuf, LintViolationRow)> { .collect() } -/// Build lint expansion, reusing the compiler's scope only for the same discovery. +/// Build lint expansion with the compiled target and vars, reusing the compiler's scope only for the same discovery. fn lint_expansion_context( run: &SqlRun<'_, '_>, ) -> PyResult { @@ -658,7 +659,10 @@ fn lint_expansion_context( lint_expansion_inputs(LintExpansionSources { project_dir: run.project_dir.clone().into_pyobject(py)?.into_any(), value_renderer: reuse.map(|reuse| reuse.value_renderer(py)), - compile: reuse, + vars: ExpansionVars::Compile { + target_name: native_field(&run.project, "effective_target_name")?, + vars: native_field(&run.project, "effective_vars")?, + }, discovered_inputs: (!run.discovered_inputs.is_none()) .then(|| run.discovered_inputs.clone()), declaration_scope: reuse.map(|reuse| reuse.declaration_scope(py)).transpose()?, @@ -713,7 +717,7 @@ fn lint_finding( }; let path: String = posix(path); let fixable: bool = violation.fix.is_some() - && !UNSAFE_RULE_FIX_CODES.contains(&violation.code.as_str()) + && SAFE_RULE_FIX_CODES.contains(&violation.code.as_str()) && fixable_paths.contains(&path); RuleFinding { code: violation.code.clone(), diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md index 34ba87480d..5cf1bcd4ec 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md @@ -170,9 +170,6 @@ still fixed. When a difference cannot be attributed, each file's edits are check file whose SQL cannot be expanded is skipped with a `format-fix-skipped` warning that gives the reason, and the run continues. A cycle or the 128-pass limit fails without writing any edit. -`--fix` attempts the fixes SQLClean marks safe and never the unsafe ones; every edit is still -checked by the compiler as above. - | Rule | Decision | Reason | | --- | --- | --- | | SQLCLEAN001 NULL comparison | Refused | Replacing `= NULL` with `IS NULL` changes three-valued logic. | @@ -181,13 +178,11 @@ checked by the compiler as above. | SQLCLEAN005 unused CTE | Applied | Unreferenced `SELECT` CTEs contribute no rows. Mutation statements and commented CTEs are excluded. | | SQLCLEAN006 redundant DISTINCT | Applied | The projected and grouping expressions are proved to coincide. | | SQLCLEAN008 bare UNION | Applied | `UNION DISTINCT` states the existing duplicate elimination, in dialects that support it. | -| SQLCLEAN012 redundant ELSE NULL | Applied | A `CASE` without `ELSE` already yields `NULL`. | -| SQLCLEAN013 parenthesized DISTINCT | Refused when unproved | Removing parentheses can change operator precedence, so such an edit is refused with the reason. | -| SQLCLEAN015 empty statement | Applied | A repeated terminator adds no statement. | +| SQLCLEAN013 parenthesized DISTINCT | Refused | Removing parentheses can change operator precedence. | | SQLCLEAN017 constant row count | Applied | `COUNT(1)` and `COUNT(*)` both count every row. | -| SQLCLEAN023 unused table alias | Applied | An alias no expression references is removed. | +| SQLCLEAN023 unused table alias | Refused | Not yet reviewed as meaning-preserving. | | SQLCLEAN025 implicit inner join | Applied | A bare `JOIN ... ON` is an inner join. | -| SQLCLEAN030 boolean CASE simplification | Applied when proved | The `COALESCE` rewrite is kept only when recompiling shows the same output types and nullability; otherwise the file reports `format-fix-verification-failed`. | +| SQLCLEAN030 boolean CASE simplification | Refused | The `COALESCE` rewrite is not proved portable to every supported dialect. | | SQLCLEAN042 unused CTE output | Applied when row-neutral | Only a column reference or deterministic scalar expression is removed, from a CTE without `DISTINCT`, `DISTINCT ON`, `GROUP BY ALL`, `ORDER BY ALL`, an ungrouped aggregate, or positional `GROUP BY`/`ORDER BY`. Other removals could change the rows and are refused with the reason. | `--fix` also rewrites a header description that escapes double quotes as `\"` or doubles them as diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index 26dc9feaef..6e710ee77e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -367,3 +367,11 @@ class DefaultRuleSelectionCase: expected_exit: int expected_finding: bool expected_notices: int + + +@dataclass(frozen=True) +class SqlRuleTargetVarsCase: + description: str + commands: tuple[tuple[str, ...], ...] + expected_outcomes: tuple[tuple[int, bool, bool], ...] + extra_files: dict[str, str] = field(default_factory=dict) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_rule_target_vars.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_rule_target_vars.py new file mode 100644 index 0000000000..12a7bde3a6 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_rule_target_vars.py @@ -0,0 +1,154 @@ +"""E2E coverage of SQL Rules linting the SQL the command's `--target` and `--vars` expand.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import SqlRuleTargetVarsCase +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_PROJECT_TOML: str = ( + 'name = "shop"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "warehouse.duckdb"\n\n' + '[vars]\nprobe = "1"\n\n[targets.dev]\n\n[targets.prod]\n\n' + '[targets.prod.vars]\nprobe = "NULL"\nregion = "us"\n' +) +_MODEL_PATH: str = "models/orders.sql" +_MODEL_SQL: str = ( + "MODEL (description 'Test model orders.');\n" + 'SELECT o.id FROM __ref("raw_orders") AS o WHERE o.id = @@probe\n' +) +_UPSTREAM_FILES: dict[str, str] = { + "models/raw_orders.sql": "MODEL (description 'Test model raw orders.');\nSELECT 1 AS id\n" +} +_PROD_ONLY_MODEL: dict[str, str] = { + "models/regions.sql": ( + "MODEL (description 'Test model regions.');\nSELECT '@@region' AS region\n" + ) +} +_AUDIT_PATH: str = "audits/singular/orders_have_ids.sql" +_AUDIT_BODY: str = ( + 'SELECT o.id\nFROM __ref("orders") AS o\n' + 'INNER JOIN __ref("raw_orders") AS r ON o.id = r.id\nWHERE o.id = NULL\n' +) +_TEST_PATH: str = "tests/unit/test_orders.sql" +_TEST_BODY: str = ( + "WITH __ref__raw_orders AS (SELECT 1 AS id),\n" + "__expected__orders AS (SELECT 1 AS id WHERE 1 = NULL)\nSELECT 1\n" +) +_NULL_FINDING: str = "SQLCLEAN001" +_UNKNOWN_VAR: str = "L005" +_COMPILE: tuple[str, ...] = ("compile",) +_PROD: tuple[str, ...] = ("compile", "--target", "prod") +_NULL_VARS: tuple[str, ...] = ("compile", "--vars", '{"probe": "NULL"}') +_CLEAN: tuple[int, bool, bool] = (0, False, False) +_NULL: tuple[int, bool, bool] = (1, True, False) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlRuleTargetVarsCase( + description="default target lints its own vars", + commands=(_COMPILE,), + expected_outcomes=(_CLEAN,), + ), + SqlRuleTargetVarsCase( + description="--target lints that target's vars cold and warm", + commands=(_PROD, _PROD), + expected_outcomes=(_NULL, _NULL), + ), + SqlRuleTargetVarsCase( + description="--target without the cache lints that target's vars", + commands=(("compile", "--no-cache", "--target", "prod"),), + expected_outcomes=(_NULL,), + ), + SqlRuleTargetVarsCase( + description="--vars overrides the default target's vars cold and warm", + commands=(_NULL_VARS, _NULL_VARS), + expected_outcomes=(_NULL, _NULL), + ), + SqlRuleTargetVarsCase( + description="--vars after a default compile reuses no findings from it", + commands=(_COMPILE, _NULL_VARS), + expected_outcomes=(_CLEAN, _NULL), + ), + SqlRuleTargetVarsCase( + description="switching targets reuses no findings from the other target", + commands=(_COMPILE, _PROD, _COMPILE), + expected_outcomes=(_CLEAN, _NULL, _CLEAN), + ), + SqlRuleTargetVarsCase( + description="selected model lints with --target", + commands=(("compile", "--target", "prod", "--select", "orders"),), + expected_outcomes=(_NULL,), + ), + SqlRuleTargetVarsCase( + description="plan lints with --target", + commands=(("plan", "--target", "prod"),), + expected_outcomes=(_NULL,), + ), + SqlRuleTargetVarsCase( + description="build lints with --target", + commands=(("build", "--target", "prod"),), + expected_outcomes=(_NULL,), + ), + SqlRuleTargetVarsCase( + description="a var only the selected target defines expands without L005", + commands=(("compile", "--target", "prod", "--vars", '{"probe": "1"}'),), + expected_outcomes=(_CLEAN,), + extra_files=_PROD_ONLY_MODEL, + ), + SqlRuleTargetVarsCase( + description="--target still lints audit and SQL test files without an opt-out", + commands=(("compile", "--target", "prod", "--vars", '{"probe": "1"}'),), + expected_outcomes=(_NULL,), + extra_files={ + _AUDIT_PATH: "AUDIT ();\n" + _AUDIT_BODY, + _TEST_PATH: "TEST ();\n" + _TEST_BODY, + }, + ), + SqlRuleTargetVarsCase( + description="--target keeps audit and SQL test sql_analysis opt-outs", + commands=(("compile", "--target", "prod", "--vars", '{"probe": "1"}'),), + expected_outcomes=(_CLEAN,), + extra_files={ + _AUDIT_PATH: "AUDIT (sql_analysis false);\n" + _AUDIT_BODY, + _TEST_PATH: "TEST (sql_analysis false);\n" + _TEST_BODY, + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_target_and_vars_when_compiling_then_sql_rules_lint_that_expansion( + test_case: SqlRuleTargetVarsCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + _MODEL_PATH: _MODEL_SQL, + **_UPSTREAM_FILES, + **test_case.extra_files, + }, + ) + + outcomes: list[tuple[int, bool, bool]] = [] + outputs: list[str] = [] + for command in test_case.commands: + ran: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", *command), project_dir=project_dir + ) + output: str = ran.stdout + ran.stderr + outputs.append(output) + outcomes.append((ran.returncode, _NULL_FINDING in output, _UNKNOWN_VAR in output)) + + assert tuple(outcomes) == test_case.expected_outcomes, "\n----\n".join(outputs) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 4857d0e4eb..2effcb4699 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -17,7 +17,6 @@ class SemanticFixTestCase: expected_code: str dialect: str = "duckdb" expected_status: str = "applied" - expected_exit_code: int = 0 @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py index 46ded87c9c..64d94ccdc7 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py @@ -180,16 +180,16 @@ def test_given_safe_native_fix_when_formatting_then_compiler_and_query_results_a expected_status="refused", ), SemanticFixTestCase( - "unused table alias proven by recompiling", + "unused table alias", "SELECT order_id FROM (SELECT 1 AS order_id) AS o", "SQLCLEAN023", + expected_status="refused", ), SemanticFixTestCase( - "boolean CASE whose rewrite changes inferred nullability", + "boolean CASE", "SELECT CASE WHEN 1 = 2 THEN TRUE ELSE FALSE END AS active", "SQLCLEAN030", expected_status="refused", - expected_exit_code=1, ), ], ids=lambda case: case.description, @@ -205,9 +205,8 @@ def test_given_dialect_sensitive_rewrite_when_fixing_then_requires_semantic_proo model.write_text( 'MODEL (description "Orders", database example, schema analytics);\n' + test_case.sql ) - exit_code: int = main(["--project-dir", str(tmp_path), "format", "--fix", "--json"]) + assert main(["--project-dir", str(tmp_path), "format", "--fix", "--json"]) == 0 payload: dict[str, Any] = json.loads(capsys.readouterr().out) - assert exit_code == test_case.expected_exit_code assert any( fix["code"] == test_case.expected_code and fix["status"] == test_case.expected_status for fix in payload["rule_fixes"] diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py index d32e7d0435..c882cb0704 100644 --- a/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py @@ -36,3 +36,12 @@ class StructPassthroughTestCase: declared_type: str expected_exit_code: int expected_codes: tuple[str, ...] + + +@dataclass(frozen=True) +class SqlRuleCompileVarsTestCase: + """A compile's effective vars and whether SQL Rules report the NULL comparison they expand.""" + + description: str + effective_vars: dict[str, object] + expected_codes: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py index e46fe043ac..4f9c1d8253 100644 --- a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py @@ -55,6 +55,8 @@ def test_given_increasing_findings_when_publishing_cache_then_findings_are_store ] project: SimpleNamespace = SimpleNamespace( settings=SimpleNamespace(sql_analysis=True), + effective_target_name=None, + effective_vars={}, models=models, seeds=(), sources=(), diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_compile_vars.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_compile_vars.py new file mode 100644 index 0000000000..5ee168dde0 --- /dev/null +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_compile_vars.py @@ -0,0 +1,78 @@ +"""SQL Rules expand with the compile's effective target and vars when no early lint is reused.""" + +from pathlib import Path +from types import SimpleNamespace +from typing import Any, cast + +import pytest + +import sqlbuild._native as native +from sqlbuild._native import SqlRulesEvaluation +from tests.integration.src.sqlbuild.rule_engine.native_rows._test_types import ( + SqlRuleCompileVarsTestCase, +) + +_PROJECT_TOML: str = ( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[vars]\nprobe = "1"\n\n[targets.dev]\n\n[targets.prod.vars]\nprobe = "NULL"\n' +) +_MODEL_PATH: Path = Path("models/orders.sql") +_MODEL_SQL: str = ( + 'MODEL (description "Orders");\nSELECT o.order_id FROM orders AS o WHERE o.order_id = @@probe\n' +) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlRuleCompileVarsTestCase( + "compiled prod vars, not the default", {"probe": "NULL"}, ("SQLCLEAN001",) + ), + SqlRuleCompileVarsTestCase("compiled vars that are clean", {"probe": "2"}, ()), + ], + ids=lambda case: case.description, +) +def test_given_no_early_lint_when_running_sql_rules_then_compile_vars_are_expanded( + test_case: SqlRuleCompileVarsTestCase, tmp_path: Path +) -> None: + (tmp_path / "models").mkdir() + (tmp_path / "sqlbuild_project.toml").write_text(_PROJECT_TOML, encoding="utf-8") + (tmp_path / _MODEL_PATH).write_text(_MODEL_SQL, encoding="utf-8") + model: SimpleNamespace = SimpleNamespace( + relative_path=_MODEL_PATH, + authored_sql="SELECT o.order_id FROM orders AS o", + query_sql="SELECT o.order_id FROM orders AS o", + config=SimpleNamespace(values={}), + dynamic_column_contract=None, + relation_key_row=("model", "orders", "", "", "", False, None, [], []), + ) + project: SimpleNamespace = SimpleNamespace( + settings=SimpleNamespace(sql_analysis=True), + effective_target_name="prod", + effective_vars=test_case.effective_vars, + models=(model,), + seeds=(), + sources=(), + sql_expansions={}, + ) + plan: SimpleNamespace = SimpleNamespace( + project_dir=tmp_path, + project=project, + model_paths=frozenset((_MODEL_PATH,)), + selected=(SimpleNamespace(code="SQLCLEAN001", custom=False),), + effective_config=SimpleNamespace(thresholds={}, cache=SimpleNamespace(enabled=False)), + ) + + result: SqlRulesEvaluation = native.run_planned_sql_rules( + plan=cast(Any, plan), + graph=cast(Any, SimpleNamespace(project=project)), + dialect="duckdb", + discovered_inputs=cast(Any, None), + early_lint=None, + ) + + assert tuple(finding.code for finding in result.findings) == test_case.expected_codes + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/cli/format.mdx b/website/src/content/docs/docs/cli/format.mdx index 66465a86d4..13963130ee 100644 --- a/website/src/content/docs/docs/cli/format.mdx +++ b/website/src/content/docs/docs/cli/format.mdx @@ -161,9 +161,6 @@ still fixed. When a difference cannot be attributed, each file's edits are check file whose SQL cannot be expanded is skipped with a `format-fix-skipped` warning that gives the reason, and the run continues. A cycle or the 128-pass limit fails without writing any edit. -`--fix` attempts the fixes SQLClean marks safe and never the unsafe ones; every edit is still -checked by the compiler as above. - | Rule | Decision | Reason | | --- | --- | --- | | SQLCLEAN001 NULL comparison | Refused | Replacing `= NULL` with `IS NULL` changes three-valued logic. | @@ -172,13 +169,11 @@ checked by the compiler as above. | SQLCLEAN005 unused CTE | Applied | Unreferenced `SELECT` CTEs contribute no rows. Mutation statements and commented CTEs are excluded. | | SQLCLEAN006 redundant DISTINCT | Applied | The projected and grouping expressions are proved to coincide. | | SQLCLEAN008 bare UNION | Applied | `UNION DISTINCT` states the existing duplicate elimination, in dialects that support it. | -| SQLCLEAN012 redundant ELSE NULL | Applied | A `CASE` without `ELSE` already yields `NULL`. | -| SQLCLEAN013 parenthesized DISTINCT | Refused when unproved | Removing parentheses can change operator precedence, so such an edit is refused with the reason. | -| SQLCLEAN015 empty statement | Applied | A repeated terminator adds no statement. | +| SQLCLEAN013 parenthesized DISTINCT | Refused | Removing parentheses can change operator precedence. | | SQLCLEAN017 constant row count | Applied | `COUNT(1)` and `COUNT(*)` both count every row. | -| SQLCLEAN023 unused table alias | Applied | An alias no expression references is removed. | +| SQLCLEAN023 unused table alias | Refused | Not yet reviewed as meaning-preserving. | | SQLCLEAN025 implicit inner join | Applied | A bare `JOIN ... ON` is an inner join. | -| SQLCLEAN030 boolean CASE simplification | Applied when proved | The `COALESCE` rewrite is kept only when recompiling shows the same output types and nullability; otherwise the file reports `format-fix-verification-failed`. | +| SQLCLEAN030 boolean CASE simplification | Refused | The `COALESCE` rewrite is not proved portable to every supported dialect. | | SQLCLEAN042 unused CTE output | Applied when row-neutral | Only a column reference or deterministic scalar expression is removed, from a CTE without `DISTINCT`, `DISTINCT ON`, `GROUP BY ALL`, `ORDER BY ALL`, an ungrouped aggregate, or positional `GROUP BY`/`ORDER BY`. Other removals could change the rows and are refused with the reason. | `--fix` also rewrites a header description that escapes double quotes as `\"` or doubles them as From 27c5768d5dd998c6f6472097df205cca7797b274 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:34:45 +0000 Subject: [PATCH 4/8] test(compile): split perf guards into default Rules and compiler-only Default SQL Rules lint every compiled model, a measured 1.14x to 1.19x cold wall cost over select = []. Compiler-only cases keep the previous budgets and the 1.10 same-runner ratio; default-Rules cases scale the cold budgets by the measured cost. Also pin sqlclean to the PIVOT USING fix and keep two fixtures' derived tables out of SQLCLEAN039. --- .github/workflows/verify.yml | 16 +++++++ Cargo.lock | 2 +- Cargo.toml | 2 +- .../_helpers/measure.py | 9 +++- .../_helpers/report.py | 3 ++ .../compile_performance_ratio/main/compare.py | 46 ++++++++++++++++--- .../analysis_corpus/analysis_session.py | 11 ++++- .../references/docs/concepts/rules.md | 5 ++ .../clickhouse/test_clickhouse_nullability.py | 4 +- .../main/compile/COMPILE_BENCHMARKS.md | 11 +++++ .../cli/commands/main/compile/_test_types.py | 4 ++ .../cli/commands/main/compile/helpers.py | 15 ++++-- ...fresh_process_compile_cache_performance.py | 37 +++++++++++++++ .../test_fresh_process_compile_performance.py | 35 ++++++++++++++ .../src/content/docs/docs/concepts/rules.mdx | 5 ++ 15 files changed, 187 insertions(+), 18 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 3f51d3fbb9..578dba2637 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -671,8 +671,18 @@ jobs: max_ratio: "1.10" edit_max_ratio: "1.20" edit_max_cpu_ratio: "1.75" + # Base predates default SQL Rules, so the compiler-only head keeps the 1.10 limit. - kind: fresh + label: fresh compiler-only + compiler_only: true max_ratio: "1.10" + # Default SQL Rules add a measured cold cost (1.14x wall, 1.28x CPU over select = [] + # on the same machine), so only the cold limits scale by it; warm and edit keep 1.10. + - kind: fresh + label: fresh default Rules + max_ratio: "1.10" + cold_max_ratio: "1.25" + cold_max_cpu_ratio: "1.41" - kind: dense label: contracts CPU vs base gate_phase: contracts_cpu_ms @@ -746,12 +756,18 @@ jobs: env: KIND: ${{ matrix.kind }} MAX_RATIO: ${{ matrix.max_ratio }} + COLD_MAX_RATIO: ${{ matrix.cold_max_ratio }} + COLD_MAX_CPU_RATIO: ${{ matrix.cold_max_cpu_ratio }} EDIT_MAX_RATIO: ${{ matrix.edit_max_ratio }} EDIT_MAX_CPU_RATIO: ${{ matrix.edit_max_cpu_ratio }} + COMPILER_ONLY: ${{ matrix.compiler_only && '--compiler-only' || '' }} run: | uv run python -m scripts.compare_compile_performance \ --kind "${KIND}" \ --modes cold warm edit \ + ${COMPILER_ONLY:+"$COMPILER_ONLY"} \ + ${COLD_MAX_RATIO:+--cold-max-ratio "$COLD_MAX_RATIO"} \ + ${COLD_MAX_CPU_RATIO:+--cold-max-cpu-ratio "$COLD_MAX_CPU_RATIO"} \ ${EDIT_MAX_RATIO:+--edit-max-ratio "$EDIT_MAX_RATIO"} \ ${EDIT_MAX_CPU_RATIO:+--edit-max-cpu-ratio "$EDIT_MAX_CPU_RATIO"} \ --max-ratio "${MAX_RATIO}" \ diff --git a/Cargo.lock b/Cargo.lock index 005477f929..31ca85daa3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1501,7 +1501,7 @@ dependencies = [ [[package]] name = "sqlclean" version = "0.4.1" -source = "git+https://github.com/sqlstack-io/sqlclean.git?rev=65eb3f32683a6a6fc84ca3b8637d2cabda750425#65eb3f32683a6a6fc84ca3b8637d2cabda750425" +source = "git+https://github.com/sqlstack-io/sqlclean.git?rev=aed227a01101d434a2e43098468400989b60537f#aed227a01101d434a2e43098468400989b60537f" dependencies = [ "polyglot-sql", "polyglot-sql-function-catalogs", diff --git a/Cargo.toml b/Cargo.toml index 27c2dfe453..1a07e98c98 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -140,7 +140,7 @@ sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } sqlbuild-state = { path = "crates/sqlbuild-state" } # SQL lint Rules, quality Rules, the formatter and the canonical token stream. -sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", rev = "65eb3f32683a6a6fc84ca3b8637d2cabda750425" } +sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", rev = "aed227a01101d434a2e43098468400989b60537f" } stacker = "0.1" syn = { version = "2", features = ["full", "visit", "extra-traits"] } tempfile = "3" diff --git a/scripts/compile_performance_ratio/_helpers/measure.py b/scripts/compile_performance_ratio/_helpers/measure.py index 05add8ed2a..30b05481ac 100644 --- a/scripts/compile_performance_ratio/_helpers/measure.py +++ b/scripts/compile_performance_ratio/_helpers/measure.py @@ -52,10 +52,14 @@ ) -def write_benchmark_project(*, kind: str, project_dir: Path, models: int) -> None: - """Write the dense or fresh benchmark project used by both builds.""" +def write_benchmark_project( + *, kind: str, project_dir: Path, models: int, compiler_only: bool = False +) -> None: + """Write the benchmark project; a fresh `compiler_only` project sets `select = []`.""" if kind == DENSE_KIND: + if compiler_only: + raise CompileComparisonError("compiler-only comparisons support the fresh project only") write_dense_compile_project(project_dir=project_dir, model_count=models) return write_semantic_compile_project( @@ -67,6 +71,7 @@ def write_benchmark_project(*, kind: str, project_dir: Path, models: int) -> Non macro_count=max(1, round(models * FRESH_MACRO_SHARE)), test_count=max(1, round(models * FRESH_TEST_SHARE)), audit_count=max(1, round(models * FRESH_AUDIT_SHARE)), + compiler_only=compiler_only, ) diff --git a/scripts/compile_performance_ratio/_helpers/report.py b/scripts/compile_performance_ratio/_helpers/report.py index 25b26a6851..6eb5fd03f8 100644 --- a/scripts/compile_performance_ratio/_helpers/report.py +++ b/scripts/compile_performance_ratio/_helpers/report.py @@ -21,6 +21,7 @@ def comparison_markdown( mode_cpu_max_ratios: Mapping[str, float] | None = None, gate_phases: tuple[str, ...] = (), compile_args: tuple[str, ...] = (), + compiler_only: bool = False, ) -> str: """Summarize medians, ratios and per-phase timings of every mode as Markdown tables.""" @@ -34,6 +35,8 @@ def comparison_markdown( lines.extend((f"Gated phases: {', '.join(gate_phases)}.", "")) if compile_args: lines.extend((f"Compile arguments: `{' '.join(compile_args)}`.", "")) + if compiler_only: + lines.extend(("Head project sets `select = []`: compiler only, no default SQL Rules.", "")) for comparison in comparisons: lines.extend( ( diff --git a/scripts/compile_performance_ratio/main/compare.py b/scripts/compile_performance_ratio/main/compare.py index 6bb3d0da36..5add67c360 100644 --- a/scripts/compile_performance_ratio/main/compare.py +++ b/scripts/compile_performance_ratio/main/compare.py @@ -16,6 +16,7 @@ from scripts.compile_performance_ratio._helpers.report import append_summary, comparison_markdown from scripts.compile_performance_ratio._helpers.verdict import phase_failures, ratio_failures from scripts.compile_performance_ratio.constants import ( + COLD_MODE, COMPILE_MODES, DEFAULT_MAX_RATIO, DEFAULT_RUNS, @@ -41,7 +42,12 @@ def compare_compile_performance(argv: list[str] | None = None) -> int: Path(root) / "base" / project_name if per_side_projects else head_project_dir ) print(f"Generating {args.kind} project with {args.models} models", file=sys.stderr) - write_benchmark_project(kind=args.kind, project_dir=head_project_dir, models=args.models) + write_benchmark_project( + kind=args.kind, + project_dir=head_project_dir, + models=args.models, + compiler_only=args.compiler_only, + ) if args.base_root is not None: print(f"Generating the base project with {args.base_root}'s generator", file=sys.stderr) write_base_benchmark_project( @@ -63,13 +69,20 @@ def compare_compile_performance(argv: list[str] | None = None) -> int: modes=modes, compile_args=tuple(args.compile_args), ) - mode_max_ratios: dict[str, float] = ( - {} if args.edit_max_ratio is None else {EDIT_MODE: args.edit_max_ratio} - ) + mode_max_ratios: dict[str, float] = { + mode: limit + for mode, limit in ((COLD_MODE, args.cold_max_ratio), (EDIT_MODE, args.edit_max_ratio)) + if limit is not None + } gate_phases: tuple[str, ...] = tuple(dict.fromkeys(args.gate_phases)) - mode_cpu_max_ratios: dict[str, float] = ( - {} if args.edit_max_cpu_ratio is None else {EDIT_MODE: args.edit_max_cpu_ratio} - ) + mode_cpu_max_ratios: dict[str, float] = { + mode: limit + for mode, limit in ( + (COLD_MODE, args.cold_max_cpu_ratio), + (EDIT_MODE, args.edit_max_cpu_ratio), + ) + if limit is not None + } failures: tuple[str, ...] = ( phase_failures( comparisons=comparisons, @@ -98,6 +111,7 @@ def compare_compile_performance(argv: list[str] | None = None) -> int: mode_cpu_max_ratios=mode_cpu_max_ratios, gate_phases=gate_phases, compile_args=tuple(args.compile_args), + compiler_only=args.compiler_only, ) print(markdown) _ = append_summary(path=Path(summary_value) if summary_value else None, markdown=markdown) @@ -131,6 +145,18 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: ) parser.add_argument("--runs", type=int, default=DEFAULT_RUNS) parser.add_argument("--max-ratio", type=float, default=DEFAULT_MAX_RATIO) + parser.add_argument( + "--cold-max-ratio", + type=float, + default=None, + help="Ratio limit of the cold mode only; defaults to --max-ratio.", + ) + parser.add_argument( + "--cold-max-cpu-ratio", + type=float, + default=None, + help="CPU ratio limit for cold mode only; defaults to the mode's ratio limit.", + ) parser.add_argument( "--edit-max-cpu-ratio", type=float, @@ -156,6 +182,12 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: default=[], help="Extra `sqb compile` argument for both builds, e.g. `--compile-arg=--select=*`.", ) + parser.add_argument( + "--compiler-only", + action="store_true", + help="Write the head project with `select = []`, so both builds run the compiler without " + "default SQL Rules; the base predates default Rules.", + ) parser.add_argument( "--gate-phase", dest="gate_phases", diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/analysis_session.py b/scripts/compiler_differential/_helpers/analysis_corpus/analysis_session.py index c769c944bd..a6ee164709 100644 --- a/scripts/compiler_differential/_helpers/analysis_corpus/analysis_session.py +++ b/scripts/compiler_differential/_helpers/analysis_corpus/analysis_session.py @@ -1,7 +1,12 @@ """Failure cases owned by the native model analysis session lane.""" from scripts.compiler_differential._helpers.corpus.case_builder import failure_case -from scripts.compiler_differential.constants import FAILURE_MART_PATH, FAILURE_STAGING_PATH +from scripts.compiler_differential.constants import ( + FAILURE_BASE_CONFIG, + FAILURE_CONFIG_PATH, + FAILURE_MART_PATH, + FAILURE_STAGING_PATH, +) from scripts.compiler_differential.models import FailureCase _ORDERS: str = '__ref("stg_orders")' @@ -70,6 +75,10 @@ def analysis_session_failure_cases() -> tuple[FailureCase, ...]: name="analysis-session-enriched-subquery-type-mismatch", expected_code="K002", files={ + FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + + '\n[[rules.rule_ignores]]\nrules = ["SQLCLEAN039"]\n' + f'paths = ["{FAILURE_MART_PATH}"]\n' + 'reason = "The case checks type facts through a derived table."\n', FAILURE_MART_PATH: 'MODEL (\n description "Order totals per customer",\n' " contract enforced,\n" " columns (\n customer_id (type INTEGER),\n amount (type VARCHAR),\n ),\n" diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md index daeba27ce3..09d9a86da7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md @@ -228,6 +228,11 @@ rather than every invocation. Project Rules run once over the complete project v explicit choice for genuinely project-wide requirements, with broader invalidation. SQLBuild tracks the inputs each Rule depends on so cached findings are reused only while those inputs remain valid. +The default SQL Rules lint every compiled model, so a cold compile without a cache costs more than +the compiler alone: about 14% more wall time and 28% more CPU on a generated 3,000-model project. +Warm and one-model edit compiles reuse cached findings and cost the same. Set `select = []` to +measure or run the compiler without them. + Required CI exercises custom Rule packs over generated projects, checking compilation time, memory, and cache invalidation. Separate 20- and 100-Rule profiles measure the cost of custom validation at 5,000-model scale. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_nullability.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_nullability.py index f94a53c2d5..9bd05ca830 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_nullability.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_nullability.py @@ -100,10 +100,10 @@ ");\n\n" ), edited_select=( + "WITH r AS (SELECT toInt64(1) AS order_id, 'one' AS note)\n" "SELECT o.order_id AS order_id, o.amount AS amount, r.note AS note\n" 'FROM __source("raw_orders") AS o\n' - "LEFT JOIN (SELECT toInt64(1) AS order_id, 'one' AS note) AS r " - "ON o.order_id = r.order_id\n" + "LEFT JOIN r ON o.order_id = r.order_id\n" ), expected_return_code=1, expected_output_fragment="A713", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md index eba4f94aa9..0e2aeb4aa2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md @@ -98,6 +98,17 @@ same runner, three alternating runs per mode, and the medians of wall and CPU ti - warm: an unchanged `sqb compile --json` after one untimed run that writes the cache; - edit: `sqb compile --json` after the same one-model edit as the release comparison below. +The base branch predates the default SQL Rules, so the fresh comparison runs twice. The +compiler-only job writes the head project with `select = []` and keeps the 1.10x limit for every +mode. The default Rules job allows 1.25x cold wall and 1.41x cold CPU: the 1.10x limit times the +measured cold cost of the default Rules over `select = []` on the same machine (1.14x wall, 1.28x +CPU, five paired runs). Its warm and edit modes keep 1.10x. + +The fresh-process 3,000- and 5,000-model compile guards and the 3,000-model cache guard follow the +same split (approved 2026-11-01): `compiler_only` cases keep the previous wall budgets, and the +default Rules cases scale them by the measured cold cost (1.14x and 1.16x fresh, 1.19x for the cache +guard's cold and project-config runs). Larger cases already fit their budgets and are unchanged. + The warm and edit modes reuse the cold project. A warm run that misses the analysis cache, or an edit run that does not, stops the comparison because it would measure the wrong path. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index c7d2892773..27618ecf2a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -172,6 +172,8 @@ class FreshProcessCompilePerformanceGuardTestCase: expected_max_wall_seconds: float expected_max_rss_bytes: int expected_semantic_fingerprint: str + # `select = []`: the compiler alone, without the default SQL Rules. + compiler_only: bool = False @dataclass(frozen=True) @@ -200,6 +202,8 @@ class FreshProcessCompileCachePerformanceGuardTestCase: macro_call_interval: int scoped_macros: bool expected_macro_edit_misses: int + # `select = []`: the compiler alone, without the default SQL Rules. + compiler_only: bool = False @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 34e0792349..982be9e39d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -522,6 +522,7 @@ def run_fresh_process_semantic_compile_benchmark( test_count: int, audit_count: int, expected_max_wall_seconds: float, + compiler_only: bool = False, ) -> FreshProcessCompileBenchmarkResult: """Compile one clean semantic fixture in a fresh process without compiler caches.""" @@ -534,6 +535,7 @@ def run_fresh_process_semantic_compile_benchmark( macro_count=macro_count, test_count=test_count, audit_count=audit_count, + compiler_only=compiler_only, ) target_dir: Path = project_dir / "target" assert not target_dir.exists() @@ -562,6 +564,7 @@ def run_fresh_process_compile_cache_benchmark( expected_edit_max_wall_seconds: float, macro_call_interval: int, scoped_macros: bool, + compiler_only: bool = False, ) -> FreshProcessCompileCacheBenchmarkResult: """Measure exact and incremental cache reuse across independent CLI processes.""" @@ -576,6 +579,7 @@ def run_fresh_process_compile_cache_benchmark( audit_count=audit_count, macro_call_interval=macro_call_interval, scoped_macros=scoped_macros, + compiler_only=compiler_only, ) target_dir: Path = project_dir / "target" assert not target_dir.exists() @@ -1583,14 +1587,16 @@ def write_semantic_compile_project( scoped_macros: bool = False, shared_diamond_layers: int = 0, shared_diamond_width: int = 0, + compiler_only: bool = False, ) -> None: """Write a neutral project with broad resources and dense SQL semantics. A positive `shared_diamond_layers` adds the shared-dependency lattice from `write_shared_diamond_models`; the default leaves the compile guard fixtures unchanged. + `compiler_only` sets `select = []` so a guard measures the compiler without default Rules. """ - _layered_write_project_config(project_dir=project_dir) + _layered_write_project_config(project_dir=project_dir, compiler_only=compiler_only) _layered_write_sources(project_dir=project_dir, source_count=source_count) _layered_write_seeds(project_dir=project_dir, seed_count=seed_count) _layered_write_functions(project_dir=project_dir, function_count=function_count) @@ -1627,10 +1633,11 @@ def write_semantic_compile_project( ) -def _layered_write_project_config(*, project_dir: Path) -> None: +def _layered_write_project_config(*, project_dir: Path, compiler_only: bool = False) -> None: project_dir.mkdir(parents=True) + rules_selection: str = {True: "[rules]\nselect = []\n\n", False: ""}[compiler_only] (project_dir / "sqlbuild_project.toml").write_text( - """name = "layered_production_performance_guard" + f"""name = "layered_production_performance_guard" adapter = "duckdb" default_target = "dev" @@ -1655,7 +1662,7 @@ def _layered_write_project_config(*, project_dir: Path) -> None: [path_defaults."mart"] materialized = "table" -[[rules.rule_ignores]] +{rules_selection}[[rules.rule_ignores]] rules = ["SQLCLEAN039"] paths = ["models/**"] reason = "The workload mixes nested query shapes on purpose to cover their compile path." diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py index 62ef943bf5..e1af088aa8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py @@ -51,6 +51,41 @@ macro_count=500, test_count=2_945, audit_count=5_056, + # Default SQL Rules: 9.0 s scaled by the measured 1.19x cold cost over select = []. + expected_cold_max_wall_seconds=10.75, + expected_warm_max_wall_seconds=4.5, + expected_edit_max_wall_seconds=4.5, + expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, + expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, + expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO, + expected_max_cache_write_wall_overhead_ratio=_MAX_CACHE_WRITE_WALL_OVERHEAD_RATIO, + expected_max_rss_bytes=1152 * _MIB, + expected_max_cache_bytes=96 * _MIB, + expected_cold_fingerprint=( + "1914b788abcb452422eaf6c3a2c240ff390bab9530217285a5ca93334db7b6c3" + ), + expected_leaf_edit_fingerprint=( + "3a9480f952f11e16177274185e8bfda7eb0e65953a8ed1c8b21ad3cdb7d0b07d" + ), + expected_macro_edit_fingerprint=( + "552931889359ce882b18d4fb11fdc87341e60a8a4d2b26ad0f1635138c8b9ad8" + ), + expected_project_config_fingerprint=( + "6bf4f742469eee2be7eb5bbf2411564c924011fea0bd43becd80db4027022dac" + ), + macro_call_interval=6, + scoped_macros=True, + expected_macro_edit_misses=1, + ), + FreshProcessCompileCachePerformanceGuardTestCase( + description="models_3000_fresh_process_compiler_only_compile_cache_stays_incremental", + model_count=3_000, + source_count=713, + seed_count=141, + function_count=71, + macro_count=500, + test_count=2_945, + audit_count=5_056, expected_cold_max_wall_seconds=9.0, expected_warm_max_wall_seconds=4.5, expected_edit_max_wall_seconds=4.5, @@ -75,6 +110,7 @@ macro_call_interval=6, scoped_macros=True, expected_macro_edit_misses=1, + compiler_only=True, ), FreshProcessCompileCachePerformanceGuardTestCase( description="models_5000_fresh_process_compile_cache_stays_incremental", @@ -167,6 +203,7 @@ def test_given_semantic_project_when_compiling_across_processes_then_cache_is_in expected_edit_max_wall_seconds=test_case.expected_edit_max_wall_seconds, macro_call_interval=test_case.macro_call_interval, scoped_macros=test_case.scoped_macros, + compiler_only=test_case.compiler_only, ) measurements: dict[str, FreshProcessCompileBenchmarkResult] = { "cache_disabled": result.cache_disabled, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py index c1bd3fc98c..99db659bff 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py @@ -37,11 +37,28 @@ macro_count=37, test_count=2_945, audit_count=5_056, + # Default SQL Rules: 8.0 s scaled by the measured 1.14x cold cost over select = []. + expected_max_wall_seconds=9.2, + expected_max_rss_bytes=1088 * _MIB, + expected_semantic_fingerprint=( + "d935821df54043fabc3a6911c1926578178c9b744f2a589cbef99a10d5207543" + ), + ), + FreshProcessCompilePerformanceGuardTestCase( + description="models_3000_fresh_process_compiler_only_compile_stays_within_budget", + model_count=3_000, + source_count=713, + seed_count=141, + function_count=71, + macro_count=37, + test_count=2_945, + audit_count=5_056, expected_max_wall_seconds=8.0, expected_max_rss_bytes=1088 * _MIB, expected_semantic_fingerprint=( "d935821df54043fabc3a6911c1926578178c9b744f2a589cbef99a10d5207543" ), + compiler_only=True, ), FreshProcessCompilePerformanceGuardTestCase( description="models_5000_fresh_process_semantic_compile_stays_within_budget", @@ -52,11 +69,28 @@ macro_count=61, test_count=4_908, audit_count=8_427, + # Default SQL Rules: 12.5 s scaled by the measured 1.16x cold cost over select = []. + expected_max_wall_seconds=14.5, + expected_max_rss_bytes=1664 * _MIB, + expected_semantic_fingerprint=( + "874cbfd473846c6a335ec2f79cb6b7b0b77e8e8d58652815fad1e9d59d423c46" + ), + ), + FreshProcessCompilePerformanceGuardTestCase( + description="models_5000_fresh_process_compiler_only_compile_stays_within_budget", + model_count=5_000, + source_count=1_189, + seed_count=236, + function_count=118, + macro_count=61, + test_count=4_908, + audit_count=8_427, expected_max_wall_seconds=12.5, expected_max_rss_bytes=1664 * _MIB, expected_semantic_fingerprint=( "874cbfd473846c6a335ec2f79cb6b7b0b77e8e8d58652815fad1e9d59d423c46" ), + compiler_only=True, ), FreshProcessCompilePerformanceGuardTestCase( description="models_10000_fresh_process_semantic_compile_stays_within_budget", @@ -91,6 +125,7 @@ def test_given_scaled_semantic_project_when_compiling_fresh_then_preserves_resou test_count=test_case.test_count, audit_count=test_case.audit_count, expected_max_wall_seconds=test_case.expected_max_wall_seconds, + compiler_only=test_case.compiler_only, ) _LOGGER.info( "fresh-process semantic compile models=%d wall=%.3fs peak_rss_bytes=%d " diff --git a/website/src/content/docs/docs/concepts/rules.mdx b/website/src/content/docs/docs/concepts/rules.mdx index 5bec227174..8f761a9ee0 100644 --- a/website/src/content/docs/docs/concepts/rules.mdx +++ b/website/src/content/docs/docs/concepts/rules.mdx @@ -218,6 +218,11 @@ rather than every invocation. Project Rules run once over the complete project v explicit choice for genuinely project-wide requirements, with broader invalidation. SQLBuild tracks the inputs each Rule depends on so cached findings are reused only while those inputs remain valid. +The default SQL Rules lint every compiled model, so a cold compile without a cache costs more than +the compiler alone: about 14% more wall time and 28% more CPU on a generated 3,000-model project. +Warm and one-model edit compiles reuse cached findings and cost the same. Set `select = []` to +measure or run the compiler without them. + Required CI exercises custom Rule packs over generated projects, checking compilation time, memory, and cache invalidation. Separate 20- and 100-Rule profiles measure the cost of custom validation at 5,000-model scale. From 6a71468fa2cefc4baca79fcfc3fd94c2d43b92d0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 13:10:09 +0000 Subject: [PATCH 5/8] test(compile): keep compiler-only ratios valid after default Rules land Pass compiler_only to the base generator when the base supports it, so the compiler-only job keeps both sides without Rules once main runs them. Mark the default-Rules cold allowance as temporary, reject --compiler-only with the dense project at parse time, assert that default-Rules guard cases run Rules, and pin sqlclean to the v0.4.2 release commit. --- .github/workflows/verify.yml | 3 ++- Cargo.lock | 4 ++-- Cargo.toml | 2 +- scripts/compile_performance_ratio/_helpers/measure.py | 10 ++++++++-- scripts/compile_performance_ratio/_helpers/report.py | 7 ++++++- scripts/compile_performance_ratio/constants.py | 6 ++++-- scripts/compile_performance_ratio/main/compare.py | 11 ++++++++--- .../cli/commands/main/compile/COMPILE_BENCHMARKS.md | 8 ++++---- .../test_fresh_process_compile_cache_performance.py | 5 +++++ .../compile/test_fresh_process_compile_performance.py | 3 +++ 10 files changed, 43 insertions(+), 16 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 578dba2637..71b1576ecd 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -671,11 +671,12 @@ jobs: max_ratio: "1.10" edit_max_ratio: "1.20" edit_max_cpu_ratio: "1.75" - # Base predates default SQL Rules, so the compiler-only head keeps the 1.10 limit. + # Both sides set select = [] (an older base has no default Rules), so 1.10 holds. - kind: fresh label: fresh compiler-only compiler_only: true max_ratio: "1.10" + # Temporary migration allowance; restore 1.10 once the base also runs default Rules. # Default SQL Rules add a measured cold cost (1.14x wall, 1.28x CPU over select = [] # on the same machine), so only the cold limits scale by it; warm and edit keep 1.10. - kind: fresh diff --git a/Cargo.lock b/Cargo.lock index 31ca85daa3..9e4b8cb6a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1500,8 +1500,8 @@ dependencies = [ [[package]] name = "sqlclean" -version = "0.4.1" -source = "git+https://github.com/sqlstack-io/sqlclean.git?rev=aed227a01101d434a2e43098468400989b60537f#aed227a01101d434a2e43098468400989b60537f" +version = "0.4.2" +source = "git+https://github.com/sqlstack-io/sqlclean.git?rev=571e40a746b3d8d20d6118620f05428eca45243c#571e40a746b3d8d20d6118620f05428eca45243c" dependencies = [ "polyglot-sql", "polyglot-sql-function-catalogs", diff --git a/Cargo.toml b/Cargo.toml index 1a07e98c98..2caa060782 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -140,7 +140,7 @@ sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } sqlbuild-state = { path = "crates/sqlbuild-state" } # SQL lint Rules, quality Rules, the formatter and the canonical token stream. -sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", rev = "aed227a01101d434a2e43098468400989b60537f" } +sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", rev = "571e40a746b3d8d20d6118620f05428eca45243c" } stacker = "0.1" syn = { version = "2", features = ["full", "visit", "extra-traits"] } tempfile = "3" diff --git a/scripts/compile_performance_ratio/_helpers/measure.py b/scripts/compile_performance_ratio/_helpers/measure.py index 30b05481ac..b9db0aded3 100644 --- a/scripts/compile_performance_ratio/_helpers/measure.py +++ b/scripts/compile_performance_ratio/_helpers/measure.py @@ -100,7 +100,13 @@ def run_checkout_generator( def write_base_benchmark_project( - *, base_root: Path, python: Path, kind: str, project_dir: Path, models: int + *, + base_root: Path, + python: Path, + kind: str, + project_dir: Path, + models: int, + compiler_only: bool = False, ) -> None: """Write the base project with the base checkout's own generator, run by `python`.""" @@ -108,7 +114,7 @@ def write_base_benchmark_project( checkout=base_root, python=python, entry=BASE_GENERATOR_ENTRY, - arguments=(kind, str(project_dir), str(models)), + arguments=(kind, str(project_dir), str(models), str(int(compiler_only))), environment=_compile_environment(), library_paths=(), ) diff --git a/scripts/compile_performance_ratio/_helpers/report.py b/scripts/compile_performance_ratio/_helpers/report.py index 6eb5fd03f8..e9fbe0dbda 100644 --- a/scripts/compile_performance_ratio/_helpers/report.py +++ b/scripts/compile_performance_ratio/_helpers/report.py @@ -36,7 +36,12 @@ def comparison_markdown( if compile_args: lines.extend((f"Compile arguments: `{' '.join(compile_args)}`.", "")) if compiler_only: - lines.extend(("Head project sets `select = []`: compiler only, no default SQL Rules.", "")) + lines.extend( + ( + "Projects set `select = []` where supported: compiler only, no default SQL Rules.", + "", + ) + ) for comparison in comparisons: lines.extend( ( diff --git a/scripts/compile_performance_ratio/constants.py b/scripts/compile_performance_ratio/constants.py index 550ce619f3..b2f1d6e6a1 100644 --- a/scripts/compile_performance_ratio/constants.py +++ b/scripts/compile_performance_ratio/constants.py @@ -36,10 +36,12 @@ EXCLUDED_ENVIRONMENT_PREFIX: str = "DBT_" COMPILE_ENTRY: str = "import sys; from sqlbuild.cli.entry.main.entry import main; sys.exit(main())" BASE_GENERATOR_ENTRY: str = ( - "import sys; from pathlib import Path; " + "import inspect, sys; from pathlib import Path; " "from scripts.compile_performance_ratio._helpers.measure import write_benchmark_project; " + "options = {'compiler_only': sys.argv[4] == '1'} " + "if 'compiler_only' in inspect.signature(write_benchmark_project).parameters else {}; " "write_benchmark_project(" - "kind=sys.argv[1], project_dir=Path(sys.argv[2]), models=int(sys.argv[3]))" + "kind=sys.argv[1], project_dir=Path(sys.argv[2]), models=int(sys.argv[3]), **options)" ) PYTHONPATH_KEY: str = "PYTHONPATH" PHASE_NOISE_FLOOR_MS: float = 50.0 diff --git a/scripts/compile_performance_ratio/main/compare.py b/scripts/compile_performance_ratio/main/compare.py index 5add67c360..d18427fec7 100644 --- a/scripts/compile_performance_ratio/main/compare.py +++ b/scripts/compile_performance_ratio/main/compare.py @@ -20,6 +20,7 @@ COMPILE_MODES, DEFAULT_MAX_RATIO, DEFAULT_RUNS, + DENSE_KIND, EDIT_MODE, NOISE_FLOOR_SECONDS, PHASE_NOISE_FLOOR_MS, @@ -56,6 +57,7 @@ def compare_compile_performance(argv: list[str] | None = None) -> int: kind=args.kind, project_dir=base_project_dir, models=args.models, + compiler_only=args.compiler_only, ) modes: tuple[str, ...] = tuple(dict.fromkeys(args.modes)) comparisons: tuple[CompileComparison, ...] = compare_builds( @@ -185,8 +187,8 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: parser.add_argument( "--compiler-only", action="store_true", - help="Write the head project with `select = []`, so both builds run the compiler without " - "default SQL Rules; the base predates default Rules.", + help="Write both projects with `select = []`, so both builds run the compiler without " + "default SQL Rules (fresh projects only).", ) parser.add_argument( "--gate-phase", @@ -202,4 +204,7 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: default=PHASE_NOISE_FLOOR_MS, help="Milliseconds of slack added to a gated phase's ratio allowance.", ) - return parser.parse_args(argv) + args: argparse.Namespace = parser.parse_args(argv) + if args.compiler_only and args.kind == DENSE_KIND: + parser.error("--compiler-only supports --kind fresh only") + return args diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md index 0e2aeb4aa2..d913908f08 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md @@ -98,11 +98,11 @@ same runner, three alternating runs per mode, and the medians of wall and CPU ti - warm: an unchanged `sqb compile --json` after one untimed run that writes the cache; - edit: `sqb compile --json` after the same one-model edit as the release comparison below. -The base branch predates the default SQL Rules, so the fresh comparison runs twice. The -compiler-only job writes the head project with `select = []` and keeps the 1.10x limit for every -mode. The default Rules job allows 1.25x cold wall and 1.41x cold CPU: the 1.10x limit times the +The fresh comparison runs twice. The compiler-only job writes both projects with `select = []` +(a base older than that option has no default SQL Rules) and keeps the 1.10x limit for every mode. The default Rules job allows 1.25x cold wall and 1.41x cold CPU: the 1.10x limit times the measured cold cost of the default Rules over `select = []` on the same machine (1.14x wall, 1.28x -CPU, five paired runs). Its warm and edit modes keep 1.10x. +CPU, five paired runs). Its warm and edit modes keep 1.10x. The cold allowance is temporary: once +the base also runs default Rules, it returns to 1.10x. The fresh-process 3,000- and 5,000-model compile guards and the 3,000-model cache guard follow the same split (approved 2026-11-01): `compiler_only` cases keep the previous wall budgets, and the diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py index e1af088aa8..11986a17f9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py @@ -4,6 +4,7 @@ import logging from pathlib import Path +from typing import cast import pytest @@ -271,6 +272,10 @@ def test_given_semantic_project_when_compiling_across_processes_then_cache_is_in assert result.cache_bytes < test_case.expected_max_cache_bytes assert result.cache_disabled.semantic_fingerprint == test_case.expected_cold_fingerprint + cold_timings: dict[str, int] = cast( + dict[str, int], result.cache_disabled.payload["compile_timings"] + ) + assert (cold_timings["rule_cache_misses"] > 0) is not test_case.compiler_only assert result.cold.semantic_fingerprint == test_case.expected_cold_fingerprint assert result.warm.semantic_fingerprint == test_case.expected_cold_fingerprint assert result.leaf_edit.semantic_fingerprint == test_case.expected_leaf_edit_fingerprint diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py index 99db659bff..afa13933e8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py @@ -4,6 +4,7 @@ import logging from pathlib import Path +from typing import cast import pytest @@ -155,5 +156,7 @@ def test_given_scaled_semantic_project_when_compiling_fresh_then_preserves_resou "warnings": 0, } assert result.semantic_fingerprint == test_case.expected_semantic_fingerprint + timings: dict[str, int] = cast(dict[str, int], result.payload["compile_timings"]) + assert (timings["rule_cache_misses"] > 0) is not test_case.compiler_only assert result.elapsed_seconds < test_case.expected_max_wall_seconds assert result.peak_rss_bytes < test_case.expected_max_rss_bytes From d7fcf5fe5447a8b4e9e22f0cfc378f14366a9029 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 14:52:24 +0000 Subject: [PATCH 6/8] test(compile): split the 5,000- and 10,000-model guards for default Rules Default SQL Rules cost 1.09x on the 10,000-model compile and 1.20x and 1.25x on the 5,000- and 10,000-model cache guards' cold runs, measured against select = [] on the same machine. Scale those cold budgets and keep the previous budgets as compiler-only cases, as for 3,000 models. --- .../main/compile/COMPILE_BENCHMARKS.md | 10 ++- ...fresh_process_compile_cache_performance.py | 73 +++++++++++++++++++ .../test_fresh_process_compile_performance.py | 17 +++++ 3 files changed, 96 insertions(+), 4 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md index d913908f08..6f2a6865ef 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md @@ -104,10 +104,12 @@ measured cold cost of the default Rules over `select = []` on the same machine ( CPU, five paired runs). Its warm and edit modes keep 1.10x. The cold allowance is temporary: once the base also runs default Rules, it returns to 1.10x. -The fresh-process 3,000- and 5,000-model compile guards and the 3,000-model cache guard follow the -same split (approved 2026-11-01): `compiler_only` cases keep the previous wall budgets, and the -default Rules cases scale them by the measured cold cost (1.14x and 1.16x fresh, 1.19x for the cache -guard's cold and project-config runs). Larger cases already fit their budgets and are unchanged. +Every fresh-process compile and cache guard follows the same split (approved 2026-11-01): +`compiler_only` cases keep the previous wall budgets, and the default Rules cases scale the cold +budget by the cold cost measured over `select = []` on the same machine: 1.14x, 1.16x and 1.09x for +the 3,000-, 5,000- and 10,000-model compiles, and 1.19x, 1.20x and 1.25x for the cache guards, whose +cold budget also bounds the cache-disabled and project-config runs. Warm, edit and memory budgets +are unchanged; warm and edit runs measured within 5% of `select = []`. The warm and edit modes reuse the cold project. A warm run that misses the analysis cache, or an edit run that does not, stops the comparison because it would measure the wrong path. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py index 11986a17f9..5d2ec19f50 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py @@ -122,6 +122,41 @@ macro_count=834, test_count=4_908, audit_count=8_427, + # Default SQL Rules: 14.5 s scaled by the measured 1.20x cold cost over select = []. + expected_cold_max_wall_seconds=17.4, + expected_warm_max_wall_seconds=7.0, + expected_edit_max_wall_seconds=7.0, + expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, + expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, + expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO, + expected_max_cache_write_wall_overhead_ratio=_MAX_CACHE_WRITE_WALL_OVERHEAD_RATIO, + expected_max_rss_bytes=1664 * _MIB, + expected_max_cache_bytes=160 * _MIB, + expected_cold_fingerprint=( + "2f1e5dfd29025c25e0dffc6e74f4c2be020a5ef133ecd081f44e0c55c24858b4" + ), + expected_leaf_edit_fingerprint=( + "708f04a565a71b3981c9beba420aff691c8467eb30c5cdedd45c2518d45b31ba" + ), + expected_macro_edit_fingerprint=( + "7dd4d45c2b21f61cbc32d33b0e0cf98721975b9f8948277cc15dc947355cfc25" + ), + expected_project_config_fingerprint=( + "69f4fe914afc955d690e9c5b9074ca69dc58b8b7a0886a421b888385e6b066cb" + ), + macro_call_interval=6, + scoped_macros=True, + expected_macro_edit_misses=1, + ), + FreshProcessCompileCachePerformanceGuardTestCase( + description="models_5000_fresh_process_compiler_only_compile_cache_stays_incremental", + model_count=5_000, + source_count=1_189, + seed_count=236, + function_count=118, + macro_count=834, + test_count=4_908, + audit_count=8_427, expected_cold_max_wall_seconds=14.5, expected_warm_max_wall_seconds=7.0, expected_edit_max_wall_seconds=7.0, @@ -146,6 +181,7 @@ macro_call_interval=6, scoped_macros=True, expected_macro_edit_misses=1, + compiler_only=True, ), FreshProcessCompileCachePerformanceGuardTestCase( description="models_10000_fresh_process_compile_cache_stays_incremental", @@ -156,6 +192,42 @@ macro_count=1_667, test_count=9_816, audit_count=16_855, + # Default SQL Rules: 28.0 s scaled by the measured 1.25x cold cost over select = []. + expected_cold_max_wall_seconds=35.0, + expected_warm_max_wall_seconds=14.0, + expected_edit_max_wall_seconds=13.5, + expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, + expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, + expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO, + expected_max_cache_write_wall_overhead_ratio=_MAX_CACHE_WRITE_WALL_OVERHEAD_RATIO, + # Temporary memory exception (2026-10-10); remove after conversion optimisation. + expected_max_rss_bytes=3008 * _MIB, + expected_max_cache_bytes=320 * _MIB, + expected_cold_fingerprint=( + "ecdd7f2cf961fab739aa806b6d90283916cffad2007e1e2f3ba90a466db6cb75" + ), + expected_leaf_edit_fingerprint=( + "f67b73b56d7ad7677cbfa649e7933e8f294f9dd7c99f3660d7c42e1d01b804ff" + ), + expected_macro_edit_fingerprint=( + "5554615dbc5fe9dbfd7555771167f3380222cb3da468b4313076adecef0ff9c3" + ), + expected_project_config_fingerprint=( + "ebc15f5381746d060005ecc61f477b1b0338640198197f6788f9beab5da88db5" + ), + macro_call_interval=6, + scoped_macros=True, + expected_macro_edit_misses=1, + ), + FreshProcessCompileCachePerformanceGuardTestCase( + description="models_10000_fresh_process_compiler_only_compile_cache_stays_incremental", + model_count=10_000, + source_count=2_377, + seed_count=471, + function_count=236, + macro_count=1_667, + test_count=9_816, + audit_count=16_855, expected_cold_max_wall_seconds=28.0, expected_warm_max_wall_seconds=14.0, expected_edit_max_wall_seconds=13.5, @@ -181,6 +253,7 @@ macro_call_interval=6, scoped_macros=True, expected_macro_edit_misses=1, + compiler_only=True, ), ], ids=lambda case: case.description, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py index afa13933e8..2fea56120f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py @@ -102,11 +102,28 @@ macro_count=123, test_count=9_816, audit_count=16_855, + # Default SQL Rules: 25.5 s scaled by the measured 1.09x cold cost over select = []. + expected_max_wall_seconds=28.0, + expected_max_rss_bytes=int(2.5 * _GIB), + expected_semantic_fingerprint=( + "5ccdabeaabfdfba1ecbae80259dd3c6eb801d683df140b7fb7a249b9555308c6" + ), + ), + FreshProcessCompilePerformanceGuardTestCase( + description="models_10000_fresh_process_compiler_only_compile_stays_within_budget", + model_count=10_000, + source_count=2_377, + seed_count=471, + function_count=236, + macro_count=123, + test_count=9_816, + audit_count=16_855, expected_max_wall_seconds=25.5, expected_max_rss_bytes=int(2.5 * _GIB), expected_semantic_fingerprint=( "5ccdabeaabfdfba1ecbae80259dd3c6eb801d683df140b7fb7a249b9555308c6" ), + compiler_only=True, ), ], ids=lambda case: case.description, From 1a576ec0c38935bd31ad23f65a73f35ae1547c2a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:26:23 +0000 Subject: [PATCH 7/8] test(compile): keep one compiler-only guard and calibrate Rules budgets on CI Replace the compiler-only twin at every guard size with one paired guard: the 3,000-model cache guard runs default Rules and select = [] in the same job and covers cold, warm, edit and project-config edit compiles at the previous budgets. Drop the compiler-only same-runner ratio job and the --compiler-only option of the ratio script. Scale each default Rules budget by the larger of the CI same-job and local alternating medians, excluding pairs whose Rules-independent phases exceed twice their median. Give the project-config edit its own budget, since it reruns every rule. --- .github/workflows/verify.yml | 12 +-- .../_helpers/measure.py | 19 +--- .../_helpers/report.py | 8 -- .../compile_performance_ratio/constants.py | 6 +- .../compile_performance_ratio/main/compare.py | 21 +--- .../main/compile/COMPILE_BENCHMARKS.md | 39 ++++++-- .../cli/commands/main/compile/_test_types.py | 3 +- .../cli/commands/main/compile/helpers.py | 5 +- ...fresh_process_compile_cache_performance.py | 99 ++++--------------- .../test_fresh_process_compile_performance.py | 63 ++---------- 10 files changed, 71 insertions(+), 204 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 71b1576ecd..28bf653589 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -671,14 +671,10 @@ jobs: max_ratio: "1.10" edit_max_ratio: "1.20" edit_max_cpu_ratio: "1.75" - # Both sides set select = [] (an older base has no default Rules), so 1.10 holds. - - kind: fresh - label: fresh compiler-only - compiler_only: true - max_ratio: "1.10" # Temporary migration allowance; restore 1.10 once the base also runs default Rules. - # Default SQL Rules add a measured cold cost (1.14x wall, 1.28x CPU over select = [] - # on the same machine), so only the cold limits scale by it; warm and edit keep 1.10. + # Default SQL Rules add a cold cost over select = [] (1.19x to 1.20x wall, 1.32x to + # 1.33x CPU in this job), so only the cold limits scale; warm and edit keep 1.10. + # The compiler-only cache guard at 3,000 models keeps the select = [] budgets. - kind: fresh label: fresh default Rules max_ratio: "1.10" @@ -761,12 +757,10 @@ jobs: COLD_MAX_CPU_RATIO: ${{ matrix.cold_max_cpu_ratio }} EDIT_MAX_RATIO: ${{ matrix.edit_max_ratio }} EDIT_MAX_CPU_RATIO: ${{ matrix.edit_max_cpu_ratio }} - COMPILER_ONLY: ${{ matrix.compiler_only && '--compiler-only' || '' }} run: | uv run python -m scripts.compare_compile_performance \ --kind "${KIND}" \ --modes cold warm edit \ - ${COMPILER_ONLY:+"$COMPILER_ONLY"} \ ${COLD_MAX_RATIO:+--cold-max-ratio "$COLD_MAX_RATIO"} \ ${COLD_MAX_CPU_RATIO:+--cold-max-cpu-ratio "$COLD_MAX_CPU_RATIO"} \ ${EDIT_MAX_RATIO:+--edit-max-ratio "$EDIT_MAX_RATIO"} \ diff --git a/scripts/compile_performance_ratio/_helpers/measure.py b/scripts/compile_performance_ratio/_helpers/measure.py index b9db0aded3..05add8ed2a 100644 --- a/scripts/compile_performance_ratio/_helpers/measure.py +++ b/scripts/compile_performance_ratio/_helpers/measure.py @@ -52,14 +52,10 @@ ) -def write_benchmark_project( - *, kind: str, project_dir: Path, models: int, compiler_only: bool = False -) -> None: - """Write the benchmark project; a fresh `compiler_only` project sets `select = []`.""" +def write_benchmark_project(*, kind: str, project_dir: Path, models: int) -> None: + """Write the dense or fresh benchmark project used by both builds.""" if kind == DENSE_KIND: - if compiler_only: - raise CompileComparisonError("compiler-only comparisons support the fresh project only") write_dense_compile_project(project_dir=project_dir, model_count=models) return write_semantic_compile_project( @@ -71,7 +67,6 @@ def write_benchmark_project( macro_count=max(1, round(models * FRESH_MACRO_SHARE)), test_count=max(1, round(models * FRESH_TEST_SHARE)), audit_count=max(1, round(models * FRESH_AUDIT_SHARE)), - compiler_only=compiler_only, ) @@ -100,13 +95,7 @@ def run_checkout_generator( def write_base_benchmark_project( - *, - base_root: Path, - python: Path, - kind: str, - project_dir: Path, - models: int, - compiler_only: bool = False, + *, base_root: Path, python: Path, kind: str, project_dir: Path, models: int ) -> None: """Write the base project with the base checkout's own generator, run by `python`.""" @@ -114,7 +103,7 @@ def write_base_benchmark_project( checkout=base_root, python=python, entry=BASE_GENERATOR_ENTRY, - arguments=(kind, str(project_dir), str(models), str(int(compiler_only))), + arguments=(kind, str(project_dir), str(models)), environment=_compile_environment(), library_paths=(), ) diff --git a/scripts/compile_performance_ratio/_helpers/report.py b/scripts/compile_performance_ratio/_helpers/report.py index e9fbe0dbda..25b26a6851 100644 --- a/scripts/compile_performance_ratio/_helpers/report.py +++ b/scripts/compile_performance_ratio/_helpers/report.py @@ -21,7 +21,6 @@ def comparison_markdown( mode_cpu_max_ratios: Mapping[str, float] | None = None, gate_phases: tuple[str, ...] = (), compile_args: tuple[str, ...] = (), - compiler_only: bool = False, ) -> str: """Summarize medians, ratios and per-phase timings of every mode as Markdown tables.""" @@ -35,13 +34,6 @@ def comparison_markdown( lines.extend((f"Gated phases: {', '.join(gate_phases)}.", "")) if compile_args: lines.extend((f"Compile arguments: `{' '.join(compile_args)}`.", "")) - if compiler_only: - lines.extend( - ( - "Projects set `select = []` where supported: compiler only, no default SQL Rules.", - "", - ) - ) for comparison in comparisons: lines.extend( ( diff --git a/scripts/compile_performance_ratio/constants.py b/scripts/compile_performance_ratio/constants.py index b2f1d6e6a1..550ce619f3 100644 --- a/scripts/compile_performance_ratio/constants.py +++ b/scripts/compile_performance_ratio/constants.py @@ -36,12 +36,10 @@ EXCLUDED_ENVIRONMENT_PREFIX: str = "DBT_" COMPILE_ENTRY: str = "import sys; from sqlbuild.cli.entry.main.entry import main; sys.exit(main())" BASE_GENERATOR_ENTRY: str = ( - "import inspect, sys; from pathlib import Path; " + "import sys; from pathlib import Path; " "from scripts.compile_performance_ratio._helpers.measure import write_benchmark_project; " - "options = {'compiler_only': sys.argv[4] == '1'} " - "if 'compiler_only' in inspect.signature(write_benchmark_project).parameters else {}; " "write_benchmark_project(" - "kind=sys.argv[1], project_dir=Path(sys.argv[2]), models=int(sys.argv[3]), **options)" + "kind=sys.argv[1], project_dir=Path(sys.argv[2]), models=int(sys.argv[3]))" ) PYTHONPATH_KEY: str = "PYTHONPATH" PHASE_NOISE_FLOOR_MS: float = 50.0 diff --git a/scripts/compile_performance_ratio/main/compare.py b/scripts/compile_performance_ratio/main/compare.py index d18427fec7..ce2492e0fc 100644 --- a/scripts/compile_performance_ratio/main/compare.py +++ b/scripts/compile_performance_ratio/main/compare.py @@ -20,7 +20,6 @@ COMPILE_MODES, DEFAULT_MAX_RATIO, DEFAULT_RUNS, - DENSE_KIND, EDIT_MODE, NOISE_FLOOR_SECONDS, PHASE_NOISE_FLOOR_MS, @@ -43,12 +42,7 @@ def compare_compile_performance(argv: list[str] | None = None) -> int: Path(root) / "base" / project_name if per_side_projects else head_project_dir ) print(f"Generating {args.kind} project with {args.models} models", file=sys.stderr) - write_benchmark_project( - kind=args.kind, - project_dir=head_project_dir, - models=args.models, - compiler_only=args.compiler_only, - ) + write_benchmark_project(kind=args.kind, project_dir=head_project_dir, models=args.models) if args.base_root is not None: print(f"Generating the base project with {args.base_root}'s generator", file=sys.stderr) write_base_benchmark_project( @@ -57,7 +51,6 @@ def compare_compile_performance(argv: list[str] | None = None) -> int: kind=args.kind, project_dir=base_project_dir, models=args.models, - compiler_only=args.compiler_only, ) modes: tuple[str, ...] = tuple(dict.fromkeys(args.modes)) comparisons: tuple[CompileComparison, ...] = compare_builds( @@ -113,7 +106,6 @@ def compare_compile_performance(argv: list[str] | None = None) -> int: mode_cpu_max_ratios=mode_cpu_max_ratios, gate_phases=gate_phases, compile_args=tuple(args.compile_args), - compiler_only=args.compiler_only, ) print(markdown) _ = append_summary(path=Path(summary_value) if summary_value else None, markdown=markdown) @@ -184,12 +176,6 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: default=[], help="Extra `sqb compile` argument for both builds, e.g. `--compile-arg=--select=*`.", ) - parser.add_argument( - "--compiler-only", - action="store_true", - help="Write both projects with `select = []`, so both builds run the compiler without " - "default SQL Rules (fresh projects only).", - ) parser.add_argument( "--gate-phase", dest="gate_phases", @@ -204,7 +190,4 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: default=PHASE_NOISE_FLOOR_MS, help="Milliseconds of slack added to a gated phase's ratio allowance.", ) - args: argparse.Namespace = parser.parse_args(argv) - if args.compiler_only and args.kind == DENSE_KIND: - parser.error("--compiler-only supports --kind fresh only") - return args + return parser.parse_args(argv) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md index 6f2a6865ef..edf7119dbe 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md @@ -98,18 +98,37 @@ same runner, three alternating runs per mode, and the medians of wall and CPU ti - warm: an unchanged `sqb compile --json` after one untimed run that writes the cache; - edit: `sqb compile --json` after the same one-model edit as the release comparison below. -The fresh comparison runs twice. The compiler-only job writes both projects with `select = []` -(a base older than that option has no default SQL Rules) and keeps the 1.10x limit for every mode. The default Rules job allows 1.25x cold wall and 1.41x cold CPU: the 1.10x limit times the -measured cold cost of the default Rules over `select = []` on the same machine (1.14x wall, 1.28x -CPU, five paired runs). Its warm and edit modes keep 1.10x. The cold allowance is temporary: once +The fresh comparison writes both projects with the default SQL Rules, and a base older than them +runs none. It allows 1.25x cold wall and 1.41x cold CPU, which covers the cold cost of the default +Rules over `select = []` measured by this job (1.19x and 1.20x wall, 1.32x and 1.33x CPU, two runs of +three alternating pairs). Its warm and edit modes keep 1.10x. The cold allowance is temporary: once the base also runs default Rules, it returns to 1.10x. -Every fresh-process compile and cache guard follows the same split (approved 2026-11-01): -`compiler_only` cases keep the previous wall budgets, and the default Rules cases scale the cold -budget by the cold cost measured over `select = []` on the same machine: 1.14x, 1.16x and 1.09x for -the 3,000-, 5,000- and 10,000-model compiles, and 1.19x, 1.20x and 1.25x for the cache guards, whose -cold budget also bounds the cache-disabled and project-config runs. Warm, edit and memory budgets -are unchanged; warm and edit runs measured within 5% of `select = []`. +Every fresh-process compile and cache guard runs the default SQL Rules (approved 2026-11-01). One +guard keeps `select = []` and the previous budgets: the 3,000-model cache guard, which runs in the +same job as its default Rules twin and covers the cold, warm, edit and project-config edit compiles. +Each default Rules wall budget is the previous budget times the larger of two medians of default +Rules over `select = []`: same-job pairs in CI and alternating local pairs after one discarded +warm-up pair. A pair is excluded when a phase the Rules do not run (discover, attachment, test input +compile, write) takes more than twice its median over the other runs. The project-config edit has +its own budget because it reruns every rule on less parallel work. + +| Guard and run | CI same-job pairs | Local alternating | Factor | Budget | +| --- | --- | --- | --- | --- | +| Fresh 3,000 | 1.220, 1.239, 1.278 | 1.163 (5 pairs) | 1.24 | 9.9 s | +| Fresh 5,000 | 1.238, 1.205; 1.465 excluded (write 4.9 s) | 1.161 (4) | 1.23 | 15.4 s | +| Fresh 10,000 | 1.125 | 1.120 (6) | 1.13 | 28.8 s | +| Cache 3,000 cold | 1.221, 1.158, 1.166 | 1.114 (3) | 1.17 | 10.5 s | +| Cache 3,000 project-config edit | 1.203, 1.276, 1.232 | 1.170 (3) | 1.24 | 11.2 s | +| Cache 5,000 cold | 1.174 | 1.107 (3) | 1.18 | 17.1 s | +| Cache 5,000 project-config edit | 1.497 isolated, see below | 1.165 (3) | 1.25 | 18.1 s | +| Cache 10,000 cold | 1.198 | 1.118 (3, fixed order) | 1.20 | 33.6 s | +| Cache 10,000 project-config edit | 1.316 | 1.249 (3, fixed order) | 1.32 | 37.0 s | + +The single 5,000-model project-config pair took 1.39x its own cold compile against 1.08x to 1.13x on +three other CI runs, so it uses the local median times that size's CI over local cold ratio (1.06). +Warm, edit, memory and ratio budgets are unchanged, and the cold budget only sets the timeout of the +cache-disabled run. The warm and edit modes reuse the cold project. A warm run that misses the analysis cache, or an edit run that does not, stops the comparison because it would measure the wrong path. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 27618ecf2a..291727b67a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -172,8 +172,6 @@ class FreshProcessCompilePerformanceGuardTestCase: expected_max_wall_seconds: float expected_max_rss_bytes: int expected_semantic_fingerprint: str - # `select = []`: the compiler alone, without the default SQL Rules. - compiler_only: bool = False @dataclass(frozen=True) @@ -189,6 +187,7 @@ class FreshProcessCompileCachePerformanceGuardTestCase: expected_cold_max_wall_seconds: float expected_warm_max_wall_seconds: float expected_edit_max_wall_seconds: float + expected_project_config_edit_max_wall_seconds: float expected_max_warm_to_cold_ratio: float expected_max_edit_to_cold_ratio: float expected_max_cache_write_cpu_overhead_ratio: float diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 982be9e39d..caaf69d744 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -522,7 +522,6 @@ def run_fresh_process_semantic_compile_benchmark( test_count: int, audit_count: int, expected_max_wall_seconds: float, - compiler_only: bool = False, ) -> FreshProcessCompileBenchmarkResult: """Compile one clean semantic fixture in a fresh process without compiler caches.""" @@ -535,7 +534,6 @@ def run_fresh_process_semantic_compile_benchmark( macro_count=macro_count, test_count=test_count, audit_count=audit_count, - compiler_only=compiler_only, ) target_dir: Path = project_dir / "target" assert not target_dir.exists() @@ -562,6 +560,7 @@ def run_fresh_process_compile_cache_benchmark( expected_cold_max_wall_seconds: float, expected_warm_max_wall_seconds: float, expected_edit_max_wall_seconds: float, + expected_project_config_edit_max_wall_seconds: float, macro_call_interval: int, scoped_macros: bool, compiler_only: bool = False, @@ -642,7 +641,7 @@ def run_fresh_process_compile_cache_benchmark( project_config_edit: FreshProcessCompileBenchmarkResult = _run_fresh_process_compile_benchmark( project_dir=project_dir, label=f"cache-project-config-edit-{model_count}", - expected_max_wall_seconds=expected_cold_max_wall_seconds, + expected_max_wall_seconds=expected_project_config_edit_max_wall_seconds, compile_args=(), ) after_project_config_edit: FreshProcessCompileBenchmarkResult = ( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py index 5d2ec19f50..3dadbbb059 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py @@ -52,10 +52,12 @@ macro_count=500, test_count=2_945, audit_count=5_056, - # Default SQL Rules: 9.0 s scaled by the measured 1.19x cold cost over select = []. - expected_cold_max_wall_seconds=10.75, + # Default SQL Rules scale 9.0 s by the CI cost over select = []: 1.17x cold and + # 1.24x for the project-config edit, which reruns every rule. + expected_cold_max_wall_seconds=10.5, expected_warm_max_wall_seconds=4.5, expected_edit_max_wall_seconds=4.5, + expected_project_config_edit_max_wall_seconds=11.2, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO, @@ -87,9 +89,11 @@ macro_count=500, test_count=2_945, audit_count=5_056, + # The one select = [] guard: the same runner as its default Rules twin, previous budgets. expected_cold_max_wall_seconds=9.0, expected_warm_max_wall_seconds=4.5, expected_edit_max_wall_seconds=4.5, + expected_project_config_edit_max_wall_seconds=9.0, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO, @@ -122,10 +126,12 @@ macro_count=834, test_count=4_908, audit_count=8_427, - # Default SQL Rules: 14.5 s scaled by the measured 1.20x cold cost over select = []. - expected_cold_max_wall_seconds=17.4, + # Default SQL Rules scale 14.5 s by the CI cost over select = []: 1.18x cold and + # 1.25x for the project-config edit, which reruns every rule. + expected_cold_max_wall_seconds=17.1, expected_warm_max_wall_seconds=7.0, expected_edit_max_wall_seconds=7.0, + expected_project_config_edit_max_wall_seconds=18.1, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO, @@ -148,41 +154,6 @@ scoped_macros=True, expected_macro_edit_misses=1, ), - FreshProcessCompileCachePerformanceGuardTestCase( - description="models_5000_fresh_process_compiler_only_compile_cache_stays_incremental", - model_count=5_000, - source_count=1_189, - seed_count=236, - function_count=118, - macro_count=834, - test_count=4_908, - audit_count=8_427, - expected_cold_max_wall_seconds=14.5, - expected_warm_max_wall_seconds=7.0, - expected_edit_max_wall_seconds=7.0, - expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, - expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, - expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO, - expected_max_cache_write_wall_overhead_ratio=_MAX_CACHE_WRITE_WALL_OVERHEAD_RATIO, - expected_max_rss_bytes=1664 * _MIB, - expected_max_cache_bytes=160 * _MIB, - expected_cold_fingerprint=( - "2f1e5dfd29025c25e0dffc6e74f4c2be020a5ef133ecd081f44e0c55c24858b4" - ), - expected_leaf_edit_fingerprint=( - "708f04a565a71b3981c9beba420aff691c8467eb30c5cdedd45c2518d45b31ba" - ), - expected_macro_edit_fingerprint=( - "7dd4d45c2b21f61cbc32d33b0e0cf98721975b9f8948277cc15dc947355cfc25" - ), - expected_project_config_fingerprint=( - "69f4fe914afc955d690e9c5b9074ca69dc58b8b7a0886a421b888385e6b066cb" - ), - macro_call_interval=6, - scoped_macros=True, - expected_macro_edit_misses=1, - compiler_only=True, - ), FreshProcessCompileCachePerformanceGuardTestCase( description="models_10000_fresh_process_compile_cache_stays_incremental", model_count=10_000, @@ -192,45 +163,12 @@ macro_count=1_667, test_count=9_816, audit_count=16_855, - # Default SQL Rules: 28.0 s scaled by the measured 1.25x cold cost over select = []. - expected_cold_max_wall_seconds=35.0, - expected_warm_max_wall_seconds=14.0, - expected_edit_max_wall_seconds=13.5, - expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, - expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, - expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO, - expected_max_cache_write_wall_overhead_ratio=_MAX_CACHE_WRITE_WALL_OVERHEAD_RATIO, - # Temporary memory exception (2026-10-10); remove after conversion optimisation. - expected_max_rss_bytes=3008 * _MIB, - expected_max_cache_bytes=320 * _MIB, - expected_cold_fingerprint=( - "ecdd7f2cf961fab739aa806b6d90283916cffad2007e1e2f3ba90a466db6cb75" - ), - expected_leaf_edit_fingerprint=( - "f67b73b56d7ad7677cbfa649e7933e8f294f9dd7c99f3660d7c42e1d01b804ff" - ), - expected_macro_edit_fingerprint=( - "5554615dbc5fe9dbfd7555771167f3380222cb3da468b4313076adecef0ff9c3" - ), - expected_project_config_fingerprint=( - "ebc15f5381746d060005ecc61f477b1b0338640198197f6788f9beab5da88db5" - ), - macro_call_interval=6, - scoped_macros=True, - expected_macro_edit_misses=1, - ), - FreshProcessCompileCachePerformanceGuardTestCase( - description="models_10000_fresh_process_compiler_only_compile_cache_stays_incremental", - model_count=10_000, - source_count=2_377, - seed_count=471, - function_count=236, - macro_count=1_667, - test_count=9_816, - audit_count=16_855, - expected_cold_max_wall_seconds=28.0, + # Default SQL Rules scale 28.0 s by the CI cost over select = []: 1.20x cold and + # 1.32x for the project-config edit, which reruns every rule. + expected_cold_max_wall_seconds=33.6, expected_warm_max_wall_seconds=14.0, expected_edit_max_wall_seconds=13.5, + expected_project_config_edit_max_wall_seconds=37.0, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO, @@ -253,7 +191,6 @@ macro_call_interval=6, scoped_macros=True, expected_macro_edit_misses=1, - compiler_only=True, ), ], ids=lambda case: case.description, @@ -275,6 +212,9 @@ def test_given_semantic_project_when_compiling_across_processes_then_cache_is_in expected_cold_max_wall_seconds=test_case.expected_cold_max_wall_seconds, expected_warm_max_wall_seconds=test_case.expected_warm_max_wall_seconds, expected_edit_max_wall_seconds=test_case.expected_edit_max_wall_seconds, + expected_project_config_edit_max_wall_seconds=( + test_case.expected_project_config_edit_max_wall_seconds + ), macro_call_interval=test_case.macro_call_interval, scoped_macros=test_case.scoped_macros, compiler_only=test_case.compiler_only, @@ -341,7 +281,10 @@ def test_given_semantic_project_when_compiling_across_processes_then_cache_is_in measurement.elapsed_seconds / result.cold.elapsed_seconds <= test_case.expected_max_edit_to_cold_ratio ) - assert result.project_config_edit.elapsed_seconds < test_case.expected_cold_max_wall_seconds + assert ( + result.project_config_edit.elapsed_seconds + < test_case.expected_project_config_edit_max_wall_seconds + ) assert result.cache_bytes < test_case.expected_max_cache_bytes assert result.cache_disabled.semantic_fingerprint == test_case.expected_cold_fingerprint diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py index 2fea56120f..1c7aea4754 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py @@ -38,29 +38,13 @@ macro_count=37, test_count=2_945, audit_count=5_056, - # Default SQL Rules: 8.0 s scaled by the measured 1.14x cold cost over select = []. - expected_max_wall_seconds=9.2, + # Default SQL Rules: 8.0 s scaled by the 1.24x CI cost over select = []. + expected_max_wall_seconds=9.9, expected_max_rss_bytes=1088 * _MIB, expected_semantic_fingerprint=( "d935821df54043fabc3a6911c1926578178c9b744f2a589cbef99a10d5207543" ), ), - FreshProcessCompilePerformanceGuardTestCase( - description="models_3000_fresh_process_compiler_only_compile_stays_within_budget", - model_count=3_000, - source_count=713, - seed_count=141, - function_count=71, - macro_count=37, - test_count=2_945, - audit_count=5_056, - expected_max_wall_seconds=8.0, - expected_max_rss_bytes=1088 * _MIB, - expected_semantic_fingerprint=( - "d935821df54043fabc3a6911c1926578178c9b744f2a589cbef99a10d5207543" - ), - compiler_only=True, - ), FreshProcessCompilePerformanceGuardTestCase( description="models_5000_fresh_process_semantic_compile_stays_within_budget", model_count=5_000, @@ -70,29 +54,13 @@ macro_count=61, test_count=4_908, audit_count=8_427, - # Default SQL Rules: 12.5 s scaled by the measured 1.16x cold cost over select = []. - expected_max_wall_seconds=14.5, + # Default SQL Rules: 12.5 s scaled by the 1.23x CI cost over select = []. + expected_max_wall_seconds=15.4, expected_max_rss_bytes=1664 * _MIB, expected_semantic_fingerprint=( "874cbfd473846c6a335ec2f79cb6b7b0b77e8e8d58652815fad1e9d59d423c46" ), ), - FreshProcessCompilePerformanceGuardTestCase( - description="models_5000_fresh_process_compiler_only_compile_stays_within_budget", - model_count=5_000, - source_count=1_189, - seed_count=236, - function_count=118, - macro_count=61, - test_count=4_908, - audit_count=8_427, - expected_max_wall_seconds=12.5, - expected_max_rss_bytes=1664 * _MIB, - expected_semantic_fingerprint=( - "874cbfd473846c6a335ec2f79cb6b7b0b77e8e8d58652815fad1e9d59d423c46" - ), - compiler_only=True, - ), FreshProcessCompilePerformanceGuardTestCase( description="models_10000_fresh_process_semantic_compile_stays_within_budget", model_count=10_000, @@ -102,28 +70,12 @@ macro_count=123, test_count=9_816, audit_count=16_855, - # Default SQL Rules: 25.5 s scaled by the measured 1.09x cold cost over select = []. - expected_max_wall_seconds=28.0, - expected_max_rss_bytes=int(2.5 * _GIB), - expected_semantic_fingerprint=( - "5ccdabeaabfdfba1ecbae80259dd3c6eb801d683df140b7fb7a249b9555308c6" - ), - ), - FreshProcessCompilePerformanceGuardTestCase( - description="models_10000_fresh_process_compiler_only_compile_stays_within_budget", - model_count=10_000, - source_count=2_377, - seed_count=471, - function_count=236, - macro_count=123, - test_count=9_816, - audit_count=16_855, - expected_max_wall_seconds=25.5, + # Default SQL Rules: 25.5 s scaled by the 1.13x CI cost over select = []. + expected_max_wall_seconds=28.8, expected_max_rss_bytes=int(2.5 * _GIB), expected_semantic_fingerprint=( "5ccdabeaabfdfba1ecbae80259dd3c6eb801d683df140b7fb7a249b9555308c6" ), - compiler_only=True, ), ], ids=lambda case: case.description, @@ -143,7 +95,6 @@ def test_given_scaled_semantic_project_when_compiling_fresh_then_preserves_resou test_count=test_case.test_count, audit_count=test_case.audit_count, expected_max_wall_seconds=test_case.expected_max_wall_seconds, - compiler_only=test_case.compiler_only, ) _LOGGER.info( "fresh-process semantic compile models=%d wall=%.3fs peak_rss_bytes=%d " @@ -174,6 +125,6 @@ def test_given_scaled_semantic_project_when_compiling_fresh_then_preserves_resou } assert result.semantic_fingerprint == test_case.expected_semantic_fingerprint timings: dict[str, int] = cast(dict[str, int], result.payload["compile_timings"]) - assert (timings["rule_cache_misses"] > 0) is not test_case.compiler_only + assert timings["rule_cache_misses"] > 0 assert result.elapsed_seconds < test_case.expected_max_wall_seconds assert result.peak_rss_bytes < test_case.expected_max_rss_bytes From 34d84cf3a9658ba2903deb52e0a074d7ac0b8365 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:33:03 +0000 Subject: [PATCH 8/8] docs(compile): correct the 5,000-model project-config budget and cold budget scope The stated method gives 1.24x for the 5,000-model project-config edit, so its budget is 18.0 s. The cold budget bounds the cold run as well as the cache-disabled timeout. --- .../cli/commands/main/compile/COMPILE_BENCHMARKS.md | 6 +++--- .../compile/test_fresh_process_compile_cache_performance.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md index edf7119dbe..5c4260e128 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md @@ -121,14 +121,14 @@ its own budget because it reruns every rule on less parallel work. | Cache 3,000 cold | 1.221, 1.158, 1.166 | 1.114 (3) | 1.17 | 10.5 s | | Cache 3,000 project-config edit | 1.203, 1.276, 1.232 | 1.170 (3) | 1.24 | 11.2 s | | Cache 5,000 cold | 1.174 | 1.107 (3) | 1.18 | 17.1 s | -| Cache 5,000 project-config edit | 1.497 isolated, see below | 1.165 (3) | 1.25 | 18.1 s | +| Cache 5,000 project-config edit | 1.497 isolated, see below | 1.165 (3) | 1.24 | 18.0 s | | Cache 10,000 cold | 1.198 | 1.118 (3, fixed order) | 1.20 | 33.6 s | | Cache 10,000 project-config edit | 1.316 | 1.249 (3, fixed order) | 1.32 | 37.0 s | The single 5,000-model project-config pair took 1.39x its own cold compile against 1.08x to 1.13x on three other CI runs, so it uses the local median times that size's CI over local cold ratio (1.06). -Warm, edit, memory and ratio budgets are unchanged, and the cold budget only sets the timeout of the -cache-disabled run. +Warm, edit, memory and ratio budgets are unchanged. The cold budget bounds the cold run and sets the +cache-disabled timeout, and no longer the project-config edit. The warm and edit modes reuse the cold project. A warm run that misses the analysis cache, or an edit run that does not, stops the comparison because it would measure the wrong path. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py index 3dadbbb059..f904e26fd1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py @@ -127,11 +127,11 @@ test_count=4_908, audit_count=8_427, # Default SQL Rules scale 14.5 s by the CI cost over select = []: 1.18x cold and - # 1.25x for the project-config edit, which reruns every rule. + # 1.24x for the project-config edit, which reruns every rule. expected_cold_max_wall_seconds=17.1, expected_warm_max_wall_seconds=7.0, expected_edit_max_wall_seconds=7.0, - expected_project_config_edit_max_wall_seconds=18.1, + expected_project_config_edit_max_wall_seconds=18.0, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO,