perf: run semantic validation on the shared scope tree

Cargo.toml

@@ -13,7 +13,7 @@ exclude = [
 ]
 
 [workspace.package]
-version = "0.13.7"
+version = "0.13.8"
 edition = "2021"
 license = "MIT"
 authors = ["polyglot contributors"]

README.md

@@ -306,7 +306,7 @@ If you want to disable `stacker` for a native Rust build, turn off default featu
 
 ```toml
 [dependencies]
-polyglot-sql = { version = "0.13.7", default-features = false, features = ["all-dialects", "transpile"] }
+polyglot-sql = { version = "0.13.8", default-features = false, features = ["all-dialects", "transpile"] }
 ```
 
 That can reduce overhead slightly on trusted inputs, but you lose the default stack-growth protection for deeply nested SQL.

crates/polyglot-sql-function-catalogs/README.md

@@ -25,7 +25,7 @@ With `polyglot-sql` compile-time wiring (recommended):
 
 ```toml
 [dependencies]
-polyglot-sql = { version = "0.13.7", features = ["function-catalog-clickhouse"] }
+polyglot-sql = { version = "0.13.8", features = ["function-catalog-clickhouse"] }
 ```
 
 Then run schema validation with type checks:

crates/polyglot-sql/Cargo.toml

@@ -100,14 +100,14 @@ function-catalog-all-dialects = [
 
 [dependencies]
 regex = { version = "1", optional = true }
-polyglot-sql-ast-derive = { path = "../polyglot-sql-ast-derive", version = "0.13.7" }
+polyglot-sql-ast-derive = { path = "../polyglot-sql-ast-derive", version = "0.13.8" }
 serde = { workspace = true }
 serde_json = { workspace = true }
 thiserror = { workspace = true }
 unicode-segmentation = { workspace = true }
 stacker = { version = "0.1", optional = true }
 ts-rs = { version = "12.0", features = ["serde-compat"], optional = true }
-polyglot-sql-function-catalogs = { path = "../polyglot-sql-function-catalogs", version = "0.13.7", optional = true, default-features = false }
+polyglot-sql-function-catalogs = { path = "../polyglot-sql-function-catalogs", version = "0.13.8", optional = true, default-features = false }
 
 [dev-dependencies]
 pretty_assertions = "1.4"

crates/polyglot-sql/README.md

@@ -25,12 +25,12 @@ By default, `polyglot-sql` enables the full public API. Parser-only consumers ca
 disable default features and opt into only the dialect parsers they need:
 
 ```toml
-polyglot-sql = { version = "0.13.7", default-features = false }
+polyglot-sql = { version = "0.13.8", default-features = false }
 ```
 
 ```toml
 polyglot-sql = {
-    version = "0.13.7",
+    version = "0.13.8",
     default-features = false,
     features = ["dialect-clickhouse"],
 }
@@ -44,14 +44,14 @@ Examples:
 ```toml
 # Parse and generate SQL for one dialect.
 polyglot-sql = {
-    version = "0.13.7",
+    version = "0.13.8",
     default-features = false,
     features = ["generate", "dialect-clickhouse"],
 }
 
 # Cross-dialect transpilation.
 polyglot-sql = {
-    version = "0.13.7",
+    version = "0.13.8",
     default-features = false,
     features = ["transpile", "dialect-clickhouse", "dialect-postgresql"],
 }

crates/polyglot-sql/src/lineage.rs

@@ -1048,6 +1048,9 @@ struct LineageScopeContext<'a> {
     scopes: Vec<IndexedScope<'a>>,
     scope_ids: HashMap<*const Scope, ScopeId>,
     resolved_states: RefCell<HashSet<LineageResolutionState>>,
+    /// Child-scope lookups by source name; each lookup is a pure function of the
+    /// immutable scope tree, and derived-table matching compares whole queries.
+    child_scopes: RefCell<HashMap<(ScopeId, String), Option<ScopeId>>>,
     /// Usage analysis must not claim undeclared qualifiers as physical tables.
     conservative: bool,
 }
@@ -1228,6 +1231,7 @@ impl<'a> LineageScopeContext<'a> {
             scopes: Vec::new(),
             scope_ids: HashMap::new(),
             resolved_states: RefCell::new(HashSet::new()),
+            child_scopes: RefCell::new(HashMap::new()),
             conservative: false,
             schema: None,
         };
@@ -3544,6 +3548,20 @@ fn find_child_scope(
     context: &LineageScopeContext,
     scope_id: ScopeId,
     source_name: &str,
+) -> Option<ScopeId> {
+    let key = (scope_id, source_name.to_owned());
+    if let Some(found) = context.child_scopes.borrow().get(&key) {
+        return *found;
+    }
+    let found = find_child_scope_uncached(context, scope_id, source_name);
+    context.child_scopes.borrow_mut().insert(key, found);
+    found
+}
+
+fn find_child_scope_uncached(
+    context: &LineageScopeContext,
+    scope_id: ScopeId,
+    source_name: &str,
 ) -> Option<ScopeId> {
     let indexed = context.indexed(scope_id);
     let scope = &indexed.scope;

crates/polyglot-sql/src/query_analysis.rs

@@ -850,6 +850,8 @@ struct NullabilityContext<'a> {
     scopes: Vec<NullabilityScope<'a>>,
     scope_ids: HashMap<*const Scope, usize>,
     outputs: RefCell<HashMap<(usize, usize), ProjectionNullability>>,
+    /// Derived-table scope per (frame, source name); matching compares whole queries.
+    derived_scopes: RefCell<HashMap<(*const Scope, String), Option<usize>>>,
     resolving: RefCell<HashSet<(usize, usize)>>,
     uncertain_columns: &'a HashMap<(usize, usize), ReferenceConfidence>,
 }
@@ -888,6 +890,7 @@ impl<'a> NullabilityContext<'a> {
             empty_schema: MappingSchema::with_dialect(dialect),
             dialect,
             scopes: Vec::new(),
+            derived_scopes: RefCell::new(HashMap::new()),
             scope_ids: HashMap::new(),
             outputs: RefCell::new(HashMap::new()),
             resolving: RefCell::new(HashSet::new()),
@@ -2807,10 +2810,16 @@ impl NullabilityContext<'_> {
             }
         }
         if source.kind == SourceKind::DerivedTable && source.is_scope {
-            return frame.derived.iter().copied().find(|id| {
+            let key = (frame.scope as *const Scope, name.to_owned());
+            if let Some(found) = self.derived_scopes.borrow().get(&key) {
+                return *found;
+            }
+            let found = frame.derived.iter().copied().find(|id| {
                 crate::scope::scope_query(&self.scopes[*id].scope.expression)
                     == crate::scope::scope_query(&source.expression)
             });
+            self.derived_scopes.borrow_mut().insert(key, found);
+            return found;
         }
         None
     }

crates/polyglot-sql/src/scope.rs

@@ -75,19 +75,54 @@ pub(crate) fn scope_query(expression: &Expression) -> &Expression {
 /// declarations remain available, but do not themselves cause ambiguity.
 pub(crate) fn selected_reference_scope(scope: &Scope) -> Scope {
     let query = scope_query(&scope.expression);
-    let aliases: HashSet<_> = walk_in_scope(query, false)
+    let selected_expression = match query {
+        Expression::Select(select) => Expression::Select(Box::new(select_without_with(select))),
+        _ => query.clone(),
+    };
+    selected_scope_view(scope, selected_expression)
+}
+
+/// `selected_reference_scope` for a scope tree its caller owns. The query is
+/// moved into the view instead of copied, leaving `scope.expression` as NULL;
+/// the removed WITH clause is returned. Child scopes are unaffected.
+pub(crate) fn take_selected_reference_scope(
+    scope: &mut Scope,
+) -> (Scope, Option<crate::expressions::With>) {
+    fn into_scope_query(expression: Expression) -> Expression {
+        match expression {
+            Expression::Cte(cte) => into_scope_query(cte.this),
+            Expression::Subquery(subquery) => into_scope_query(subquery.this),
+            Expression::Paren(paren) => into_scope_query(paren.this),
+            Expression::Alias(alias) => into_scope_query(alias.this),
+            Expression::Annotated(annotated) => into_scope_query(annotated.this),
+            Expression::Prepare(prepare) => into_scope_query(prepare.statement),
+            Expression::CreateTable(create) if create.as_select.is_some() => {
+                into_scope_query(create.as_select.unwrap())
+            }
+            expression => expression,
+        }
+    }
+    let mut query = into_scope_query(std::mem::replace(
+        &mut scope.expression,
+        Expression::Null(crate::expressions::Null),
+    ));
+    let with = match &mut query {
+        Expression::Select(select) => select.with.take(),
+        _ => None,
+    };
+    (selected_scope_view(scope, query), with)
+}
+
+fn selected_scope_view(scope: &Scope, query: Expression) -> Scope {
+    let aliases: HashSet<_> = walk_in_scope(&query, false)
         .filter_map(|node| match node {
             Expression::Table(table) => {
                 Some(table.alias.as_ref().unwrap_or(&table.name).name.clone())
             }
             _ => None,
         })
         .collect();
-    let selected_expression = match query {
-        Expression::Select(select) => Expression::Select(Box::new(select_without_with(select))),
-        _ => query.clone(),
-    };
-    let mut selected = Scope::new(selected_expression);
+    let mut selected = Scope::new(query);
     selected.cte_sources = scope.cte_sources.clone();
     selected.sources = scope
         .sources