From 6aac5c48a2a90491ded49d6538be1dff2266c92e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 4 Nov 2026 18:29:54 +0000 Subject: [PATCH 1/8] wip: consumption records core and decision simulation Append-only consumption rows, the --changed decision over recent executions, and a seeded simulation whose oracle tracks what every model read. Validated so far: the core crate's tests and clippy only; nothing is wired into builds yet. --- crates/sqlbuild-core/src/constants.rs | 2 + .../src/consumption/_helpers/decision.rs | 166 +++++ .../src/consumption/_helpers/history.rs | 164 +++++ .../src/consumption/_helpers/mod.rs | 3 + .../src/consumption/_helpers/rules.rs | 33 + .../src/consumption/constants.rs | 51 ++ .../src/consumption/main/changed_run_set.rs | 13 + .../src/consumption/main/consumed_inputs.rs | 42 ++ .../consumption/main/consumption_history.rs | 10 + .../src/consumption/main/event_id.rs | 19 + .../src/consumption/main/mod.rs | 8 + .../consumption/main/prune_executions_sql.rs | 15 + .../consumption/main/recent_executions_sql.rs | 21 + .../src/consumption/main/started_append.rs | 24 + .../src/consumption/main/succeeded_appends.rs | 28 + crates/sqlbuild-state/src/consumption/mod.rs | 7 + .../sqlbuild-state/src/consumption/models.rs | 122 ++++ .../sqlbuild-state/src/consumption/tests.rs | 6 + .../src/consumption/tests/helpers.rs | 577 ++++++++++++++++++ .../src/consumption/tests/test_simulation.rs | 50 ++ .../src/consumption/tests/test_types.rs | 92 +++ crates/sqlbuild-state/src/lib.rs | 1 + 22 files changed, 1454 insertions(+) create mode 100644 crates/sqlbuild-state/src/consumption/_helpers/decision.rs create mode 100644 crates/sqlbuild-state/src/consumption/_helpers/history.rs create mode 100644 crates/sqlbuild-state/src/consumption/_helpers/mod.rs create mode 100644 crates/sqlbuild-state/src/consumption/_helpers/rules.rs create mode 100644 crates/sqlbuild-state/src/consumption/constants.rs create mode 100644 crates/sqlbuild-state/src/consumption/main/changed_run_set.rs create mode 100644 crates/sqlbuild-state/src/consumption/main/consumed_inputs.rs create mode 100644 crates/sqlbuild-state/src/consumption/main/consumption_history.rs create mode 100644 crates/sqlbuild-state/src/consumption/main/event_id.rs create mode 100644 crates/sqlbuild-state/src/consumption/main/mod.rs create mode 100644 crates/sqlbuild-state/src/consumption/main/prune_executions_sql.rs create mode 100644 crates/sqlbuild-state/src/consumption/main/recent_executions_sql.rs create mode 100644 crates/sqlbuild-state/src/consumption/main/started_append.rs create mode 100644 crates/sqlbuild-state/src/consumption/main/succeeded_appends.rs create mode 100644 crates/sqlbuild-state/src/consumption/mod.rs create mode 100644 crates/sqlbuild-state/src/consumption/models.rs create mode 100644 crates/sqlbuild-state/src/consumption/tests.rs create mode 100644 crates/sqlbuild-state/src/consumption/tests/helpers.rs create mode 100644 crates/sqlbuild-state/src/consumption/tests/test_simulation.rs create mode 100644 crates/sqlbuild-state/src/consumption/tests/test_types.rs diff --git a/crates/sqlbuild-core/src/constants.rs b/crates/sqlbuild-core/src/constants.rs index 75a4f1cdc8..7f5096db14 100644 --- a/crates/sqlbuild-core/src/constants.rs +++ b/crates/sqlbuild-core/src/constants.rs @@ -22,6 +22,8 @@ pub const PYTHON_INIT_MODULE_STEM: &str = "__init__"; pub const FINGERPRINT_TABLE_NAME: &str = "_sqlbuild_fingerprints"; /// State table of observed source freshness. pub const SOURCE_FRESHNESS_TABLE_NAME: &str = "_sqlbuild_source_freshness"; +/// State table of the inputs each model and seed execution consumed. +pub const CONSUMPTION_TABLE_NAME: &str = "_sqlbuild_consumption"; /// State table of model migration events. pub const MIGRATION_TABLE_NAME: &str = "_sqlbuild_migrations"; /// State table of column migration events. diff --git a/crates/sqlbuild-state/src/consumption/_helpers/decision.rs b/crates/sqlbuild-state/src/consumption/_helpers/decision.rs new file mode 100644 index 0000000000..211c8a548f --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/_helpers/decision.rs @@ -0,0 +1,166 @@ +//! Decide which selected nodes `--changed` runs, from their facts and their recent executions. + +use std::collections::{BTreeSet, HashMap}; + +use crate::consumption::_helpers::history::Window; +use crate::consumption::_helpers::rules::DecisionRules; +use crate::consumption::constants::{NODE_INPUT, OPAQUE_INPUT, SOURCE_INPUT}; +use crate::consumption::models::{ConsumptionHistory, InputFact, NodeFacts, NodeKey, RunReason}; + +/// The state table key of an input: its kind and name. +pub(crate) fn input_key(input: &InputFact) -> (&'static str, String) { + match input { + InputFact::Source { name, .. } => (SOURCE_INPUT, name.clone()), + InputFact::Node(key) => (NODE_INPUT, node_input_name(key)), + InputFact::Opaque(name) => (OPAQUE_INPUT, name.clone()), + } +} + +/// The recorded name of an upstream node input: `:`. +pub(crate) fn node_input_name(key: &NodeKey) -> String { + format!("{}:{}", key.kind, key.name) +} + +/// Each selected node in input order with the reason it runs, or None when it is skipped. +pub(crate) fn decide( + rules: &R, + nodes: &[NodeFacts], + history: &ConsumptionHistory, +) -> Vec<(NodeKey, Option)> { + let by_key: HashMap<&NodeKey, &NodeFacts> = + nodes.iter().map(|node| (&node.key, node)).collect(); + let mut stale: HashMap<&NodeKey, Option> = HashMap::new(); + for node in topological(nodes, &by_key) { + let reason: Option = match own_reason(rules, node, &by_key, history) { + Some(reason) => Some(reason), + None if rules.follows_upstream() => stale_upstream(node, &stale), + None => None, + }; + stale.insert(&node.key, reason); + } + nodes + .iter() + .filter(|node| node.selected) + .map(|node| (node.key.clone(), stale.remove(&node.key).flatten())) + .collect() +} + +/// The first upstream of `node` that runs or is stale. +fn stale_upstream( + node: &NodeFacts, + stale: &HashMap<&NodeKey, Option>, +) -> Option { + for input in &node.inputs { + if let InputFact::Node(upstream) = input + && matches!(stale.get(upstream), Some(Some(_))) + { + return Some(RunReason::UpstreamRuns(upstream.clone())); + } + } + None +} + +/// Nodes with every upstream that has facts before it. +fn topological<'a>( + nodes: &'a [NodeFacts], + by_key: &HashMap<&NodeKey, &'a NodeFacts>, +) -> Vec<&'a NodeFacts> { + let mut ordered: Vec<&'a NodeFacts> = Vec::with_capacity(nodes.len()); + let mut placed: BTreeSet<&NodeKey> = BTreeSet::new(); + let mut stack: Vec<(&'a NodeFacts, usize)> = Vec::new(); + for root in nodes { + if placed.contains(&root.key) { + continue; + } + let mut entered: BTreeSet<&NodeKey> = BTreeSet::from([&root.key]); + stack.push((root, 0)); + while let Some((node, next)) = stack.pop() { + let upstream: Option<&'a NodeFacts> = + node.inputs.get(next).and_then(|input| match input { + InputFact::Node(key) => by_key.get(key).copied(), + _ => None, + }); + if next < node.inputs.len() { + stack.push((node, next + 1)); + if let Some(upstream) = upstream + && !placed.contains(&upstream.key) + && entered.insert(&upstream.key) + { + stack.push((upstream, 0)); + } + continue; + } + if placed.insert(&node.key) { + ordered.push(node); + } + } + } + ordered +} + +/// Why the node's own facts or records make it run, ignoring its upstreams' staleness. +fn own_reason( + rules: &R, + node: &NodeFacts, + by_key: &HashMap<&NodeKey, &NodeFacts>, + history: &ConsumptionHistory, +) -> Option { + if let Some(forced) = node.forced.iter().find(|forced| rules.honours(forced)) { + return Some(RunReason::Forced(forced.clone())); + } + let Some(window) = history.window(&node.relation) else { + return Some(RunReason::NoRecord); + }; + if window.ambiguous { + return Some(RunReason::AmbiguousOrder); + } + execution_reason(rules, node, (by_key, history), &window) +} + +fn execution_reason( + rules: &R, + node: &NodeFacts, + (by_key, history): (&HashMap<&NodeKey, &NodeFacts>, &ConsumptionHistory), + window: &Window<'_>, +) -> Option { + let expected: BTreeSet<(&str, String)> = node.inputs.iter().map(input_key).collect(); + for execution in window.executions { + if !execution.complete { + return Some(RunReason::Unfinished); + } + if execution.version_hash.as_deref() != Some(node.version_hash.as_str()) { + return Some(RunReason::DefinitionChanged); + } + let recorded: BTreeSet<(&str, String)> = execution + .inputs + .iter() + .map(|input| (input.kind.as_str(), input.name.clone())) + .collect(); + if recorded != expected { + return Some(RunReason::InputsChanged); + } + for input in &node.inputs { + let (kind, name) = input_key(input); + let current: Option<&str> = match input { + InputFact::Source { version, .. } => version.as_deref(), + InputFact::Node(upstream) if rules.compares_node_inputs() => by_key + .get(upstream) + .and_then(|facts| history.latest_complete(&facts.relation)), + InputFact::Node(_) => continue, + InputFact::Opaque(_) => None, + }; + let Some(current) = current else { + return Some(RunReason::InputUnknown(name)); + }; + let read: Option<&str> = execution + .inputs + .iter() + .find(|recorded| recorded.kind == kind && recorded.name == name) + .map(|recorded| recorded.version.as_str()); + if read != Some(current) { + return Some(RunReason::InputChanged(name)); + } + } + } + None +} diff --git a/crates/sqlbuild-state/src/consumption/_helpers/history.rs b/crates/sqlbuild-state/src/consumption/_helpers/history.rs new file mode 100644 index 0000000000..39c4855eca --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/_helpers/history.rs @@ -0,0 +1,164 @@ +//! Rebuild executions from deduplicated rows and read the window `--changed` compares. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use crate::consumption::_helpers::rules::DecisionRules; +use crate::consumption::constants::{ORDER_SKEW_MICROS, SUCCEEDED_PHASE}; +use crate::consumption::models::{ + ConsumedInput, ConsumptionHistory, ConsumptionRow, Execution, RelationHistory, RelationKey, +}; + +/// The rows of one execution with the earliest and latest times seen for each event. +#[derive(Default)] +struct ExecutionRows<'a> { + rows: Vec<&'a ConsumptionRow>, + earliest: i64, + latest: i64, +} + +/// Recent executions per relation, from rows read newest first with at most `limit` per relation. +pub(crate) fn build_history( + rules: &R, + rows: &[ConsumptionRow], + limit: usize, +) -> ConsumptionHistory { + let mut events: HashMap<&str, (&ConsumptionRow, i64, i64)> = HashMap::new(); + for row in rows { + let entry = events.entry(row.row.event_id.as_str()).or_insert(( + row, + row.recorded_at, + row.recorded_at, + )); + if row.recorded_at < entry.1 { + *entry = (row, row.recorded_at, entry.2); + } + entry.2 = entry.2.max(row.recorded_at); + } + let mut grouped: BTreeMap<(&RelationKey, &str), ExecutionRows<'_>> = BTreeMap::new(); + for (row, earliest, latest) in events.into_values() { + let key = (&row.row.relation, row.row.execution_id.as_str()); + let execution = grouped.entry(key).or_insert(ExecutionRows { + rows: Vec::new(), + earliest, + latest, + }); + execution.earliest = execution.earliest.min(earliest); + execution.latest = execution.latest.max(latest); + execution.rows.push(row); + } + let mut relations: HashMap = HashMap::new(); + for ((relation, execution_id), execution_rows) in grouped { + relations + .entry(relation.clone()) + .or_default() + .executions + .push(execution(rules, execution_id, &execution_rows)); + } + for history in relations.values_mut() { + history.executions.sort_by(|left, right| { + (right.started_at, &right.execution_id).cmp(&(left.started_at, &left.execution_id)) + }); + history.truncated = history.executions.len() >= limit; + } + ConsumptionHistory { relations } +} + +fn execution( + rules: &R, + execution_id: &str, + rows: &ExecutionRows<'_>, +) -> Execution { + let succeeded: Vec<&ConsumptionRow> = rows + .rows + .iter() + .copied() + .filter(|row| row.row.phase == SUCCEEDED_PHASE) + .collect(); + let versions: BTreeSet<&str> = rows + .rows + .iter() + .map(|row| row.row.version_hash.as_str()) + .collect(); + let counts: BTreeSet = succeeded.iter().map(|row| row.row.input_count).collect(); + let mut inputs: BTreeMap<(&str, &str), BTreeSet<&str>> = BTreeMap::new(); + let mut inputless: bool = false; + for row in &succeeded { + match &row.row.input { + Some(input) => { + inputs + .entry((input.kind.as_str(), input.name.as_str())) + .or_default() + .insert(input.version.as_str()); + } + None => inputless = true, + } + } + let consistent: bool = versions.len() == 1 + && counts.len() == 1 + && inputs.values().all(|versions| versions.len() == 1); + let expected: Option = counts.first().copied(); + let complete: bool = consistent + && match expected { + Some(0) => inputless && inputs.is_empty(), + Some(count) => !inputless && i64::try_from(inputs.len()) == Ok(count), + None => false, + }; + Execution { + execution_id: execution_id.to_owned(), + started_at: rules.execution_time(rows.earliest, rows.latest), + complete, + version_hash: (versions.len() == 1) + .then(|| versions.first().map(|version| (*version).to_owned())) + .flatten(), + inputs: if complete { + inputs.into_iter().filter_map(single_version).collect() + } else { + Vec::new() + }, + } +} + +/// The input with its one recorded version. +fn single_version( + ((kind, name), versions): ((&str, &str), BTreeSet<&str>), +) -> Option { + let version: &str = versions.first()?; + Some(ConsumedInput { + kind: kind.to_owned(), + name: name.to_owned(), + version: version.to_owned(), + }) +} + +/// The executions `--changed` must all check: those within the order skew of the newest. +pub(crate) struct Window<'a> { + pub(crate) executions: &'a [Execution], + /// Older unread executions may also fall inside the skew. + pub(crate) ambiguous: bool, +} + +impl ConsumptionHistory { + /// The newest executions of `relation` within the skew, or None when it has none. + pub(crate) fn window(&self, relation: &RelationKey) -> Option> { + let history: &RelationHistory = self.relations.get(relation)?; + let newest: i64 = history.executions.first()?.started_at; + let length: usize = history + .executions + .iter() + .take_while(|execution| execution.started_at >= newest - ORDER_SKEW_MICROS) + .count(); + Some(Window { + executions: &history.executions[..length], + ambiguous: history.truncated && length == history.executions.len(), + }) + } + + /// The execution ID of `relation`'s newest execution when it alone is newest and complete. + pub fn latest_complete(&self, relation: &RelationKey) -> Option<&str> { + let window: Window<'_> = self.window(relation)?; + match window.executions { + [only] if !window.ambiguous && only.complete => Some(only.execution_id.as_str()), + _ => None, + } + } +} diff --git a/crates/sqlbuild-state/src/consumption/_helpers/mod.rs b/crates/sqlbuild-state/src/consumption/_helpers/mod.rs new file mode 100644 index 0000000000..d7ef1a72d2 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod decision; +pub(crate) mod history; +pub(crate) mod rules; diff --git a/crates/sqlbuild-state/src/consumption/_helpers/rules.rs b/crates/sqlbuild-state/src/consumption/_helpers/rules.rs new file mode 100644 index 0000000000..24a08a25aa --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/_helpers/rules.rs @@ -0,0 +1,33 @@ +//! The decision rules `--changed` applies; tests swap single rules to prove the simulation notices. + +use crate::consumption::models::ForcedRun; + +/// Each rule of the decision, with the shipped behaviour as its default. +pub(crate) trait DecisionRules { + /// An execution's time: the earliest of its rows, so a late duplicate cannot make it newer. + fn execution_time(&self, earliest: i64, latest: i64) -> i64 { + let _ = latest; + earliest + } + + /// Whether upstream model and seed inputs compare their recorded execution IDs. + fn compares_node_inputs(&self) -> bool { + true + } + + /// Whether a stale upstream, selected or not, makes its downstream run. + fn follows_upstream(&self) -> bool { + true + } + + /// Whether the planner's own reason to run a node is honoured. + fn honours(&self, forced: &ForcedRun) -> bool { + let _ = forced; + true + } +} + +/// The shipped rules. +pub(crate) struct Shipped; + +impl DecisionRules for Shipped {} diff --git a/crates/sqlbuild-state/src/consumption/constants.rs b/crates/sqlbuild-state/src/consumption/constants.rs new file mode 100644 index 0000000000..06a1975b65 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/constants.rs @@ -0,0 +1,51 @@ +//! Columns, phases, input kinds and read bounds of the consumption state table. + +use crate::freshness::constants::{STRING_STATE_VALUE_TYPE, TEXT_TIMESTAMP_STATE_VALUE_TYPE}; + +/// State value type of an integer column. +pub const INTEGER_STATE_VALUE_TYPE: &str = "integer"; +/// The column the warehouse clock fills on insert. +pub const RECORDED_AT_COLUMN: &str = "recorded_at"; +/// Columns in declaration order with their state value types; `recorded_at` is last. +pub const CONSUMPTION_COLUMN_TYPES: [(&str, &str); 15] = [ + ("event_id", STRING_STATE_VALUE_TYPE), + ("execution_id", STRING_STATE_VALUE_TYPE), + ("phase", STRING_STATE_VALUE_TYPE), + ("run_id", STRING_STATE_VALUE_TYPE), + ("node_type", STRING_STATE_VALUE_TYPE), + ("node_name", STRING_STATE_VALUE_TYPE), + ("target_database", STRING_STATE_VALUE_TYPE), + ("target_schema", STRING_STATE_VALUE_TYPE), + ("target_name", STRING_STATE_VALUE_TYPE), + ("version_hash", STRING_STATE_VALUE_TYPE), + ("input_count", INTEGER_STATE_VALUE_TYPE), + ("input_kind", STRING_STATE_VALUE_TYPE), + ("input_name", STRING_STATE_VALUE_TYPE), + ("input_version", STRING_STATE_VALUE_TYPE), + (RECORDED_AT_COLUMN, TEXT_TIMESTAMP_STATE_VALUE_TYPE), +]; +/// Columns that may be NULL: the relation's database and the input of `started` and input-less rows. +pub const NULLABLE_COLUMNS: [&str; 4] = [ + "target_database", + "input_kind", + "input_name", + "input_version", +]; +/// Phase of the row appended before a model or seed writes. +pub const STARTED_PHASE: &str = "started"; +/// Phase of the rows appended after the write succeeded. +pub const SUCCEEDED_PHASE: &str = "succeeded"; +/// Input kind of a source, versioned by its observed `data_version_hash`. +pub const SOURCE_INPUT: &str = "source"; +/// Input kind of an upstream model or seed, versioned by its `execution_id`. +pub const NODE_INPUT: &str = "node"; +/// Input kind of anything whose version cannot be known. +pub const OPAQUE_INPUT: &str = "opaque"; +/// The version recorded for an input whose current version is unknown. +pub const UNKNOWN_VERSION: &str = "unknown"; +/// Executions read per relation, newest first. +pub const RECENT_EXECUTIONS: usize = 8; +/// Executions of one relation appended this close to the newest are all checked, in microseconds. +pub const ORDER_SKEW_MICROS: i64 = 5_000_000; +/// Event ID namespace of consumption rows. +pub(crate) const EVENT_NAMESPACE: &str = "sqlbuild.consumption.event.v1"; diff --git a/crates/sqlbuild-state/src/consumption/main/changed_run_set.rs b/crates/sqlbuild-state/src/consumption/main/changed_run_set.rs new file mode 100644 index 0000000000..1f5764554f --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/changed_run_set.rs @@ -0,0 +1,13 @@ +//! Which selected nodes `--changed` runs. + +use crate::consumption::_helpers::decision::decide; +use crate::consumption::_helpers::rules::Shipped; +use crate::consumption::models::{ConsumptionHistory, NodeFacts, NodeKey, RunReason}; + +/// Each selected node in order with why it runs, or None when nothing it or an upstream read changed. +pub fn changed_run_set( + nodes: &[NodeFacts], + history: &ConsumptionHistory, +) -> Vec<(NodeKey, Option)> { + decide(&Shipped, nodes, history) +} diff --git a/crates/sqlbuild-state/src/consumption/main/consumed_inputs.rs b/crates/sqlbuild-state/src/consumption/main/consumed_inputs.rs new file mode 100644 index 0000000000..6e13ee2c80 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/consumed_inputs.rs @@ -0,0 +1,42 @@ +//! The input versions an execution consumed, recorded with its `succeeded` rows. + +use std::collections::HashMap; + +use crate::consumption::_helpers::decision::input_key; +use crate::consumption::constants::UNKNOWN_VERSION; +use crate::consumption::models::{ + ConsumedInput, ConsumptionHistory, InputFact, NodeFacts, NodeKey, +}; + +/// Each distinct direct input with its source hash, upstream execution ID or `unknown`. +pub fn consumed_inputs( + node: &NodeFacts, + (nodes, history): (&HashMap, &ConsumptionHistory), + this_build: &HashMap, +) -> Vec { + let mut inputs: Vec = node + .inputs + .iter() + .map(|input| { + let (kind, name) = input_key(input); + let version: Option = match input { + InputFact::Source { version, .. } => version.clone(), + InputFact::Node(upstream) => this_build.get(upstream).cloned().or_else(|| { + nodes + .get(upstream) + .and_then(|facts| history.latest_complete(&facts.relation)) + .map(str::to_owned) + }), + InputFact::Opaque(_) => None, + }; + ConsumedInput { + kind: kind.to_owned(), + name, + version: version.unwrap_or_else(|| UNKNOWN_VERSION.to_owned()), + } + }) + .collect(); + inputs.sort(); + inputs.dedup_by(|left, right| left.kind == right.kind && left.name == right.name); + inputs +} diff --git a/crates/sqlbuild-state/src/consumption/main/consumption_history.rs b/crates/sqlbuild-state/src/consumption/main/consumption_history.rs new file mode 100644 index 0000000000..83ab8b9385 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/consumption_history.rs @@ -0,0 +1,10 @@ +//! The recent executions of each relation, rebuilt from rows read back. + +use crate::consumption::_helpers::history::build_history; +use crate::consumption::_helpers::rules::Shipped; +use crate::consumption::models::{ConsumptionHistory, ConsumptionRow}; + +/// Executions per relation from rows read with at most `limit` executions per relation. +pub fn consumption_history(rows: &[ConsumptionRow], limit: usize) -> ConsumptionHistory { + build_history(&Shipped, rows, limit) +} diff --git a/crates/sqlbuild-state/src/consumption/main/event_id.rs b/crates/sqlbuild-state/src/consumption/main/event_id.rs new file mode 100644 index 0000000000..4aadd86844 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/event_id.rs @@ -0,0 +1,19 @@ +//! A consumption row's identity: its execution, phase and the input it records. + +use sha2::{Digest, Sha256}; + +use crate::consumption::constants::EVENT_NAMESPACE; +use crate::consumption::models::ConsumedInput; + +/// SHA-256 of `(execution_id, phase, input_kind, input_name, input_version)`, so a retried append repeats it. +pub fn event_id(execution_id: &str, phase: &str, input: Option<&ConsumedInput>) -> String { + let identity = serde_json::json!([ + EVENT_NAMESPACE, + execution_id, + phase, + input.map(|input| input.kind.as_str()), + input.map(|input| input.name.as_str()), + input.map(|input| input.version.as_str()), + ]); + format!("{:x}", Sha256::digest(identity.to_string().as_bytes())) +} diff --git a/crates/sqlbuild-state/src/consumption/main/mod.rs b/crates/sqlbuild-state/src/consumption/main/mod.rs new file mode 100644 index 0000000000..2ed00b0cf7 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/mod.rs @@ -0,0 +1,8 @@ +pub mod changed_run_set; +pub mod consumed_inputs; +pub mod consumption_history; +pub mod event_id; +pub mod prune_executions_sql; +pub mod recent_executions_sql; +pub mod started_append; +pub mod succeeded_appends; diff --git a/crates/sqlbuild-state/src/consumption/main/prune_executions_sql.rs b/crates/sqlbuild-state/src/consumption/main/prune_executions_sql.rs new file mode 100644 index 0000000000..cd7e064f21 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/prune_executions_sql.rs @@ -0,0 +1,15 @@ +//! Prune old executions from one consumption table. + +use crate::consumption::constants::RECORDED_AT_COLUMN; + +/// Delete every row of executions ranked beyond the newest `retain` per relation, whole executions at a time. +pub fn prune_executions_sql(qualified_name: &str, retain: i64) -> String { + format!( + "DELETE FROM {table} WHERE execution_id IN (SELECT execution_id FROM (SELECT execution_id, \ +ROW_NUMBER() OVER (PARTITION BY target_database, target_schema, target_name ORDER BY started_at DESC, \ +execution_id DESC) AS execution_rank FROM (SELECT execution_id, target_database, target_schema, \ +target_name, MIN({RECORDED_AT_COLUMN}) AS started_at FROM {table} GROUP BY execution_id, \ +target_database, target_schema, target_name) AS executions) AS ranked WHERE execution_rank > {retain})", + table = qualified_name, + ) +} diff --git a/crates/sqlbuild-state/src/consumption/main/recent_executions_sql.rs b/crates/sqlbuild-state/src/consumption/main/recent_executions_sql.rs new file mode 100644 index 0000000000..9ffdb3b498 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/recent_executions_sql.rs @@ -0,0 +1,21 @@ +//! Read the recent executions of every relation in one consumption table. + +use crate::consumption::constants::{CONSUMPTION_COLUMN_TYPES, RECORDED_AT_COLUMN}; + +/// Every row of the newest `limit` executions per relation, timing each execution by its earliest row. +pub fn recent_executions_sql(qualified_name: &str, limit: usize) -> String { + let columns: Vec<&str> = CONSUMPTION_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .collect(); + format!( + "SELECT {columns} FROM {table} WHERE execution_id IN (SELECT execution_id FROM (\ +SELECT execution_id, ROW_NUMBER() OVER (PARTITION BY target_database, target_schema, target_name \ +ORDER BY started_at DESC, execution_id DESC) AS execution_rank FROM (SELECT execution_id, \ +target_database, target_schema, target_name, MIN({RECORDED_AT_COLUMN}) AS started_at FROM {table} \ +GROUP BY execution_id, target_database, target_schema, target_name) AS executions) AS ranked \ +WHERE execution_rank <= {limit})", + columns = columns.join(", "), + table = qualified_name, + ) +} diff --git a/crates/sqlbuild-state/src/consumption/main/started_append.rs b/crates/sqlbuild-state/src/consumption/main/started_append.rs new file mode 100644 index 0000000000..70df769893 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/started_append.rs @@ -0,0 +1,24 @@ +//! The row appended before a model or seed writes. + +use crate::consumption::_helpers::decision::input_key; +use crate::consumption::constants::STARTED_PHASE; +use crate::consumption::main::event_id::event_id; +use crate::consumption::models::{ConsumptionAppend, NodeFacts}; + +/// The `started` row of one execution of `node`; its input count is the node's distinct direct inputs. +pub fn started_append(node: &NodeFacts, execution_id: &str, run_id: &str) -> ConsumptionAppend { + let mut keys: Vec<(&str, String)> = node.inputs.iter().map(input_key).collect(); + keys.sort(); + keys.dedup(); + ConsumptionAppend { + event_id: event_id(execution_id, STARTED_PHASE, None), + execution_id: execution_id.to_owned(), + phase: STARTED_PHASE.to_owned(), + run_id: run_id.to_owned(), + node: node.key.clone(), + relation: node.relation.clone(), + version_hash: node.version_hash.clone(), + input_count: i64::try_from(keys.len()).unwrap_or(i64::MAX), + input: None, + } +} diff --git a/crates/sqlbuild-state/src/consumption/main/succeeded_appends.rs b/crates/sqlbuild-state/src/consumption/main/succeeded_appends.rs new file mode 100644 index 0000000000..c68cec364e --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/succeeded_appends.rs @@ -0,0 +1,28 @@ +//! The rows appended after a model or seed's write succeeded. + +use crate::consumption::constants::SUCCEEDED_PHASE; +use crate::consumption::main::event_id::event_id; +use crate::consumption::models::{ConsumedInput, ConsumptionAppend, NodeFacts}; + +/// One `succeeded` row per consumed input, or one input-less row when there are none. +pub fn succeeded_appends( + node: &NodeFacts, + (execution_id, run_id): (&str, &str), + inputs: &[ConsumedInput], +) -> Vec { + let row = |input: Option<&ConsumedInput>| ConsumptionAppend { + event_id: event_id(execution_id, SUCCEEDED_PHASE, input), + execution_id: execution_id.to_owned(), + phase: SUCCEEDED_PHASE.to_owned(), + run_id: run_id.to_owned(), + node: node.key.clone(), + relation: node.relation.clone(), + version_hash: node.version_hash.clone(), + input_count: i64::try_from(inputs.len()).unwrap_or(i64::MAX), + input: input.cloned(), + }; + if inputs.is_empty() { + return vec![row(None)]; + } + inputs.iter().map(|input| row(Some(input))).collect() +} diff --git a/crates/sqlbuild-state/src/consumption/mod.rs b/crates/sqlbuild-state/src/consumption/mod.rs new file mode 100644 index 0000000000..f01181f853 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-state/src/consumption/models.rs b/crates/sqlbuild-state/src/consumption/models.rs new file mode 100644 index 0000000000..d9f9c96cba --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/models.rs @@ -0,0 +1,122 @@ +//! Consumption rows, the per-node facts `--changed` decides from, and its run reasons. + +/// A node identity: its resource kind, such as `model` or `seed`, and its name. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct NodeKey { + pub kind: String, + pub name: String, +} + +/// The relation a node writes, as the state table keys it. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct RelationKey { + pub database: Option, + pub schema: String, + pub name: String, +} + +/// One input an execution consumed: its kind, name and the version it read. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct ConsumedInput { + pub kind: String, + pub name: String, + pub version: String, +} + +/// One row to append; the warehouse clock fills `recorded_at`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConsumptionAppend { + pub event_id: String, + pub execution_id: String, + pub phase: String, + pub run_id: String, + pub node: NodeKey, + pub relation: RelationKey, + pub version_hash: String, + pub input_count: i64, + pub input: Option, +} + +/// One row read back, with its warehouse time in microseconds since the epoch. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConsumptionRow { + pub row: ConsumptionAppend, + pub recorded_at: i64, +} + +/// A direct input of a node as this build sees it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum InputFact { + /// A source and its observed `data_version_hash`, or None when unknown. + Source { + name: String, + version: Option, + }, + /// An upstream model or seed. + Node(NodeKey), + /// An input whose version cannot be known, such as a Python node. + Opaque(String), +} + +/// Why the planner already runs a node, whatever its records say. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ForcedRun { + /// The planner detected a change of the node itself, named by its change kind. + PlannerChange(String), + FullRefresh, + RelationMissing, + CursorInterval, + /// The compiler cannot enumerate what the node reads. + InputsNotEnumerable, +} + +/// One node of the selection or its ancestry, as `--changed` decides from it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NodeFacts { + pub key: NodeKey, + pub relation: RelationKey, + pub version_hash: String, + pub selected: bool, + pub forced: Vec, + pub inputs: Vec, +} + +/// Why `--changed` runs a node. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RunReason { + Forced(ForcedRun), + NoRecord, + Unfinished, + AmbiguousOrder, + DefinitionChanged, + InputsChanged, + InputChanged(String), + InputUnknown(String), + UpstreamRuns(NodeKey), +} + +/// One execution of a relation rebuilt from its deduplicated rows. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Execution { + pub execution_id: String, + /// The earliest warehouse time of its rows. + pub started_at: i64, + /// Whether its `succeeded` rows are present, consistent and complete. + pub complete: bool, + pub version_hash: Option, + /// Input versions of a complete execution, keyed by `(kind, name)`. + pub inputs: Vec, +} + +/// The recent executions of each relation, newest first, as `--changed` reads them. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ConsumptionHistory { + pub(crate) relations: std::collections::HashMap, +} + +/// One relation's executions, newest first, and whether older ones were left unread. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct RelationHistory { + pub(crate) executions: Vec, + pub(crate) truncated: bool, +} diff --git a/crates/sqlbuild-state/src/consumption/tests.rs b/crates/sqlbuild-state/src/consumption/tests.rs new file mode 100644 index 0000000000..5ba8b6e3a0 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_simulation.rs"] +mod test_simulation; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-state/src/consumption/tests/helpers.rs b/crates/sqlbuild-state/src/consumption/tests/helpers.rs new file mode 100644 index 0000000000..04d74a8125 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/tests/helpers.rs @@ -0,0 +1,577 @@ +//! A seeded simulation of builds against a warehouse, with an oracle that knows what each model read. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use crate::consumption::_helpers::decision::decide; +use crate::consumption::_helpers::history::build_history; +use crate::consumption::_helpers::rules::DecisionRules; +use crate::consumption::constants::RECENT_EXECUTIONS; +use crate::consumption::main::consumed_inputs::consumed_inputs; +use crate::consumption::main::started_append::started_append; +use crate::consumption::main::succeeded_appends::succeeded_appends; +use crate::consumption::models::{ + ConsumptionAppend, ConsumptionHistory, ConsumptionRow, ForcedRun, InputFact, NodeFacts, + NodeKey, RelationKey, RunReason, +}; +use crate::consumption::tests::test_types::{ + Content, Derivation, Input, LatestRowTime, Miss, NoPlannerGuard, NoUpstreamRecursion, Node, + Random, Source, TransitiveOnly, World, Writer, +}; + +impl Random { + pub(crate) fn new(seed: u64) -> Self { + Self(seed) + } + + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z: u64 = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + fn below(&mut self, bound: u64) -> u64 { + self.next() % bound + } + + fn chance(&mut self, percent: u64) -> bool { + self.below(100) < percent + } +} + +const SECOND: i64 = 1_000_000; +const SIMULATED_CASES: u64 = 400; +const SIMULATED_BUILDS: usize = 14; + +impl World { + pub(crate) fn new(random: &mut Random) -> Self { + let source_count: usize = 2 + usize::try_from(random.below(2)).unwrap_or(0); + let node_count: usize = 5 + usize::try_from(random.below(5)).unwrap_or(0); + let sources: Vec = (0..source_count) + .map(|_| Source { + version: 1, + observed: random.chance(85), + monotonic: random.chance(50), + }) + .collect(); + let mut nodes: Vec = Vec::new(); + for index in 0..node_count { + let mut inputs: BTreeSet = BTreeSet::new(); + for source in 0..source_count { + if random.chance(35) { + inputs.insert(Input::Source(source)); + } + } + for upstream in 0..index { + if random.chance(40) { + inputs.insert(Input::Node(upstream)); + } + } + nodes.push(Node { + inputs: inputs.into_iter().collect(), + definition: 1, + schema: usize::try_from(random.below(2)).unwrap_or(0), + nondeterministic: random.chance(25), + enumerable: random.chance(92), + content: Content::Missing, + fingerprint: None, + }); + } + Self { + sources, + nodes, + rows: Vec::new(), + clock: 1_000 * SECOND, + nonce: 0, + flushed: Vec::new(), + trace: Vec::new(), + } + } + + fn key(index: usize) -> NodeKey { + NodeKey { + kind: "model".to_owned(), + name: format!("m{index}"), + } + } + + fn relation(&self, index: usize) -> RelationKey { + RelationKey { + database: None, + schema: format!("s{}", self.nodes[index].schema), + name: format!("m{index}"), + } + } + + fn content_id(&self, input: Input) -> u64 { + match input { + Input::Source(source) => self.sources[source].version, + Input::Node(node) => match &self.nodes[node].content { + Content::Missing => 0, + Content::Partial => 1, + Content::Data(derivation) => 1_000 + derivation.nonce, + }, + } + } + + fn fresh_own(&self, index: usize) -> bool { + let node: &Node = &self.nodes[index]; + match &node.content { + Content::Data(derivation) => { + derivation.definition == node.definition + && derivation + .inputs + .iter() + .all(|(input, read)| self.content_id(*input) == *read) + } + _ => false, + } + } + + /// Fresh and every upstream, transitively, fresh: what a skip requires. + fn fresh_closure(&self, index: usize) -> bool { + self.fresh_own(index) + && self.nodes[index].inputs.iter().all(|input| match input { + Input::Node(upstream) => self.fresh_closure(*upstream), + Input::Source(_) => true, + }) + } + + fn change_sources(&mut self, random: &mut Random) { + for (index, source) in self.sources.iter_mut().enumerate() { + if !random.chance(30) { + continue; + } + let previous: u64 = source.version; + source.version = if source.monotonic || random.chance(50) { + source.version + 1 + } else { + 1 + random.below(source.version.max(1)) + }; + self.trace + .push(format!("source s{index}: {previous} -> {}", source.version)); + } + } + + fn change_definitions(&mut self, random: &mut Random) { + for (index, node) in self.nodes.iter_mut().enumerate() { + if random.chance(12) { + node.definition += 1; + self.trace + .push(format!("definition m{index} -> {}", node.definition)); + } + } + } + + /// The reader's SQL: rows of the newest executions per relation by earliest row time. + fn read_rows(&self) -> Vec { + let mut started: BTreeMap<(RelationKey, String), i64> = BTreeMap::new(); + for row in &self.rows { + let entry = started + .entry((row.row.relation.clone(), row.row.execution_id.clone())) + .or_insert(row.recorded_at); + *entry = (*entry).min(row.recorded_at); + } + let mut by_relation: BTreeMap> = BTreeMap::new(); + for ((relation, execution), time) in started { + by_relation + .entry(relation) + .or_default() + .push((time, execution)); + } + let mut kept: BTreeSet = BTreeSet::new(); + for executions in by_relation.values_mut() { + executions.sort_by(|left, right| right.cmp(left)); + kept.extend( + executions + .iter() + .take(RECENT_EXECUTIONS) + .map(|(_, execution)| execution.clone()), + ); + } + self.rows + .iter() + .filter(|row| kept.contains(&row.row.execution_id)) + .cloned() + .collect() + } + + /// The janitor's SQL: keep the newest `retain` executions per relation. + fn prune(&mut self, retain: usize) { + let mut started: BTreeMap<(RelationKey, String), i64> = BTreeMap::new(); + for row in &self.rows { + let entry = started + .entry((row.row.relation.clone(), row.row.execution_id.clone())) + .or_insert(row.recorded_at); + *entry = (*entry).min(row.recorded_at); + } + let mut by_relation: BTreeMap> = BTreeMap::new(); + for ((relation, execution), time) in started { + by_relation + .entry(relation) + .or_default() + .push((time, execution)); + } + let mut dropped: BTreeSet = BTreeSet::new(); + for executions in by_relation.values_mut() { + executions.sort_by(|left, right| right.cmp(left)); + dropped.extend( + executions + .iter() + .skip(retain) + .map(|(_, execution)| execution.clone()), + ); + } + self.rows + .retain(|row| !dropped.contains(&row.row.execution_id)); + self.trace.push(format!("prune to {retain}")); + } + + fn append(&mut self, rows: &[ConsumptionAppend]) { + for row in rows { + self.clock += SECOND / 10; + self.rows.push(ConsumptionRow { + row: row.clone(), + recorded_at: self.clock, + }); + } + } + + fn facts( + &self, + selected: &BTreeSet, + observed: &[Option], + full_refresh: bool, + ) -> Vec { + (0..self.nodes.len()) + .map(|index| { + let node: &Node = &self.nodes[index]; + let mut forced: Vec = Vec::new(); + if full_refresh { + forced.push(ForcedRun::FullRefresh); + } + if node.content == Content::Missing { + forced.push(ForcedRun::RelationMissing); + } else if node.fingerprint != Some(node.definition) { + forced.push(ForcedRun::PlannerChange("query_changed".to_owned())); + } + if !node.enumerable { + forced.push(ForcedRun::InputsNotEnumerable); + } + NodeFacts { + key: Self::key(index), + relation: self.relation(index), + version_hash: format!("d{}", node.definition), + selected: selected.contains(&index), + forced, + inputs: node + .inputs + .iter() + .map(|input| match input { + Input::Source(source) => InputFact::Source { + name: format!("s{source}"), + version: observed[*source].map(|version| format!("v{version}")), + }, + Input::Node(upstream) => InputFact::Node(Self::key(*upstream)), + }) + .collect(), + } + }) + .collect() + } + + /// One build; under `--changed` the oracle checks every skip before anything executes. + pub(crate) fn build( + &mut self, + (rules, writer): (&R, Writer), + random: &mut Random, + step: usize, + ) -> Result<(), Miss> { + self.clock += i64::try_from(random.below(20)).unwrap_or(0) * SECOND; + self.late_duplicate(random); + self.change_sources(random); + self.change_definitions(random); + let changed: bool = random.chance(60); + let selected: BTreeSet = if random.chance(50) { + (0..self.nodes.len()).collect() + } else { + (0..self.nodes.len()) + .filter(|_| random.chance(50)) + .collect() + }; + let full_refresh: bool = random.chance(5); + let observed: Vec> = self + .sources + .iter() + .map(|source| source.observed.then_some(source.version)) + .collect(); + let facts: Vec = self.facts(&selected, &observed, full_refresh); + let history: ConsumptionHistory = + build_history(rules, &self.read_rows(), RECENT_EXECUTIONS); + let decided: Vec<(NodeKey, Option)> = decide(rules, &facts, &history); + let run: Vec = (0..self.nodes.len()) + .filter(|index| selected.contains(index)) + .filter(|index| { + !changed + || decided + .iter() + .any(|(key, reason)| *key == Self::key(*index) && reason.is_some()) + }) + .collect(); + self.trace.push(format!( + "step {step}: {} select {:?} run {:?}{}", + if changed { "--changed" } else { "plain" }, + selected, + run, + if full_refresh { " full refresh" } else { "" } + )); + if changed { + for index in &selected { + if run.contains(index) { + continue; + } + if !self.fresh_closure(*index) { + return Err(Miss { + step, + node: *index, + why: "skipped a model whose data or an upstream's is stale", + }); + } + if self.nodes[*index].fingerprint != Some(self.nodes[*index].definition) { + return Err(Miss { + step, + node: *index, + why: "skipped a model with a pending replay", + }); + } + } + } + self.execute((&facts, &history), &run, (writer, random)); + if random.chance(15) { + let retain: usize = 1 + usize::try_from(random.below(3)).unwrap_or(0); + self.prune(retain); + } + Ok(()) + } + + fn execute( + &mut self, + (facts, history): (&[NodeFacts], &ConsumptionHistory), + run: &[usize], + (writer, random): (Writer, &mut Random), + ) { + let by_key: HashMap = facts + .iter() + .map(|facts| (facts.key.clone(), facts.clone())) + .collect(); + let executions: HashMap = run + .iter() + .map(|index| { + self.nonce += 1; + (Self::key(*index), format!("e{}", self.nonce)) + }) + .collect(); + let mut failed_start: BTreeSet = BTreeSet::new(); + for schema in 0..2 { + let batch: Vec = run + .iter() + .filter(|index| self.nodes[**index].schema == schema) + .map(|index| started_append(&facts[*index], &executions[&Self::key(*index)], "run")) + .collect(); + if batch.is_empty() || writer == Writer::NoStartedRow { + continue; + } + if random.chance(4) { + self.trace + .push(format!("started append fails in s{schema}")); + failed_start.extend( + run.iter() + .filter(|index| self.nodes[**index].schema == schema), + ); + continue; + } + self.append(&batch); + } + let mut done: BTreeSet = BTreeSet::new(); + let mut pending: Vec> = vec![Vec::new(), Vec::new()]; + for index in run.iter().copied() { + let blocked: bool = self.nodes[index].inputs.iter().any(|input| match input { + Input::Node(upstream) => run.contains(upstream) && !done.contains(upstream), + Input::Source(_) => false, + }); + if blocked || failed_start.contains(&index) { + continue; + } + if random.chance(5) { + self.trace.push(format!("crash while writing m{index}")); + self.nodes[index].content = Content::Partial; + return; + } + if random.chance(6) { + self.trace.push(format!("m{index} fails")); + if random.chance(50) { + self.nodes[index].content = Content::Partial; + } + continue; + } + self.late_load(random); + self.nonce += 1; + let derivation = Derivation { + definition: self.nodes[index].definition, + inputs: self.nodes[index] + .inputs + .iter() + .map(|input| (*input, self.content_id(*input))) + .collect(), + nonce: if self.nodes[index].nondeterministic { + self.nonce + } else { + self.deterministic_nonce(index) + }, + }; + self.nodes[index].content = Content::Data(derivation); + if random.chance(85) { + self.nodes[index].fingerprint = Some(self.nodes[index].definition); + } else { + self.trace + .push(format!("fingerprint write fails for m{index}")); + } + if random.chance(4) { + self.trace.push(format!("crash after writing m{index}")); + return; + } + done.insert(index); + let key: NodeKey = Self::key(index); + let inputs = consumed_inputs(&facts[index], (&by_key, history), &executions); + pending[self.nodes[index].schema].extend(succeeded_appends( + &facts[index], + (&executions[&key], "run"), + &inputs, + )); + if random.chance(40) && !self.flush(&mut pending, random) { + return; + } + } + self.flush(&mut pending, random); + } + + /// A retried append of a recent batch lands again, later, with the same event IDs. + fn late_duplicate(&mut self, random: &mut Random) { + if self.flushed.is_empty() || !random.chance(15) { + return; + } + let recent: u64 = self.flushed.len().min(3) as u64; + let batch: usize = + self.flushed.len() - 1 - usize::try_from(random.below(recent)).unwrap_or(0); + let rows: Vec = self.flushed[batch].clone(); + let retained: bool = rows.iter().all(|duplicate| { + self.rows + .iter() + .any(|row| row.row.execution_id == duplicate.execution_id) + }); + if retained { + self.trace.push(format!("late duplicate of batch {batch}")); + self.append(&rows); + } + } + + /// A deterministic model's content depends only on what it read. + fn deterministic_nonce(&self, index: usize) -> u64 { + let node: &Node = &self.nodes[index]; + let mut value: u64 = u64::from(node.definition) * 1_000_003 + index as u64; + for input in &node.inputs { + value = value.wrapping_mul(31).wrapping_add(self.content_id(*input)); + } + value % 1_000_000_000 + } + + /// A monotonic source may load new data between observation and a model's read. + fn late_load(&mut self, random: &mut Random) { + for (index, source) in self.sources.iter_mut().enumerate() { + if source.monotonic && random.chance(3) { + source.version += 1; + self.trace + .push(format!("late load of s{index} -> {}", source.version)); + } + } + } + + /// Append each schema's pending rows as one statement; a crash loses that statement whole. + fn flush(&mut self, pending: &mut [Vec], random: &mut Random) -> bool { + for batch in pending.iter_mut() { + if batch.is_empty() { + continue; + } + if random.chance(3) { + self.trace + .push("crash during the succeeded append".to_owned()); + return false; + } + let rows: Vec = std::mem::take(batch); + self.append(&rows); + self.flushed.push(rows); + } + true + } +} + +/// Run `cases` seeded worlds of `steps` builds and return the first miss, with its seed and trace. +pub(crate) fn first_miss( + (rules, writer): (&R, Writer), + (cases, steps): (u64, usize), +) -> Option<(u64, Miss, Vec)> { + (0..cases).find_map(|seed| world_miss((rules, writer), seed, steps)) +} + +/// The first miss of one seeded world. +fn world_miss( + (rules, writer): (&R, Writer), + seed: u64, + steps: usize, +) -> Option<(u64, Miss, Vec)> { + let mut random = Random::new(seed); + let mut world = World::new(&mut random); + let miss: Miss = + (0..steps).find_map(|step| world.build((rules, writer), &mut random, step).err())?; + Some((seed, miss, world.trace)) +} + +impl DecisionRules for TransitiveOnly { + fn compares_node_inputs(&self) -> bool { + false + } +} + +impl DecisionRules for LatestRowTime { + fn execution_time(&self, earliest: i64, latest: i64) -> i64 { + let _ = earliest; + latest + } +} + +impl DecisionRules for NoUpstreamRecursion { + fn follows_upstream(&self) -> bool { + false + } +} + +impl DecisionRules for NoPlannerGuard { + fn honours(&self, forced: &ForcedRun) -> bool { + !matches!(forced, ForcedRun::PlannerChange(_)) + } +} + +/// The first miss of `rules` over the standard cases, rendered for an assertion message. +pub(crate) fn found( + rules: &R, + writer: Writer, +) -> Option<(u64, String, Vec)> { + first_miss((rules, writer), (SIMULATED_CASES, SIMULATED_BUILDS)).map(|(seed, miss, trace)| { + ( + seed, + format!("step {} m{}: {}", miss.step, miss.node, miss.why), + trace, + ) + }) +} diff --git a/crates/sqlbuild-state/src/consumption/tests/test_simulation.rs b/crates/sqlbuild-state/src/consumption/tests/test_simulation.rs new file mode 100644 index 0000000000..c515a1f587 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/tests/test_simulation.rs @@ -0,0 +1,50 @@ +use crate::consumption::_helpers::rules::Shipped; +use crate::consumption::tests::helpers::found; +use crate::consumption::tests::test_types::{ + LatestRowTime, NoPlannerGuard, NoUpstreamRecursion, SimulationTestCase, TransitiveOnly, Writer, +}; + +#[test] +fn given_random_builds_crashes_and_appends_when_deciding_changed_then_only_broken_rules_miss() { + let test_cases = [ + SimulationTestCase { + description: "the shipped rules never skip a stale model or a pending replay", + miss: found(&Shipped, Writer::Shipped), + expected_miss: false, + }, + SimulationTestCase { + description: "without the started row a crash mid-write is skipped", + miss: found(&Shipped, Writer::NoStartedRow), + expected_miss: true, + }, + SimulationTestCase { + description: "comparing sources only misses a rebuilt upstream", + miss: found(&TransitiveOnly, Writer::Shipped), + expected_miss: true, + }, + SimulationTestCase { + description: "timing executions by their latest row lets a late duplicate hide a crash", + miss: found(&LatestRowTime, Writer::Shipped), + expected_miss: true, + }, + SimulationTestCase { + description: "without upstream recursion a stale unselected upstream is ignored", + miss: found(&NoUpstreamRecursion, Writer::Shipped), + expected_miss: true, + }, + SimulationTestCase { + description: "without the planner guard a pending replay is skipped", + miss: found(&NoPlannerGuard, Writer::Shipped), + expected_miss: true, + }, + ]; + for test_case in test_cases { + assert_eq!( + test_case.miss.is_some(), + test_case.expected_miss, + "{}: {:?}", + test_case.description, + test_case.miss + ); + } +} diff --git a/crates/sqlbuild-state/src/consumption/tests/test_types.rs b/crates/sqlbuild-state/src/consumption/tests/test_types.rs new file mode 100644 index 0000000000..5d74e04f01 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/tests/test_types.rs @@ -0,0 +1,92 @@ +//! Simulated worlds, the oracle's view of their data, and broken decision rules. + +use crate::consumption::models::ConsumptionAppend; +use crate::consumption::models::ConsumptionRow; + +/// SplitMix64, so every case replays from its seed. +pub(crate) struct Random(pub(crate) u64); + +/// The broken writer variant: whether `started` rows are appended at all. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Writer { + Shipped, + NoStartedRow, +} + +/// What a relation holds: nothing, a partial write, or data derived from known inputs. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Content { + Missing, + Partial, + Data(Derivation), +} + +/// Exactly what one successful write read: its definition, each input's content and a nonce. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Derivation { + pub(crate) definition: u32, + pub(crate) inputs: Vec<(Input, u64)>, + pub(crate) nonce: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum Input { + Source(usize), + Node(usize), +} + +pub(crate) struct Source { + pub(crate) version: u64, + /// Whether its freshness signal exists; without one the version is unknown. + pub(crate) observed: bool, + /// Monotonic versions only increase; others may revert to any earlier value. + pub(crate) monotonic: bool, +} + +pub(crate) struct Node { + pub(crate) inputs: Vec, + pub(crate) definition: u32, + pub(crate) schema: usize, + pub(crate) nondeterministic: bool, + pub(crate) enumerable: bool, + pub(crate) content: Content, + /// The planner's recorded definition: written best effort after a successful write. + pub(crate) fingerprint: Option, +} + +/// One simulated project and warehouse. +pub(crate) struct World { + pub(crate) sources: Vec, + pub(crate) nodes: Vec, + pub(crate) rows: Vec, + pub(crate) clock: i64, + pub(crate) nonce: u64, + pub(crate) flushed: Vec>, + pub(crate) trace: Vec, +} + +/// A skip the oracle rejects. +#[derive(Debug)] +pub(crate) struct Miss { + pub(crate) step: usize, + pub(crate) node: usize, + pub(crate) why: &'static str, +} + +/// Compares only source inputs, as if transitive sources were all a model recorded. +pub(crate) struct TransitiveOnly; + +/// Times an execution by its latest row, so a late duplicate looks newest. +pub(crate) struct LatestRowTime; + +/// Ignores stale upstreams, selected or not. +pub(crate) struct NoUpstreamRecursion; + +/// Ignores the planner's own change detection. +pub(crate) struct NoPlannerGuard; + +pub(crate) struct SimulationTestCase { + pub(crate) description: &'static str, + pub(crate) miss: Option<(u64, String, Vec)>, + pub(crate) expected_miss: bool, +} diff --git a/crates/sqlbuild-state/src/lib.rs b/crates/sqlbuild-state/src/lib.rs index 571e27e44c..d0fcba2598 100644 --- a/crates/sqlbuild-state/src/lib.rs +++ b/crates/sqlbuild-state/src/lib.rs @@ -2,6 +2,7 @@ #![forbid(unsafe_code)] +pub mod consumption; pub mod events; pub mod execution_history; pub mod freshness; From 601b8cfe94a72e4ed76c52a174ade5856024dee3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 4 Nov 2026 18:53:03 +0000 Subject: [PATCH 2/8] wip: record consumed inputs in builds and add build --changed Adapters gain consumption-table DDL, pruning and a UTC warehouse clock; the build records started and succeeded rows and --changed skips unchanged models. Validated so far: cargo check and a local DuckDB smoke run only. --- .../src/janitor_planning/constants.rs | 4 +- .../sqlbuild-cli/src/argv/command_table.json | 11 + crates/sqlbuild-cli/src/argv/constants.rs | 3 +- crates/sqlbuild-cli/src/dispatch/constants.rs | 3 +- .../_helpers/build_command/execution.rs | 91 +++- .../src/bindings/_helpers/cli/registration.rs | 11 +- .../bindings/_helpers/consumption/facts.rs | 303 ++++++++++++++ .../src/bindings/_helpers/consumption/mod.rs | 4 + .../bindings/_helpers/consumption/recorder.rs | 373 +++++++++++++++++ .../_helpers/consumption/relation_reads.rs | 136 ++++++ .../bindings/_helpers/consumption/state_io.rs | 393 ++++++++++++++++++ .../_helpers/exported_constants/constants.rs | 9 +- .../_helpers/janitor/janitor_facts.rs | 9 +- .../src/bindings/_helpers/mod.rs | 1 + .../_helpers/planner_identity/identities.rs | 4 +- .../bindings/_helpers/records/constants.rs | 1 + src/sqlbuild/_native.pyi | 17 + .../adapter/contract/classes/base_adapter.py | 40 ++ .../adapter/contract/classes/consumption.py | 45 ++ .../contract/classes/duckdb_backed_adapter.py | 3 + .../contract/classes/strict_adapter.py | 31 ++ .../bigquery/classes/bigquery_adapter.py | 6 +- .../clickhouse/classes/clickhouse_adapter.py | 12 +- .../databricks/classes/databricks_adapter.py | 6 +- .../adapters/duckdb/classes/duckdb_adapter.py | 3 +- .../motherduck/classes/motherduck_adapter.py | 3 +- .../postgres/classes/postgres_adapter.py | 22 +- .../snowflake/classes/snowflake_adapter.py | 16 +- .../adapters/sqlite/classes/sqlite_adapter.py | 6 +- .../sqlserver/classes/sqlserver_adapter.py | 46 +- .../executor/build/classes/build_scheduler.py | 21 +- src/sqlbuild/executor/build/models.py | 4 +- src/sqlbuild/executor/build/types.py | 8 +- .../executor/build/_helpers/test_scheduler.py | 4 +- 34 files changed, 1607 insertions(+), 42 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/consumption/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/consumption/state_io.rs create mode 100644 src/sqlbuild/adapter/contract/classes/consumption.py diff --git a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs index 6f0b096877..29ab3e1097 100644 --- a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs +++ b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs @@ -1,16 +1,18 @@ //! Janitor planning names, prefixes and skip reasons. use sqlbuild_core::constants::COLUMN_MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::CONSUMPTION_TABLE_NAME; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use sqlbuild_core::constants::MIGRATION_TABLE_NAME; use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; /// SQLBuild state tables the janitor never acts on. -pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 9] = [ +pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 10] = [ FINGERPRINT_TABLE_NAME, "_sqlbuild_microbatches", SOURCE_FRESHNESS_TABLE_NAME, + CONSUMPTION_TABLE_NAME, "_sqlbuild_node_results", "_sqlbuild_audit_results", "_sqlbuild_janitor_events", diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index d352ae8208..4ca62b994c 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -2360,6 +2360,17 @@ "default": false, "group": 1 }, + { + "kind": "store_true", + "options": [ + "--changed" + ], + "dest": "changed", + "nargs": 0, + "const": true, + "default": false, + "group": 1 + }, { "kind": "store_true", "options": [ diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 3c079c4086..e4a1b23bfc 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -71,7 +71,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 184] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 185] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), @@ -126,6 +126,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 184] = [ ("no_color", NamespaceDefault::Bool(false)), ("fail_fast", NamespaceDefault::Bool(false)), ("full_refresh", NamespaceDefault::Bool(false)), + ("changed", NamespaceDefault::Bool(false)), ("as_target", NamespaceDefault::None), ("include_python", NamespaceDefault::Bool(true)), ("load_sources", NamespaceDefault::None), diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 247242fd45..9e50dee08d 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -139,7 +139,7 @@ pub(crate) const PLAN_ROUTE: Route = routed( REQUEST_KEYWORD_SHAPE, ); -const BUILD_FIELDS: [Field; 35] = [ +const BUILD_FIELDS: [Field; 36] = [ PROJECT_DIR, NO_SQL_VALIDATION, same("no_cache"), @@ -151,6 +151,7 @@ const BUILD_FIELDS: [Field; 35] = [ NO_COLOR, same("fail_fast"), same("full_refresh"), + same("changed"), same("load_sources"), attr("reload_sources", "reload"), same("include_python"), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index e92e997318..11fa400d02 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -31,6 +31,9 @@ use crate::bindings::_helpers::check::check_core::{ use crate::bindings::_helpers::check::check_results::write_check_results; use crate::bindings::_helpers::command_runtime::warehouses::describe_connection_warehouse; use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; +use crate::bindings::_helpers::consumption::recorder::{ + ConsumptionRecorder, Prepared, prepare as prepare_consumption, run_reason_text, +}; use crate::bindings::_helpers::cost_telemetry::context::CostContext; use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timestamp}; use crate::bindings::_helpers::freshness::publication::append_freshness_records; @@ -370,6 +373,20 @@ pub(crate) fn execute_build_plan<'py>( ), ], )?; + let prepared: Prepared<'py> = prepare_consumption( + (&plan, &project), + (&adapter, &connection_config), + ( + context.request_flag("changed")?, + context.request_flag("full_refresh")?, + ), + ( + native_field(&project, "run_id")?.str()?.extract()?, + native_field(&preparation.lifecycle, "on_node_complete")?, + ), + )?; + report_changed_decision(context, &prepared)?; + let recorder: &Bound<'py, ConsumptionRecorder> = &prepared.recorder; let connection: &Bound<'py, PyAny> = &preparation.connection_progress; let build_callbacks: Bound<'py, PyAny> = call_with( &host::entry(py, ExecutorHost::BuildCallbacks)?, @@ -379,10 +396,8 @@ pub(crate) fn execute_build_plan<'py>( "on_node_start", keyword_relay(native_field(callbacks, "on_node_start")?, &NODE_START)?, ), - ( - "on_node_complete", - native_field(&preparation.lifecycle, "on_node_complete")?, - ), + ("on_node_complete", recorder.getattr("on_node_complete")?), + ("before_node_write", recorder.getattr("before_node_write")?), ( "on_sub_progress", native_field(callbacks, "on_sub_progress")?, @@ -429,7 +444,10 @@ pub(crate) fn execute_build_plan<'py>( let initial_state: Bound<'py, PyAny> = call_with( &host::entry(py, ExecutorHost::BuildInitialState)?, &[ - ("precompleted_keys", lifecycle.precompleted_keys(py)?), + ( + "precompleted_keys", + with_skipped_keys(&plan, lifecycle.precompleted_keys(py)?, &prepared)?, + ), ("initial_load_results", lifecycle.ingress_load_results(py)), ("initial_failed_keys", lifecycle.blocked_keys(py)?), ], @@ -447,6 +465,9 @@ pub(crate) fn execute_build_plan<'py>( ("initial_state", initial_state), ], )?; + for warning in recorder.get().finish(py, &result)? { + emit(&context.stream, &format!("warning: {warning}\n"))?; + } let native_result = BuildExecutionResult::read(&result)?; let successful_models: Vec = native_result .model_results @@ -468,6 +489,66 @@ pub(crate) fn execute_build_plan<'py>( }) } +/// Report what `--changed` runs and skips, and any warnings from reading records. +fn report_changed_decision(context: &BuildContext<'_>, prepared: &Prepared<'_>) -> PyResult<()> { + for warning in &prepared.warnings { + emit(&context.stream, &format!("warning: {warning}\n"))?; + } + let Some(decision) = &prepared.decision else { + return Ok(()); + }; + emit( + &context.stream, + &format!( + "Changed: {} of {} selected models and seeds run; {} unchanged are skipped.\n", + decision.run, + decision.run + decision.skipped.len(), + decision.skipped.len() + ), + )?; + if context.request_flag("verbose")? { + for (key, reason) in &decision.reasons { + emit( + &context.stream, + &format!( + " runs {} {}: {}\n", + key.kind, + key.name, + run_reason_text(reason) + ), + )?; + } + } + Ok(()) +} + +/// `precompleted` with the plan keys of the models and seeds `--changed` skips. +fn with_skipped_keys<'py>( + plan: &Bound<'py, PyAny>, + precompleted: Bound<'py, PyAny>, + prepared: &Prepared<'py>, +) -> PyResult> { + let Some(decision) = &prepared.decision else { + return Ok(precompleted); + }; + let py: Python<'py> = plan.py(); + let mut keys: Vec> = precompleted.try_iter()?.collect::>()?; + for (entries, kind) in [("model_entries", "model"), ("seed_entries", "seed")] { + for entry in native_field(plan, entries)?.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + let name: String = native_field(&entry, "name")?.extract()?; + if decision + .skipped + .iter() + .any(|key| key.kind == kind && key.name == name) + { + keys.push(native_field(&entry, "key")?); + } + } + } + frozenset(py, &keys) +} + fn check_runtime<'py>( context: &BuildContext<'py>, pipeline: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 7c15b912eb..fe8fa61bb3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -8,11 +8,11 @@ use crate::bindings::_helpers::cli::{ }; use crate::bindings::_helpers::{ audit_command, build_command, build_progress, change_selection, check, clone, command_runtime, - compute_logs, contract_command, cost_command, cost_report, cost_telemetry, dbt_command, - dbt_events, dbt_interop, debug_command, diff_report, event_exporting, execution_history, - format_command, freshness, init_command, janitor, load, output_capture, plan_command, - presentation_text, query_command, recall, result_output, review_command, scenario_command, - seed_command, sql_test_command, workspace_commands, + compute_logs, consumption, contract_command, cost_command, cost_report, cost_telemetry, + dbt_command, dbt_events, dbt_interop, debug_command, diff_report, event_exporting, + execution_history, format_command, freshness, init_command, janitor, load, output_capture, + plan_command, presentation_text, query_command, recall, result_output, review_command, + scenario_command, seed_command, sql_test_command, workspace_commands, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -28,6 +28,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { janitor::registration::register(module)?; plan_command::registration::register(module)?; freshness::registration::register(module)?; + consumption::state_io::register(module)?; clone::registration::register(module)?; compute_logs::registration::register(module)?; execution_history::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs new file mode 100644 index 0000000000..2ad56b6031 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs @@ -0,0 +1,303 @@ +//! The facts `--changed` and the consumption recorder read from the plan and compiled project. + +use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_core::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; +use sqlbuild_state::consumption::models::{ForcedRun, InputFact, NodeFacts, NodeKey, RelationKey}; + +use crate::bindings::_helpers::compiled_views::{ + function_reads, model_reads, project_reads, seed_reads, +}; +use crate::bindings::_helpers::consumption::relation_reads::{Locations, model_reads_undeclared}; +use crate::bindings::_helpers::planner_identity::hashing::{graph_key, model_query_hash}; +use crate::bindings::_helpers::planner_identity::identities::hook_version_hashes; +use crate::bindings::_helpers::planner_identity::metadata::{ + function_local_hashes, model_version_identity_metadata_json, seed_identity, +}; +use crate::bindings::_helpers::records::fields::field as native_field; + +const MODEL: &str = "model"; +const SEED: &str = "seed"; +const SOURCE: &str = "source"; +const FUNCTION_KINDS: [&str; 2] = ["udf", "table_fn"]; +/// Plan reasons that mean the planner sees no change of the model itself. +const UNCHANGED_REASONS: [&str; 2] = ["no_change", "normal_incremental"]; +const FIRST_RUN: &str = "first_run"; +const FULL_REFRESH: &str = "full_refresh"; +const SKIP_ACTION: &str = "skip"; + +/// The selection's nodes and their ancestry, and the selected ones that execute. +pub(crate) struct BuildFacts { + pub(crate) nodes: Vec, + pub(crate) runnable: BTreeSet, +} + +/// One compiled model or seed of the ancestry walk. +enum Compiled<'py> { + Model(Bound<'py, PyAny>), + Seed(Bound<'py, PyAny>), +} + +fn node_key(kind: &str, name: &str) -> NodeKey { + NodeKey { + kind: kind.to_owned(), + name: name.to_owned(), + } +} + +fn enum_text(value: &Bound<'_, PyAny>) -> PyResult { + let value: Bound<'_, PyAny> = value.getattr_opt("value")?.unwrap_or_else(|| value.clone()); + value.str()?.extract() +} + +fn relation(destination: &Bound<'_, PyAny>) -> PyResult> { + let schema: Bound<'_, PyAny> = native_field(destination, "schema")?; + if schema.is_none() { + return Ok(None); + } + let database: Bound<'_, PyAny> = native_field(destination, "database")?; + Ok(Some(RelationKey { + database: if database.is_none() { + None + } else { + Some(database.extract()?) + }, + schema: schema.extract()?, + name: native_field(destination, "name")?.extract()?, + })) +} + +/// Each observed source's `data_version_hash`; sources loaded in this build are left out. +fn source_versions(plan: &Bound<'_, PyAny>) -> PyResult>> { + let mut versions: HashMap> = HashMap::new(); + let freshness: Bound<'_, PyAny> = native_field(plan, "source_freshness")?; + if !freshness.is_none() { + for record in native_field(&freshness, "observed_records")?.try_iter()? { + let record: Bound<'_, PyAny> = record?; + let name: String = native_field(&record, "source_name")?.extract()?; + let hash: String = native_field(&record, "data_version_hash")?.extract()?; + let entry = versions.entry(name).or_insert_with(|| Some(hash.clone())); + if entry.as_deref() != Some(hash.as_str()) { + *entry = None; + } + } + } + for load in native_field(plan, "source_load_entries")?.try_iter()? { + versions.insert(native_field(&load?, "name")?.extract()?, None); + } + Ok(versions) +} + +/// The direct inputs of a node, through functions to what they read. +fn inputs( + deps: Vec<(String, String)>, + function_deps: &HashMap<(String, String), Vec<(String, String)>>, + sources: &HashMap>, +) -> Vec { + let mut seen: BTreeSet<(String, String)> = BTreeSet::new(); + let mut pending: VecDeque<(String, String)> = deps.into(); + let mut facts: Vec = Vec::new(); + while let Some((kind, name)) = pending.pop_front() { + if !seen.insert((kind.clone(), name.clone())) { + continue; + } + match kind.as_str() { + MODEL | SEED => facts.push(InputFact::Node(node_key(&kind, &name))), + SOURCE => facts.push(InputFact::Source { + version: sources.get(&name).cloned().flatten(), + name, + }), + function if FUNCTION_KINDS.contains(&function) => pending.extend( + function_deps + .get(&(kind.clone(), name.clone())) + .cloned() + .unwrap_or_default(), + ), + _ => facts.push(InputFact::Opaque(format!("{kind}:{name}"))), + } + } + facts +} + +fn key_rows(keys: &Bound<'_, PyAny>) -> PyResult> { + keys.try_iter()? + .map(|key| { + let key = graph_key(&key?)?; + Ok((key.kind, key.name)) + }) + .collect() +} + +/// Why the planner runs a selected entry regardless of its records. +fn entry_forced(entry: &Bound<'_, PyAny>, full_refresh: bool) -> PyResult> { + let mut forced: Vec = Vec::new(); + if full_refresh { + forced.push(ForcedRun::FullRefresh); + } + match enum_text(&native_field(entry, "reason")?)?.as_str() { + FIRST_RUN => forced.push(ForcedRun::RelationMissing), + FULL_REFRESH => forced.push(ForcedRun::FullRefresh), + reason if UNCHANGED_REASONS.contains(&reason) => {} + reason => forced.push(ForcedRun::PlannerChange(reason.to_owned())), + } + for field in ["start_cursor_override", "end_cursor_override"] { + if entry.hasattr(field)? && !native_field(entry, field)?.is_none() { + forced.push(ForcedRun::CursorInterval); + break; + } + } + Ok(forced) +} + +/// Facts of the selected models and seeds and of every model and seed they read from. +pub(crate) fn build_facts( + (plan, project): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (changed, full_refresh): (bool, bool), +) -> PyResult { + let py: Python<'_> = plan.py(); + let mut compiled: HashMap> = HashMap::new(); + for model in project_reads::models(project)?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + compiled.insert( + node_key(MODEL, &model_reads::model_name(&model)?), + Compiled::Model(model), + ); + } + for seed in project_reads::seeds(project)?.try_iter()? { + let seed: Bound<'_, PyAny> = seed?; + compiled.insert( + node_key(SEED, &seed_reads::name(&seed)?.extract::()?), + Compiled::Seed(seed), + ); + } + let mut function_deps: HashMap<(String, String), Vec<(String, String)>> = HashMap::new(); + let functions: Bound<'_, PyAny> = project_reads::functions(project)?; + for function in functions.try_iter()? { + let function: Bound<'_, PyAny> = function?; + let key = graph_key(&function_reads::key(&function)?)?; + function_deps.insert( + (key.kind, key.name), + key_rows(&function_reads::deps(&function)?)?, + ); + } + let mut selected: BTreeMap> = BTreeMap::new(); + let mut runnable: BTreeSet = BTreeSet::new(); + for (entries, kind) in [("model_entries", MODEL), ("seed_entries", SEED)] { + for entry in native_field(plan, entries)?.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + let key: NodeKey = node_key(kind, &native_field(&entry, "name")?.extract::()?); + if enum_text(&native_field(&entry, "action")?)? != SKIP_ACTION { + runnable.insert(key.clone()); + } + let forced: Vec = if kind == MODEL { + entry_forced(&entry, full_refresh)? + } else { + seed_forced(&entry, full_refresh)? + }; + selected.insert(key, forced); + } + } + let sources: HashMap> = source_versions(plan)?; + let existing: BTreeSet = native_field(plan, "existing_model_names")? + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?; + let dialect: Option = native_field(project, "sql_analysis_dialect")?.extract()?; + let function_hashes: Bound<'_, PyDict> = function_local_hashes(&functions, dialect.as_deref())?; + let hooks: Bound<'_, PyAny> = native_field(project, "hook_functions")?; + let hook_hashes: Bound<'_, PyDict> = hook_version_hashes(&hooks)?; + let locations: Option = if changed { + Some(Locations::read(plan, project)?) + } else { + None + }; + let mut nodes: Vec = Vec::new(); + let mut pending: VecDeque = selected.keys().cloned().collect(); + let mut visited: BTreeSet = BTreeSet::new(); + while let Some(key) = pending.pop_front() { + if !visited.insert(key.clone()) { + continue; + } + let Some(node) = compiled.get(&key) else { + continue; + }; + let (destination, deps, version_hash) = match node { + Compiled::Model(model) => ( + model_reads::destination(model)?, + model_reads::dep_rows(model)?, + model_local_identity_hash( + &model_query_hash( + py, + &key.name, + ( + &model_reads::query_sql(model)?.extract::()?, + dialect.as_deref(), + ), + )?, + &model_version_identity_metadata_json( + model, + Some(function_hashes.as_any()), + Some(hook_hashes.as_any()), + )?, + ), + ), + Compiled::Seed(seed) => ( + seed_reads::destination(seed)?, + Vec::new(), + seed_identity(seed)?.0, + ), + }; + let node_inputs: Vec = inputs(deps, &function_deps, &sources); + for input in &node_inputs { + if let InputFact::Node(upstream) = input { + pending.push_back(upstream.clone()); + } + } + let mut forced: Vec = match selected.get(&key) { + Some(forced) => forced.clone(), + None if key.kind == MODEL && !existing.contains(&key.name) => { + vec![ForcedRun::RelationMissing] + } + None => Vec::new(), + }; + let relation: Option = relation(&destination)?; + if let (Compiled::Model(model), Some(locations)) = (node, &locations) + && model_reads_undeclared( + model, + (&node_inputs, relation.as_ref()), + locations, + dialect.as_deref(), + )? + { + forced.push(ForcedRun::InputsNotEnumerable); + } + let Some(relation) = relation else { + continue; + }; + nodes.push(NodeFacts { + selected: selected.contains_key(&key), + key, + relation, + version_hash, + forced, + inputs: node_inputs, + }); + } + runnable.retain(|key| nodes.iter().any(|node| node.key == *key)); + Ok(BuildFacts { nodes, runnable }) +} + +fn seed_forced(entry: &Bound<'_, PyAny>, full_refresh: bool) -> PyResult> { + let mut forced: Vec = Vec::new(); + if full_refresh { + forced.push(ForcedRun::FullRefresh); + } + match enum_text(&native_field(entry, "reason")?)?.as_str() { + FIRST_RUN => forced.push(ForcedRun::RelationMissing), + reason if UNCHANGED_REASONS.contains(&reason) => {} + reason => forced.push(ForcedRun::PlannerChange(reason.to_owned())), + } + Ok(forced) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/mod.rs new file mode 100644 index 0000000000..ce58c1c1bb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod facts; +pub(crate) mod recorder; +pub(crate) mod relation_reads; +pub(crate) mod state_io; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs new file mode 100644 index 0000000000..50b7d0d137 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs @@ -0,0 +1,373 @@ +//! Record what each model and seed execution consumed, and decide `--changed` skips, around one build. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_state::consumption::constants::RECENT_EXECUTIONS; +use sqlbuild_state::consumption::main::changed_run_set::changed_run_set; +use sqlbuild_state::consumption::main::consumed_inputs::consumed_inputs; +use sqlbuild_state::consumption::main::consumption_history::consumption_history; +use sqlbuild_state::consumption::main::started_append::started_append; +use sqlbuild_state::consumption::main::succeeded_appends::succeeded_appends; +use sqlbuild_state::consumption::models::{ + ConsumptionAppend, ConsumptionHistory, ConsumptionRow, ForcedRun, InputFact, NodeFacts, + NodeKey, RunReason, +}; +use uuid::Uuid; + +use crate::bindings::_helpers::consumption::facts::{BuildFacts, build_facts}; +use crate::bindings::_helpers::consumption::state_io::{ + Location, append, ensure_table, read_recent, +}; +use crate::bindings::_helpers::records::fields::field as native_field; + +/// Succeeded rows are flushed once this many executions are pending. +const FLUSH_EXECUTIONS: usize = 100; +/// Or once this long has passed since the last flush. +const FLUSH_INTERVAL: Duration = Duration::from_secs(30); +const MODEL_RESULT: &str = "model_name"; +const SEED_RESULT: &str = "seed_name"; +const SUCCESS: &str = "success"; +const ERROR_OUTCOME: &str = "error"; + +/// The mutable side of one build's recording. +struct Recording { + pending: BTreeMap>, + pending_executions: usize, + pending_seeds: Vec, + tables: BTreeSet, + last_flush: Instant, + warnings: Vec, +} + +/// Appends `started` rows before a build and `succeeded` rows as its models and seeds complete. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct ConsumptionRecorder { + adapter: Py, + connection: Py, + run_id: String, + nodes: HashMap, + history: ConsumptionHistory, + executions: HashMap, + failed_start: HashMap, + base_on_node_complete: Py, + recording: Mutex, +} + +/// What `--changed` decided, for the build's report. +pub(crate) struct ChangedDecision { + pub(crate) run: usize, + pub(crate) skipped: Vec, + pub(crate) reasons: Vec<(NodeKey, RunReason)>, +} + +fn kind_of(result: &Bound<'_, PyAny>) -> PyResult> { + for (field, kind) in [(MODEL_RESULT, "model"), (SEED_RESULT, "seed")] { + if let Some(name) = result.getattr_opt(field)? { + return Ok(Some((kind, name.extract()?))); + } + } + Ok(None) +} + +fn status_text(value: &Bound<'_, PyAny>) -> PyResult { + let value: Bound<'_, PyAny> = value.getattr_opt("value")?.unwrap_or_else(|| value.clone()); + Ok(value.str()?.extract::()?.to_ascii_lowercase()) +} + +impl ConsumptionRecorder { + fn lock(&self) -> PyResult> { + self.recording + .lock() + .map_err(|_| PyRuntimeError::new_err("consumption recorder state is poisoned")) + } + + /// Queue the succeeded rows of one execution that wrote its relation. + fn queue(&self, recording: &mut Recording, key: &NodeKey) { + let (Some(node), Some(execution_id)) = (self.nodes.get(key), self.executions.get(key)) + else { + return; + }; + let inputs = consumed_inputs(node, (&self.nodes, &self.history), &self.executions); + recording + .pending + .entry(Location::of(&node.relation)) + .or_default() + .extend(succeeded_appends( + node, + (execution_id, &self.run_id), + &inputs, + )); + recording.pending_executions += 1; + } + + /// Append every pending row; a failure is reported once, and its executions simply run again. + fn flush(&self, py: Python<'_>, recording: &mut Recording) { + let adapter: &Bound<'_, PyAny> = self.adapter.bind(py); + let connection: &Bound<'_, PyAny> = self.connection.bind(py); + for (location, rows) in std::mem::take(&mut recording.pending) { + let written: PyResult<()> = (|| { + if recording.tables.insert(location.clone()) { + ensure_table(adapter, connection, &location)?; + } + append(adapter, connection, &location, &rows) + })(); + if let Err(error) = written { + recording.warnings.push(format!( + "could not record consumed inputs in schema {}: {error}; those models and seeds run again under --changed", + location.schema + )); + } + } + recording.pending_executions = 0; + recording.last_flush = Instant::now(); + } + + /// Queue the seeds whose error audits passed, flush everything and close the connection. + pub(crate) fn finish( + &self, + py: Python<'_>, + build_result: &Bound<'_, PyAny>, + ) -> PyResult> { + let mut failed_audits: BTreeSet = BTreeSet::new(); + for audit in native_field(build_result, "source_audit_results")?.try_iter()? { + let audit: Bound<'_, PyAny> = audit?; + let target: Bound<'_, PyAny> = native_field(&audit, "attached_target_name")?; + if !target.is_none() && status_text(&native_field(&audit, "outcome")?)? == ERROR_OUTCOME + { + failed_audits.insert(target.extract()?); + } + } + let mut recording = self.lock()?; + for seed in std::mem::take(&mut recording.pending_seeds) { + if !failed_audits.contains(&seed.name) { + self.queue(&mut recording, &seed); + } + } + self.flush(py, &mut recording); + let adapter: &Bound<'_, PyAny> = self.adapter.bind(py); + let closed = adapter.call_method1("close", (self.connection.bind(py),)); + let warnings: Vec = std::mem::take(&mut recording.warnings); + closed?; + Ok(warnings) + } +} + +#[pymethods] +impl ConsumptionRecorder { + /// Forward a finished node, then queue a successful model, or a seed until its audits are known. + fn on_node_complete(&self, py: Python<'_>, node_result: Bound<'_, PyAny>) -> PyResult<()> { + self.base_on_node_complete.bind(py).call1((&node_result,))?; + let Some((kind, name)) = kind_of(&node_result)? else { + return Ok(()); + }; + if status_text(&native_field(&node_result, "status")?)? != SUCCESS { + return Ok(()); + } + let key = NodeKey { + kind: kind.to_owned(), + name, + }; + let mut recording = self.lock()?; + if kind == "seed" { + recording.pending_seeds.push(key); + return Ok(()); + } + self.queue(&mut recording, &key); + if recording.pending_executions >= FLUSH_EXECUTIONS + || recording.last_flush.elapsed() >= FLUSH_INTERVAL + { + self.flush(py, &mut recording); + } + Ok(()) + } + + /// Fail a model or seed whose `started` row could not be appended, before it writes. + #[pyo3(signature = (*, entry, connection))] + fn before_node_write( + &self, + entry: Bound<'_, PyAny>, + connection: Bound<'_, PyAny>, + ) -> PyResult<()> { + let _ = connection; + let name: String = native_field(&entry, "name")?.extract()?; + let kind: &str = if entry.hasattr("csv_settings")? { + "seed" + } else { + "model" + }; + let key = NodeKey { + kind: kind.to_owned(), + name, + }; + match self.failed_start.get(&key) { + Some(error) => Err(PyRuntimeError::new_err(format!( + "not written: SQLBuild could not record that this {kind} started ({error}), so a crash could not be detected later" + ))), + None => Ok(()), + } + } +} + +/// The locations whose records this build reads: all of them under `--changed`, else the +/// relations of unselected upstreams whose last execution the selected nodes record. +fn read_locations(facts: &BuildFacts, changed: bool) -> BTreeSet { + let by_key: HashMap<&NodeKey, &NodeFacts> = + facts.nodes.iter().map(|node| (&node.key, node)).collect(); + if changed { + return facts + .nodes + .iter() + .map(|node| Location::of(&node.relation)) + .collect(); + } + facts + .nodes + .iter() + .filter(|node| facts.runnable.contains(&node.key)) + .flat_map(|node| node.inputs.iter()) + .filter_map(|input| match input { + InputFact::Node(upstream) if !facts.runnable.contains(upstream) => by_key.get(upstream), + _ => None, + }) + .map(|node| Location::of(&node.relation)) + .collect() +} + +/// Everything the build needs from the recorder: the recorder itself and what `--changed` skipped. +pub(crate) struct Prepared<'py> { + pub(crate) recorder: Bound<'py, ConsumptionRecorder>, + pub(crate) decision: Option, + pub(crate) warnings: Vec, +} + +/// Read records, decide `--changed` skips, mint execution IDs and append `started` rows. +pub(crate) fn prepare<'py>( + (plan, project): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (adapter, connection_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (changed, full_refresh): (bool, bool), + (run_id, base_on_node_complete): (String, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = plan.py(); + let facts: BuildFacts = build_facts((plan, project), (changed, full_refresh))?; + let connection: Bound<'py, PyAny> = adapter.call_method1("connect", (connection_config,))?; + let mut warnings: Vec = Vec::new(); + let mut rows: Vec = Vec::new(); + for location in read_locations(&facts, changed) { + match read_recent(adapter, &connection, &location) { + Ok(read) => rows.extend(read), + Err(error) => warnings.push(format!( + "could not read consumed inputs in schema {}: {error}; its models and seeds count as changed", + location.schema + )), + } + } + let history: ConsumptionHistory = consumption_history(&rows, RECENT_EXECUTIONS); + let mut skipped: BTreeSet = BTreeSet::new(); + let decision: Option = if changed { + let mut reasons: Vec<(NodeKey, RunReason)> = Vec::new(); + for (key, reason) in changed_run_set(&facts.nodes, &history) { + if !facts.runnable.contains(&key) { + continue; + } + match reason { + Some(reason) => reasons.push((key, reason)), + None => { + skipped.insert(key); + } + } + } + Some(ChangedDecision { + run: reasons.len(), + skipped: skipped.iter().cloned().collect(), + reasons, + }) + } else { + None + }; + let executions: HashMap = facts + .runnable + .iter() + .filter(|key| !skipped.contains(*key)) + .map(|key| (key.clone(), Uuid::now_v7().to_string())) + .collect(); + let nodes: HashMap = facts + .nodes + .into_iter() + .map(|node| (node.key.clone(), node)) + .collect(); + let mut started: BTreeMap> = BTreeMap::new(); + for (key, execution_id) in &executions { + if let Some(node) = nodes.get(key) { + started + .entry(Location::of(&node.relation)) + .or_default() + .push(started_append(node, execution_id, &run_id)); + } + } + let mut failed_start: HashMap = HashMap::new(); + let mut tables: BTreeSet = BTreeSet::new(); + for (location, rows) in started { + let written: PyResult<()> = ensure_table(adapter, &connection, &location) + .and_then(|()| append(adapter, &connection, &location, &rows)); + match written { + Ok(()) => { + tables.insert(location); + } + Err(error) => { + let message: String = error.to_string(); + for row in rows { + failed_start.insert(row.node, message.clone()); + } + } + } + } + let recorder = ConsumptionRecorder { + adapter: adapter.clone().unbind(), + connection: connection.unbind(), + run_id, + nodes, + history, + executions, + failed_start, + base_on_node_complete: base_on_node_complete.unbind(), + recording: Mutex::new(Recording { + pending: BTreeMap::new(), + pending_executions: 0, + pending_seeds: Vec::new(), + tables, + last_flush: Instant::now(), + warnings: Vec::new(), + }), + }; + Ok(Prepared { + recorder: Bound::new(py, recorder)?, + decision, + warnings, + }) +} + +/// Why `--changed` runs a node, as one line for verbose output. +pub(crate) fn run_reason_text(reason: &RunReason) -> String { + match reason { + RunReason::Forced(ForcedRun::PlannerChange(kind)) => format!("planner change ({kind})"), + RunReason::Forced(ForcedRun::FullRefresh) => "full refresh".to_owned(), + RunReason::Forced(ForcedRun::RelationMissing) => "relation missing".to_owned(), + RunReason::Forced(ForcedRun::CursorInterval) => "cursor interval requested".to_owned(), + RunReason::Forced(ForcedRun::InputsNotEnumerable) => { + "inputs cannot be enumerated".to_owned() + } + RunReason::NoRecord => "no completed record".to_owned(), + RunReason::Unfinished => "last execution did not finish".to_owned(), + RunReason::AmbiguousOrder => "record order is ambiguous".to_owned(), + RunReason::DefinitionChanged => "definition changed".to_owned(), + RunReason::InputsChanged => "inputs changed".to_owned(), + RunReason::InputChanged(input) => format!("input {input} changed"), + RunReason::InputUnknown(input) => format!("input {input} has no known version"), + RunReason::UpstreamRuns(key) => format!("upstream {} {} runs", key.kind, key.name), + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs new file mode 100644 index 0000000000..7367c56436 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs @@ -0,0 +1,136 @@ +//! Whether a model's compiled SQL reads a relation that none of its declared inputs writes. + +use std::collections::HashMap; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_state::consumption::models::{InputFact, NodeKey, RelationKey}; + +use crate::bindings::_helpers::analysis::relation_names::{ + RelationRow, extract_sql_relation_names, +}; +use crate::bindings::_helpers::compiled_views::{model_reads, project_reads, seed_reads}; +use crate::bindings::_helpers::records::fields::field as native_field; + +/// The relations each model, seed and source names, by node key or source name. +pub(crate) struct Locations { + nodes: HashMap, + sources: HashMap, Option, String)>, +} + +fn optional(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.extract()?)) +} + +fn location(destination: &Bound<'_, PyAny>) -> PyResult { + Ok(RelationKey { + database: optional(native_field(destination, "database")?)?, + schema: optional(native_field(destination, "schema")?)?.unwrap_or_default(), + name: native_field(destination, "name")?.extract()?, + }) +} + +impl Locations { + pub(crate) fn read(plan: &Bound<'_, PyAny>, project: &Bound<'_, PyAny>) -> PyResult { + let mut nodes: HashMap = HashMap::new(); + for model in project_reads::models(project)?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + nodes.insert( + NodeKey { + kind: "model".to_owned(), + name: model_reads::model_name(&model)?, + }, + location(&model_reads::destination(&model)?)?, + ); + } + for seed in project_reads::seeds(project)?.try_iter()? { + let seed: Bound<'_, PyAny> = seed?; + nodes.insert( + NodeKey { + kind: "seed".to_owned(), + name: seed_reads::name(&seed)?.extract()?, + }, + location(&seed_reads::destination(&seed)?)?, + ); + } + let mut sources: HashMap, Option, String)> = HashMap::new(); + for map in ["source_map", "source_read_map"] { + for item in native_field(plan, map)?.call_method0("items")?.try_iter()? { + let (name, entry): (String, Bound<'_, PyAny>) = item?.extract()?; + let table: String = + optional(native_field(&entry, "table")?)?.unwrap_or_else(|| name.clone()); + sources.insert( + name, + ( + optional(native_field(&entry, "database")?)?, + optional(native_field(&entry, "schema")?)?, + table, + ), + ); + } + } + Ok(Self { nodes, sources }) + } + + fn allows(&self, input: &InputFact, relation: &RelationRow) -> bool { + match input { + InputFact::Node(key) => self + .nodes + .get(key) + .is_some_and(|location| names(relation, location)), + InputFact::Source { name, .. } => { + self.sources + .get(name) + .is_some_and(|(database, schema, table)| { + names( + relation, + &RelationKey { + database: database.clone(), + schema: schema.clone().unwrap_or_default(), + name: table.clone(), + }, + ) + }) + } + InputFact::Opaque(_) => false, + } + } +} + +/// Whether the SQL relation, with the qualifiers it spells, can name `location`. +fn names((name, schema, database): &RelationRow, location: &RelationKey) -> bool { + let same = |left: &str, right: &str| left.eq_ignore_ascii_case(right); + same(name, &location.name) + && schema + .as_deref() + .is_none_or(|schema| same(schema, &location.schema)) + && database.as_deref().is_none_or(|database| { + location + .database + .as_deref() + .is_some_and(|known| same(database, known)) + }) +} + +/// True when the SQL cannot be parsed or names a relation no input or the model itself writes. +pub(crate) fn model_reads_undeclared( + model: &Bound<'_, PyAny>, + (inputs, own): (&[InputFact], Option<&RelationKey>), + locations: &Locations, + dialect: Option<&str>, +) -> PyResult { + let sql: String = model_reads::query_sql(model)?.extract()?; + let Some((relations, temporary)) = extract_sql_relation_names(model.py(), &sql, dialect)? + else { + return Ok(true); + }; + Ok(relations.iter().any(|relation| { + !temporary + .iter() + .any(|created| created.eq_ignore_ascii_case(&relation.0)) + && !own.is_some_and(|own| names(relation, own)) + && !inputs.iter().any(|input| locations.allows(input, relation)) + })) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/state_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/state_io.rs new file mode 100644 index 0000000000..cee3088181 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/state_io.rs @@ -0,0 +1,393 @@ +//! Consumption table DDL, recent-execution reads and chunked appends through adapter callbacks. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet, PyInt, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::constants::CONSUMPTION_TABLE_NAME; +use sqlbuild_state::consumption::constants::{ + CONSUMPTION_COLUMN_TYPES, INTEGER_STATE_VALUE_TYPE, NULLABLE_COLUMNS, RECENT_EXECUTIONS, + RECORDED_AT_COLUMN, +}; +use sqlbuild_state::consumption::main::prune_executions_sql::prune_executions_sql; +use sqlbuild_state::consumption::main::recent_executions_sql::recent_executions_sql; +use sqlbuild_state::consumption::models::{ + ConsumedInput, ConsumptionAppend, ConsumptionRow, NodeKey, RelationKey, +}; +use sqlbuild_state::freshness::constants::STRING_STATE_VALUE_TYPE; +use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; + +use crate::bindings::_helpers::boundary::python_calls::{call_with, imported}; +use crate::bindings::_helpers::contracts::state_literals::render_state_sql_literal; +use crate::bindings::_helpers::freshness::support::presumed_utc_datetime; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; +use crate::bindings::_helpers::state_rows::timestamps::date_time_typed; + +/// Rows per `INSERT`; SQL Server accepts at most 1,000 rows in one `VALUES` list. +const ROWS_PER_STATEMENT: usize = 500; + +/// One destination schema of the consumption table. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub(crate) struct Location { + pub(crate) database: Option, + pub(crate) schema: String, +} + +impl Location { + pub(crate) fn of(relation: &RelationKey) -> Self { + Self { + database: relation.database.clone(), + schema: relation.schema.clone(), + } + } + + fn keywords<'py>(&self, py: Python<'py>) -> Vec<(&'static str, Bound<'py, PyAny>)> { + vec![ + ( + "database", + self.database + .as_deref() + .map_or_else(|| py.None().into_bound(py), |database| text(py, database)), + ), + ("schema", text(py, &self.schema)), + ] + } +} + +fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +/// The adapter's qualified name of the consumption table in `(database, schema)`. +fn qualified_name( + render_qualified_name: &Bound<'_, PyAny>, + (database, schema): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult> { + let py: Python<'_> = render_qualified_name.py(); + let name: Bound<'_, PyAny> = call_with( + render_qualified_name, + &[ + ("database", database.clone()), + ("schema", schema.clone()), + ("name", text(py, CONSUMPTION_TABLE_NAME)), + ], + )?; + if name.is_none() { + return Ok(None); + } + Ok(Some(name.extract()?)) +} + +fn table_of(adapter: &Bound<'_, PyAny>, location: &Location) -> PyResult> { + let py: Python<'_> = adapter.py(); + let database: Bound<'_, PyAny> = location + .database + .as_deref() + .map_or_else(|| py.None().into_bound(py), |database| text(py, database)); + qualified_name( + &native_field(adapter, "render_qualified_name")?, + (&database, &text(py, &location.schema)), + ) +} + +/// Create-if-missing DDL for the consumption table, with adapter-rendered column types. +#[pyfunction] +#[pyo3(signature = (*, database, schema, render_qualified_name, render_framework_type))] +fn build_consumption_create_table_sql( + database: &Bound<'_, PyAny>, + schema: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, + render_framework_type: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = database.py(); + let Some(table) = qualified_name(render_qualified_name, (database, schema))? else { + return Ok(String::new()); + }; + let columns: Bound<'_, PyAny> = PyTuple::new( + py, + CONSUMPTION_COLUMN_TYPES.iter().map(|(column, _)| *column), + )? + .into_any(); + let types: Bound<'_, PyDict> = PyDict::new(py); + for (column, declared) in CONSUMPTION_COLUMN_TYPES { + types.set_item(column, declared)?; + } + let required: Vec<&str> = CONSUMPTION_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .filter(|column| !NULLABLE_COLUMNS.contains(column)) + .collect(); + state_table_create_sql( + &table, + ( + &columns, + &types.into_any(), + &PyFrozenSet::new(py, &required)?.into_any(), + ), + render_framework_type, + ) +} + +/// SQL deleting whole consumption executions beyond the newest `retain_versions` per relation. +#[pyfunction] +#[pyo3(signature = (*, database, schema, render_qualified_name, retain_versions))] +fn build_consumption_prune_sql( + database: &Bound<'_, PyAny>, + schema: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, + retain_versions: i64, +) -> PyResult { + Ok(qualified_name(render_qualified_name, (database, schema))? + .map(|table| prune_executions_sql(&table, retain_versions)) + .unwrap_or_default()) +} + +fn execute<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + sql: &str, +) -> PyResult> { + call_with( + &native_field(adapter, "execute")?, + &[ + ("connection", connection.clone()), + ("sql", text(adapter.py(), sql)), + ], + ) +} + +/// Create the table and its indexes in `location` when missing. +pub(crate) fn ensure_table( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + location: &Location, +) -> PyResult<()> { + let py: Python<'_> = adapter.py(); + let keywords = location.keywords(py); + let create: String = call_with( + &native_field(adapter, "render_create_consumption_table_sql")?, + &keywords, + )? + .extract()?; + if !create.is_empty() { + execute(adapter, connection, &create)?; + } + for index in call_with( + &native_field(adapter, "render_create_consumption_index_sqls")?, + &keywords, + )? + .try_iter()? + { + execute(adapter, connection, &index?.extract::()?)?; + } + Ok(()) +} + +/// Whether the warehouse catalog lists the consumption table in `location`. +fn table_exists( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + location: &Location, +) -> PyResult { + let py: Python<'_> = adapter.py(); + let mut keywords = location.keywords(py); + keywords.push(("connection", connection.clone())); + keywords.push(("name", text(py, CONSUMPTION_TABLE_NAME))); + call_with(&native_field(adapter, "get_columns")?, &keywords)?.is_truthy() +} + +/// Rows of the newest executions per relation in `location`; none when the table is missing. +pub(crate) fn read_recent( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + location: &Location, +) -> PyResult> { + let Some(table) = table_of(adapter, location)? else { + return Ok(Vec::new()); + }; + if !table_exists(adapter, connection, location)? { + return Ok(Vec::new()); + } + let result: Bound<'_, PyAny> = execute( + adapter, + connection, + &recent_executions_sql(&table, RECENT_EXECUTIONS), + )?; + result + .call_method0("fetchall")? + .try_iter()? + .map(|row| read_row(&row?)) + .collect() +} + +fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.str()?.extract()?)) +} + +fn required_text(value: &Bound<'_, PyAny>) -> PyResult { + Ok(optional_text(value)?.unwrap_or_default()) +} + +/// Microseconds since the epoch of a warehouse timestamp, naive values read as UTC. +fn epoch_micros(value: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = value.py(); + let moment: Bound<'_, PyAny> = if date_time_typed(value)? { + value.clone() + } else { + imported(py, "datetime", "datetime")?.call_method1("fromisoformat", (value.str()?,))? + }; + let utc: Bound<'_, PyAny> = presumed_utc_datetime(&moment)?; + let epoch: Bound<'_, PyAny> = imported(py, "datetime", "datetime")? + .call_method1("fromtimestamp", (0, imported(py, "datetime", "UTC")?))?; + let micros: Bound<'_, PyAny> = imported(py, "datetime", "timedelta")?.call1((0, 0, 1))?; + utc.call_method1("__sub__", (epoch,))? + .call_method1("__floordiv__", (micros,))? + .extract() +} + +fn read_row(row: &Bound<'_, PyAny>) -> PyResult { + let cell = |index: usize| row.get_item(index); + let input_kind: Option = optional_text(&cell(11)?)?; + let input: Option = match input_kind { + Some(kind) => Some(ConsumedInput { + kind, + name: required_text(&cell(12)?)?, + version: required_text(&cell(13)?)?, + }), + None => None, + }; + let count: Bound<'_, PyAny> = cell(10)?; + let input_count: i64 = if count.is_instance_of::() { + count.extract()? + } else { + count + .str()? + .extract::()? + .trim() + .parse() + .unwrap_or(-1) + }; + Ok(ConsumptionRow { + row: ConsumptionAppend { + event_id: required_text(&cell(0)?)?, + execution_id: required_text(&cell(1)?)?, + phase: required_text(&cell(2)?)?, + run_id: required_text(&cell(3)?)?, + node: NodeKey { + kind: required_text(&cell(4)?)?, + name: required_text(&cell(5)?)?, + }, + relation: RelationKey { + database: optional_text(&cell(6)?)?, + schema: required_text(&cell(7)?)?, + name: required_text(&cell(8)?)?, + }, + version_hash: required_text(&cell(9)?)?, + input_count, + input, + }, + recorded_at: epoch_micros(&cell(14)?)?, + }) +} + +fn literal(py: Python<'_>, value: Option<&str>) -> PyResult { + let value: Bound<'_, PyAny> = + value.map_or_else(|| py.None().into_bound(py), |value| text(py, value)); + render_state_sql_literal(py, &value, STRING_STATE_VALUE_TYPE, None) +} + +fn row_cells(py: Python<'_>, row: &ConsumptionAppend, recorded_at: &str) -> PyResult> { + let input = row.input.as_ref(); + Ok(vec![ + literal(py, Some(&row.event_id))?, + literal(py, Some(&row.execution_id))?, + literal(py, Some(&row.phase))?, + literal(py, Some(&row.run_id))?, + literal(py, Some(&row.node.kind))?, + literal(py, Some(&row.node.name))?, + literal(py, row.relation.database.as_deref())?, + literal(py, Some(&row.relation.schema))?, + literal(py, Some(&row.relation.name))?, + literal(py, Some(&row.version_hash))?, + render_state_sql_literal( + py, + &PyInt::new(py, row.input_count).into_any(), + INTEGER_STATE_VALUE_TYPE, + None, + )?, + literal(py, input.map(|input| input.kind.as_str()))?, + literal(py, input.map(|input| input.name.as_str()))?, + literal(py, input.map(|input| input.version.as_str()))?, + recorded_at.to_owned(), + ]) +} + +/// Append `rows` to `location`'s table in statements of whole executions where they fit. +pub(crate) fn append( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + location: &Location, + rows: &[ConsumptionAppend], +) -> PyResult<()> { + let py: Python<'_> = adapter.py(); + let Some(table) = table_of(adapter, location)? else { + return Ok(()); + }; + let recorded_at: String = adapter + .call_method0("render_current_utc_timestamp")? + .extract()?; + let columns: Vec<&str> = CONSUMPTION_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .collect(); + debug_assert_eq!(columns.last(), Some(&RECORDED_AT_COLUMN)); + for chunk in execution_chunks(rows) { + let cells: Vec> = chunk + .iter() + .map(|row| row_cells(py, row, &recorded_at)) + .collect::>()?; + execute( + adapter, + connection, + &values_insert_sql(&table, &columns, &cells), + )?; + } + Ok(()) +} + +/// Contiguous rows in statements of whole executions, splitting only an execution that alone overflows. +fn execution_chunks(rows: &[ConsumptionAppend]) -> Vec<&[ConsumptionAppend]> { + let mut chunks: Vec<&[ConsumptionAppend]> = Vec::new(); + let (mut start, mut index): (usize, usize) = (0, 0); + while index < rows.len() { + let end: usize = index + + rows[index..] + .iter() + .take_while(|row| row.execution_id == rows[index].execution_id) + .count(); + if end - start > ROWS_PER_STATEMENT && index > start { + chunks.push(&rows[start..index]); + start = index; + } + while end - start > ROWS_PER_STATEMENT { + chunks.push(&rows[start..start + ROWS_PER_STATEMENT]); + start += ROWS_PER_STATEMENT; + } + index = end; + } + if start < rows.len() { + chunks.push(&rows[start..]); + } + chunks +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_consumption_create_table_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(build_consumption_prune_sql, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index 511a468e3d..5366b69604 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -18,10 +18,10 @@ use sqlbuild_core::command_output::constants::{ INVOCATION_CONTEXT_ENV, }; use sqlbuild_core::constants::{ - COLUMN_MIGRATION_TABLE_NAME, CTE_AST_KIND, FINGERPRINT_TABLE_NAME, MIGRATION_TABLE_NAME, - OLD_NAME_VIEW_TABLE_NAME, PROJECT_CONFIG_FILENAME, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, - SETTINGS_SECTION, SOURCE_FRESHNESS_TABLE_NAME, STAGED_PROMOTION_MODE, - TABLE_PROMOTION_MODE_SETTING_KEY, ZERO_DAY_DURATION, + COLUMN_MIGRATION_TABLE_NAME, CONSUMPTION_TABLE_NAME, CTE_AST_KIND, FINGERPRINT_TABLE_NAME, + MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, PROJECT_CONFIG_FILENAME, + PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, SETTINGS_SECTION, SOURCE_FRESHNESS_TABLE_NAME, + STAGED_PROMOTION_MODE, TABLE_PROMOTION_MODE_SETTING_KEY, ZERO_DAY_DURATION, }; use sqlbuild_core::diagnostic_routing::constants::{ SQL_DIGEST_FIELD, SQL_PRIVATE_RECORD_FIELD, SQL_TEXT_FIELD, @@ -84,6 +84,7 @@ pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ ("INTEGRATION_RESULT_PATH_ENV", RESULT_PATH_ENV), ("FINGERPRINT_TABLE_NAME", FINGERPRINT_TABLE_NAME), ("SOURCE_FRESHNESS_TABLE_NAME", SOURCE_FRESHNESS_TABLE_NAME), + ("CONSUMPTION_TABLE_NAME", CONSUMPTION_TABLE_NAME), ("MIGRATION_TABLE_NAME", MIGRATION_TABLE_NAME), ("COLUMN_MIGRATION_TABLE_NAME", COLUMN_MIGRATION_TABLE_NAME), ("OLD_NAME_VIEW_TABLE_NAME", OLD_NAME_VIEW_TABLE_NAME), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs index 052592d9e5..fceeda9567 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs @@ -25,7 +25,7 @@ const EXECUTION_LOGGER: &str = "sqlbuild.execution"; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; -use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; +use sqlbuild_core::constants::{CONSUMPTION_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME}; /// The adapter and its open connection. pub(crate) struct Warehouse<'py> { @@ -475,7 +475,7 @@ pub(crate) fn gather_warehouse_facts<'py>( }) } -/// Fingerprint and source freshness history prune candidates for target schemas. +/// Fingerprint, source freshness and consumption history prune candidates for target schemas. pub(crate) fn direct_state_prune_candidates<'py>( warehouse: &Warehouse<'py>, target_schemas: &[SchemaKey], @@ -494,6 +494,7 @@ pub(crate) fn direct_state_prune_candidates<'py>( schema.clone(), SOURCE_FRESHNESS_TABLE_NAME, )); + locations.push((database.clone(), schema.clone(), CONSUMPTION_TABLE_NAME)); } let lookup: HashSet = relation_lookup(warehouse, &locations)?; let mut candidates: Vec> = Vec::new(); @@ -507,6 +508,10 @@ pub(crate) fn direct_state_prune_candidates<'py>( SOURCE_FRESHNESS_TABLE_NAME, "render_prune_source_freshness_history_sql", ), + ( + CONSUMPTION_TABLE_NAME, + "render_prune_consumption_history_sql", + ), ] { if !lookup_exists(&lookup, (database.as_deref(), schema, table_name)) { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 732a57220e..1b33b89fce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -28,6 +28,7 @@ pub(crate) mod compiler_enums; pub(crate) mod compiler_errors; pub(crate) mod compiler_records; pub(crate) mod compute_logs; +pub(crate) mod consumption; pub(crate) mod contract_command; pub(crate) mod contract_records; pub(crate) mod contracts; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs index cc372a504b..e6041ffcc6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs @@ -51,7 +51,9 @@ fn string_pairs(mapping: &Bound<'_, PyAny>) -> PyResult> { } /// `{name: identity.version_hash}` of the discovered Python hooks. -fn hook_version_hashes<'py>(hook_functions: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn hook_version_hashes<'py>( + hook_functions: &Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = hook_functions.py(); let hashes: Bound<'py, PyDict> = PyDict::new(py); let identities: Bound<'py, PyAny> = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index 3fdd402cc8..c34a7e8402 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -243,6 +243,7 @@ const BUILD_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("no_color", FieldDefault::False), FieldSpec::new("fail_fast", FieldDefault::False), FieldSpec::new("full_refresh", FieldDefault::False), + FieldSpec::new("changed", FieldDefault::False), FieldSpec::new("load_sources", FieldDefault::None), FieldSpec::new("reload_sources", FieldDefault::False), FieldSpec::new("include_python", FieldDefault::True), diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8a67d7d699..9bba402174 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -184,6 +184,7 @@ COMMAND_OUTPUT_LOSS_RECORD_TYPE: str INTEGRATION_RESULT_PATH_ENV: str FINGERPRINT_TABLE_NAME: str SOURCE_FRESHNESS_TABLE_NAME: str +CONSUMPTION_TABLE_NAME: str MIGRATION_TABLE_NAME: str COLUMN_MIGRATION_TABLE_NAME: str OLD_NAME_VIEW_TABLE_NAME: str @@ -5455,6 +5456,20 @@ def evaluate_source_freshness_age_policy( def build_source_freshness_read_latest_sql( *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] ) -> str: ... +def build_consumption_create_table_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], +) -> str: ... +def build_consumption_prune_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + retain_versions: int, +) -> str: ... def build_source_freshness_create_table_sql( *, database: str | None, @@ -6452,6 +6467,7 @@ class BuildCommandRequest(NativeRecord): no_color: bool fail_fast: bool full_refresh: bool + changed: bool load_sources: bool | None reload_sources: bool include_python: bool @@ -6490,6 +6506,7 @@ class BuildCommandRequest(NativeRecord): no_color: bool = False, fail_fast: bool = False, full_refresh: bool = False, + changed: bool = False, load_sources: bool | None = None, reload_sources: bool = False, include_python: bool = True, diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 683da84321..dfc315a9e2 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -2285,6 +2285,34 @@ def render_create_source_freshness_index_sqls( del database, schema return () + def render_create_consumption_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the consumption table when it is missing.""" + + from sqlbuild._native import build_consumption_create_table_sql + + return build_consumption_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + + def render_create_consumption_index_sqls( + self, + *, + database: str | None, + schema: str, + ) -> tuple[str, ...]: + """Render optional consumption table index DDL statements.""" + + del database, schema + return () + + def render_current_utc_timestamp(self) -> str: + """Render the warehouse clock as a UTC timestamp expression.""" + + return "CURRENT_TIMESTAMP" + def render_insert_source_freshness_records_sql( self, *, @@ -2432,6 +2460,18 @@ def render_prune_source_freshness_history_sql( del database, schema, retain_versions raise AdapterUserError(message="direct-state history pruning is adapter-specific") + def render_prune_consumption_history_sql( + self, + *, + database: str | None, + schema: str, + retain_versions: int, + ) -> str: + """Render SQL that prunes old consumption executions.""" + + del database, schema, retain_versions + raise AdapterUserError(message="direct-state history pruning is adapter-specific") + def sql_analysis_dialect(self) -> str | None: """Return the configured SQL analysis dialect name, if any.""" diff --git a/src/sqlbuild/adapter/contract/classes/consumption.py b/src/sqlbuild/adapter/contract/classes/consumption.py new file mode 100644 index 0000000000..4d405b1bc7 --- /dev/null +++ b/src/sqlbuild/adapter/contract/classes/consumption.py @@ -0,0 +1,45 @@ +"""Default adapter implementations for the consumption-record state table.""" + +from __future__ import annotations + +from typing import Any, cast + + +class ConsumptionMixin: + """Provide portable consumption-table DDL and pruning for first-class warehouse adapters.""" + + def render_create_consumption_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the consumption table when it is missing.""" + + from sqlbuild._native import build_consumption_create_table_sql + + adapter: Any = cast(Any, self) + return build_consumption_create_table_sql( + database=database, + schema=schema, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + ) + + def render_create_consumption_index_sqls( + self, *, database: str | None, schema: str + ) -> tuple[str, ...]: + """Return no index DDL unless the warehouse adapter explicitly opts in.""" + + del database, schema + return () + + def render_prune_consumption_history_sql( + self, *, database: str | None, schema: str, retain_versions: int + ) -> str: + """Render SQL deleting whole executions beyond the newest `retain_versions` per relation.""" + + from sqlbuild._native import build_consumption_prune_sql + + adapter: Any = cast(Any, self) + return build_consumption_prune_sql( + database=database, + schema=schema, + render_qualified_name=adapter.render_qualified_name, + retain_versions=retain_versions, + ) diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 119592a31a..1dde873e0c 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -341,6 +341,9 @@ def render_replace_table_from_relation( def render_current_timestamp(self) -> str: return "CURRENT_TIMESTAMP" + def render_current_utc_timestamp(self) -> str: + return "CAST(get_current_timestamp() AT TIME ZONE 'UTC' AS TIMESTAMP)" + def drop_view( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index ae849af077..3504c8f4c4 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -1034,6 +1034,26 @@ def render_create_source_freshness_index_sqls( """Render optional source freshness table index DDL statements.""" ... + @abstractmethod + def render_create_consumption_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the consumption table when it is missing.""" + ... + + @abstractmethod + def render_create_consumption_index_sqls( + self, + *, + database: str | None, + schema: str, + ) -> tuple[str, ...]: + """Render optional consumption table index DDL statements.""" + ... + + @abstractmethod + def render_current_utc_timestamp(self) -> str: + """Render the warehouse clock as a UTC timestamp expression.""" + ... + @abstractmethod def render_insert_source_freshness_records_sql( self, @@ -1116,6 +1136,17 @@ def render_prune_source_freshness_history_sql( """Render SQL that prunes old source freshness history rows.""" ... + @abstractmethod + def render_prune_consumption_history_sql( + self, + *, + database: str | None, + schema: str, + retain_versions: int, + ) -> str: + """Render SQL that prunes old consumption executions.""" + ... + @abstractmethod def sql_analysis_dialect(self) -> str | None: """Return the SQL analysis dialect name for this adapter, if any.""" diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 7c6500412d..dd0a9ca300 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -36,6 +36,7 @@ from sqlbuild.adapter.contract.classes.historical_timestamp_snapshot_sql import ( HistoricalTimestampSnapshotSql, ) +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder @@ -148,7 +149,7 @@ from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -class BigQueryAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): +class BigQueryAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): """BigQuery adapter backed by google-cloud-bigquery.""" def get_columns_for_relations( @@ -1065,6 +1066,9 @@ def render_merge( def render_current_timestamp(self) -> str: return "CURRENT_TIMESTAMP" + def render_current_utc_timestamp(self) -> str: + return "CURRENT_TIMESTAMP()" + def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: match type_name: case LoaderLogicalType.BOOLEAN: diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 7a4bf15518..3a33dd0088 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -30,6 +30,7 @@ _validate_rectangular_typed_array, ) from sqlbuild.adapter.contract.classes.historical_snapshot_sql import render_is_distinct_from +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder @@ -153,7 +154,7 @@ ) -class ClickHouseAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): +class ClickHouseAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): """ClickHouse adapter that builds tables aside and rewrites whole partitions.""" adapter_name: ClassVar[str] = BuiltinAdapter.CLICKHOUSE.value @@ -674,6 +675,9 @@ def render_max_cursor_at_or_before( def render_current_timestamp(self) -> str: return "now64(6, 'UTC')" + def render_current_utc_timestamp(self) -> str: + return "now64(6, 'UTC')" + def normalize_row_diff_numeric_type(self, column_type: str) -> str | None: return normalize_numeric_family(type_sql=canonical_type(column_type), dialect=None) @@ -1917,6 +1921,12 @@ def render_create_source_freshness_table_sql(self, *, database: str | None, sche super().render_create_source_freshness_table_sql(database=database, schema=schema) ) + + def render_create_consumption_table_sql(self, *, database: str | None, schema: str) -> str: + return _state_table_ddl( + super().render_create_consumption_table_sql(database=database, schema=schema) + ) + def render_create_node_result_table_sql(self, *, database: str | None, schema: str) -> str: return _state_table_ddl( super().render_create_node_result_table_sql(database=database, schema=schema) diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index a7eae3813d..44c79bf7d6 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -35,6 +35,7 @@ from sqlbuild.adapter.contract.classes.historical_timestamp_snapshot_sql import ( HistoricalTimestampSnapshotSql, ) +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder @@ -140,7 +141,7 @@ from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -class DatabricksAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): +class DatabricksAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): """Databricks adapter backed by databricks-sql-connector.""" def get_columns_for_relations( @@ -851,6 +852,9 @@ def create_function( def render_current_timestamp(self) -> str: return "CURRENT_TIMESTAMP" + def render_current_utc_timestamp(self) -> str: + return "current_timestamp()" + def sql_analysis_dialect(self) -> str | None: """Return the SQL analysis dialect name for this adapter, if any.""" diff --git a/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py b/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py index 7294fd39c3..7f9e86563e 100644 --- a/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py +++ b/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py @@ -8,12 +8,13 @@ from sqlbuild.adapter.contract.classes.base_adapter import _quote_sql_string from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.microbatches.constants import MICROBATCH_GENERATION_COMMENT_PREFIX -class DuckDbAdapter(MicrobatchMixin, DuckDbBackedAdapter): +class DuckDbAdapter(MicrobatchMixin, ConsumptionMixin, DuckDbBackedAdapter): """DuckDB adapter allowing implicit main for targetless and named local targets.""" adapter_name: ClassVar[str] = BuiltinAdapter.DUCKDB.value diff --git a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py index 434c33a209..27b289c9ff 100644 --- a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py +++ b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py @@ -9,6 +9,7 @@ from urllib.parse import urlencode from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.types import BuiltinAdapter, LoaderLogicalType @@ -21,7 +22,7 @@ ) -class MotherDuckAdapter(MicrobatchMixin, DuckDbBackedAdapter): +class MotherDuckAdapter(MicrobatchMixin, ConsumptionMixin, DuckDbBackedAdapter): """MotherDuck adapter backed by DuckDB's MotherDuck connection support.""" adapter_name: ClassVar[str] = BuiltinAdapter.MOTHERDUCK.value diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 05bb55cd76..5a08f97b33 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -37,6 +37,7 @@ from sqlbuild.adapter.contract.classes.historical_timestamp_snapshot_sql import ( HistoricalTimestampSnapshotSql, ) +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder @@ -121,7 +122,7 @@ from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -class PostgresAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): +class PostgresAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): """PostgreSQL adapter backed by psycopg.""" def get_columns_for_relations( @@ -977,6 +978,9 @@ def render_merge( def render_current_timestamp(self) -> str: return "CURRENT_TIMESTAMP" + def render_current_utc_timestamp(self) -> str: + return "(now() AT TIME ZONE 'UTC')" + def create_table_as( self, *, @@ -1706,6 +1710,22 @@ def render_create_source_freshness_index_sqls( "observed_at DESC, run_id DESC)", ) + + def render_create_consumption_index_sqls( + self, *, database: str | None, schema: str + ) -> tuple[str, ...]: + from sqlbuild._native import CONSUMPTION_TABLE_NAME + + table_name: str | None = self.render_qualified_name( + database=database, schema=schema, name=CONSUMPTION_TABLE_NAME + ) + if table_name is None: + return () + return ( + "CREATE INDEX IF NOT EXISTS _sqlbuild_consumption_relation_idx " + f"ON {table_name} (target_schema, target_name, recorded_at)", + ) + def render_insert_source_freshness_records_sql( self, *, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index c66c388a16..5e6d84b86f 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -38,6 +38,7 @@ from sqlbuild.adapter.contract.classes.historical_timestamp_snapshot_sql import ( HistoricalTimestampSnapshotSql, ) +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder @@ -195,7 +196,7 @@ _BULK_COLUMN_RELATION_CHUNK_SIZE: int = 2_000 -class SnowflakeAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): +class SnowflakeAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): """Snowflake adapter backed by snowflake-connector-python.""" adapter_name: ClassVar[str] = BuiltinAdapter.SNOWFLAKE.value @@ -649,6 +650,16 @@ def render_create_source_freshness_index_sqls( del database, schema return () + + def render_create_consumption_table_sql(self, *, database: str | None, schema: str) -> str: + return self._with_state_table_retention( + create_sql=super().render_create_consumption_table_sql( + database=database, schema=schema + ), + database=database, + schema=schema, + ) + def render_create_microbatch_state_table_sql(self, *, database: str | None, schema: str) -> str: return self._with_state_table_retention( create_sql=super().render_create_microbatch_state_table_sql( @@ -1247,6 +1258,9 @@ def render_merge( def render_current_timestamp(self) -> str: return "CURRENT_TIMESTAMP" + def render_current_utc_timestamp(self) -> str: + return "SYSDATE()" + def create_table_as( self, *, diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py index a04a722fec..a437c23000 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py @@ -42,6 +42,7 @@ from sqlbuild.adapter.contract.classes.historical_timestamp_snapshot_sql import ( HistoricalTimestampSnapshotSql, ) +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder @@ -164,7 +165,7 @@ _FALSE_SEED_VALUES: frozenset[str] = frozenset({"false", "f", "no", "n", "0"}) -class SqliteAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): +class SqliteAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): """SQLite adapter: one database file, SQLBuild schemas stored as `schema__name` tables.""" adapter_name: ClassVar[str] = BuiltinAdapter.SQLITE.value @@ -988,6 +989,9 @@ def render_current_timestamp(self) -> str: return "strftime('%Y-%m-%d %H:%M:%f', 'now')" + def render_current_utc_timestamp(self) -> str: + return "strftime('%Y-%m-%dT%H:%M:%f', 'now')" + def create_table_as( self, *, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 7bafe69563..afeac8d5de 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -37,6 +37,7 @@ from sqlbuild.adapter.contract.classes.historical_timestamp_snapshot_sql import ( HistoricalTimestampSnapshotSql, ) +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder @@ -130,7 +131,7 @@ def _render_tsql_distinct_condition(*, left: str, right: str) -> str: ) -class SqlServerAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): +class SqlServerAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): """Microsoft SQL Server adapter backed by pymssql.""" def get_columns_for_relations( @@ -528,6 +529,46 @@ def render_create_source_freshness_index_sqls( "observed_at DESC, run_id DESC)", ) + + def render_create_consumption_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild._native import CONSUMPTION_TABLE_NAME + + create_sql: str = ( + super() + .render_create_consumption_table_sql(database=database, schema=schema) + .replace("CREATE TABLE IF NOT EXISTS ", "CREATE TABLE ", 1) + ) + for column in ("target_schema", "target_name"): + create_sql = create_sql.replace(f"{column} NVARCHAR(MAX)", f"{column} NVARCHAR(450)") + return self._create_table_if_missing_sql( + create_sql=create_sql, + database=database, + schema=schema, + table_name=CONSUMPTION_TABLE_NAME, + ) + + def render_create_consumption_index_sqls( + self, *, database: str | None, schema: str + ) -> tuple[str, ...]: + from sqlbuild._native import CONSUMPTION_TABLE_NAME + + table_name: str | None = self.render_qualified_name( + database=database, schema=schema, name=CONSUMPTION_TABLE_NAME + ) + if table_name is None: + return () + index_name: str = "_sqlbuild_consumption_relation_idx" + escaped_table_name: str = table_name.replace("'", "''") + return ( + "IF NOT EXISTS (" + "SELECT 1 FROM sys.indexes " + f"WHERE name = '{index_name}' " + f"AND object_id = OBJECT_ID(N'{escaped_table_name}')" + ") " + f"CREATE INDEX {index_name} ON {table_name} " + "(target_schema, target_name, recorded_at)", + ) + def render_insert_source_freshness_records_sql( self, *, @@ -2246,6 +2287,9 @@ def render_create_initial_snapshot_destination( def render_current_timestamp(self) -> str: return "CURRENT_TIMESTAMP" + def render_current_utc_timestamp(self) -> str: + return "SYSUTCDATETIME()" + def render_delete_insert( self, *, diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 06203a8fcd..0231a20d37 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -89,7 +89,7 @@ SourceAuditRunResult, SourceLoadPlanEntry, ) -from sqlbuild.executor.build.types import BeforeModelMaterializeCallback +from sqlbuild.executor.build.types import BeforeNodeWriteCallback from sqlbuild.executor.custom.models import MaterializationResult from sqlbuild.executor.functions.constants import FUNCTION_ENTRY_MISSING_CODE from sqlbuild.executor.functions.main._execute import execute_function @@ -209,9 +209,7 @@ def __init__( self._on_node_start: NodeStartCallback | None = callbacks.on_node_start self._on_node_complete: Callable[[object], None] | None = callbacks.on_node_complete self._on_progress: Callable[[str], None] | None = callbacks.on_progress - self._before_model_materialize: BeforeModelMaterializeCallback | None = ( - callbacks.before_model_materialize - ) + self._before_node_write: BeforeNodeWriteCallback | None = callbacks.before_node_write self._custom_materializations: Mapping[str, Callable[..., MaterializationResult]] = ( customizations.custom_materializations or {} ) @@ -986,6 +984,17 @@ def _execute_seed_node(self, *, key: CompiledObjectKey, connection: Any) -> Seed sqlbuild_phase="load", sqlbuild_kind="seed", ): + try: + if self._before_node_write is not None: + self._before_node_write(entry=seed_entry, connection=connection) + except Exception as error: + return SeedExecutionResult( + seed_name=seed_entry.name, + status=ExecutionStatus.FAILED, + error_code=BUILD_WORKER_FAILED_CODE, + error_message=str(error), + duration_ms=int((time.monotonic() - start) * 1000), + ) result: SeedExecutionResult = execute_seed( seed_entry=seed_entry, adapter=self._adapter, @@ -1148,6 +1157,8 @@ def _execute_model_node( ) as deadline, ): try: + if self._before_node_write is not None: + self._before_node_write(entry=model_entry, connection=connection) table_type_entry: TableTypePlanEntry | None = self._table_type_entries.get( model_entry.name ) @@ -1157,8 +1168,6 @@ def _execute_model_node( apply_table_type_conversion( entry=table_type_entry, adapter=self._adapter, connection=connection ) - if self._before_model_materialize is not None: - self._before_model_materialize(entry=model_entry, connection=connection) if ( model_entry.incremental_mode == IncrementalMode.MICROBATCH and model_entry.batch_concurrency > 1 diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index 1a0cedf690..ece162eb01 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -23,7 +23,7 @@ from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.cost.models import StatementExecutionTelemetry from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.executor.build.types import BeforeModelMaterializeCallback, BuildStatus +from sqlbuild.executor.build.types import BeforeNodeWriteCallback, BuildStatus from sqlbuild.executor.custom.models import MaterializationResult from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder @@ -107,7 +107,7 @@ class BuildCallbacks: on_batch_complete: Callable[[MicrobatchBatchProgress], None] | None = None on_node_start: NodeStartCallback | None = None on_node_complete: Callable[[object], None] | None = None - before_model_materialize: BeforeModelMaterializeCallback | None = None + before_node_write: BeforeNodeWriteCallback | None = None on_connection_start: Callable[[int], None] | None = None on_connection_complete: ConnectionElapsedCallback | None = None on_connection_error: ConnectionElapsedCallback | None = None diff --git a/src/sqlbuild/executor/build/types.py b/src/sqlbuild/executor/build/types.py index b8075be76a..109032ccc1 100644 --- a/src/sqlbuild/executor/build/types.py +++ b/src/sqlbuild/executor/build/types.py @@ -6,14 +6,16 @@ from typing import TYPE_CHECKING, Any, Protocol if TYPE_CHECKING: - from sqlbuild._native import ModelPlanEntry + from sqlbuild._native import ModelPlanEntry, SeedPlanEntry from sqlbuild.executor.scheduling.types import ExecutionStatus as ExecutionStatus from sqlbuild.runtime.contracts.types import ExecutionResourceKind as ExecutionResourceKind -class BeforeModelMaterializeCallback(Protocol): - def __call__(self, *, entry: ModelPlanEntry, connection: Any) -> None: ... +class BeforeNodeWriteCallback(Protocol): + """Called before a model or seed changes its relation; raising fails that node.""" + + def __call__(self, *, entry: ModelPlanEntry | SeedPlanEntry, connection: Any) -> None: ... class BuildStatus(StrEnum): diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py index a573ae34f9..2437fb74f4 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py @@ -218,7 +218,7 @@ def test_given_model_materialize_hook_when_build_runs_then_it_prepares_or_fails_ ), ) - def before_model_materialize(entry: ModelPlanEntry, *, connection: object) -> None: + def before_node_write(entry: ModelPlanEntry, *, connection: object) -> None: events.append(f"hook:{entry.name}") hook_actions: dict[bool, Callable[[], object]] = { True: lambda: (_ for _ in ()).throw(RuntimeError("hook failed")), @@ -243,7 +243,7 @@ def before_model_materialize(entry: ModelPlanEntry, *, connection: object) -> No ), callbacks=BuildCallbacks( on_node_start=lambda name, *, resource_kind: events.append(f"start:{name}"), - before_model_materialize=before_model_materialize, + before_node_write=before_node_write, ), ) loaded_rows: tuple[tuple[object, ...], ...] = fetch_rows_or_empty( From 7cbc6bf5085fcc9b72c2c7502cd736706d7c18ec Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:42:59 +0000 Subject: [PATCH 3/8] wip: changed builds through CLI and scheduler with adapter clocks and docs Consumption I/O moves behind a native-host edge, scheduled jobs accept the changed setting and report skipped models as successful, the janitor prunes the table, and the build docs describe --changed. Validated locally: make check-ci, cargo tests for sqlbuild-state, DuckDB and SQLite end-to-end CLI, crash and scheduler tests, and manual PostgreSQL, ClickHouse and SQL Server recorded_at checks. Not yet reviewed; timings pending. --- Makefile | 1 + .../changed_builds/models/customer_orders.sql | 5 + .../changed_builds/models/stg_customers.sql | 2 + .../changed_builds/models/stg_orders.sql | 2 + .../edge_cases/changed_builds/sources/raw.yml | 17 ++ .../_helpers/build_command/execution.rs | 14 +- .../src/bindings/_helpers/cli/registration.rs | 12 +- .../compiled_project/snapshot_execution.rs | 8 + .../src/bindings/_helpers/consumption/mod.rs | 1 - .../bindings/_helpers/consumption/recorder.rs | 110 +++++----- .../_helpers/consumption/relation_reads.rs | 19 +- .../consumption_state.rs} | 142 ++++++++++--- .../src/bindings/_helpers/native_host/mod.rs | 1 + .../{main => _helpers}/event_id.rs | 2 +- .../src/consumption/_helpers/mod.rs | 1 + .../src/consumption/main/mod.rs | 1 - .../src/consumption/main/started_append.rs | 2 +- .../src/consumption/main/succeeded_appends.rs | 2 +- fensu.toml | 2 + .../sqlbuild/references/docs/cli/build.md | 53 +++++ .../sqlbuild/references/docs/cli/janitor.md | 5 +- .../docs/concepts/project-configuration.md | 2 +- .../bigquery/classes/bigquery_adapter.py | 2 +- .../clickhouse/classes/clickhouse_adapter.py | 3 +- .../databricks/classes/databricks_adapter.py | 2 +- .../adapters/duckdb/classes/duckdb_adapter.py | 2 +- .../motherduck/classes/motherduck_adapter.py | 2 +- .../postgres/classes/postgres_adapter.py | 3 +- .../snowflake/classes/snowflake_adapter.py | 3 +- .../adapters/sqlite/classes/sqlite_adapter.py | 2 +- .../sqlserver/classes/sqlserver_adapter.py | 3 +- .../cli/commands/main/build/_test_types.py | 40 ++++ .../cli/commands/main/build/helpers.py | 194 ++++++++++++++++++ .../main/build/test_changed_builds.py | 85 ++++++++ .../publication/_test_types.py | 12 ++ .../compiled_project/publication/helpers.py | 56 +++++ .../publication/test_changed.py | 116 +++++++++++ .../executor/build/_helpers/test_scheduler.py | 3 +- website/src/content/docs/docs/cli/build.mdx | 52 +++++ website/src/content/docs/docs/cli/janitor.mdx | 5 +- .../docs/concepts/project-configuration.mdx | 2 +- 41 files changed, 870 insertions(+), 121 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/changed_builds/models/customer_orders.sql create mode 100644 benchmarks/duckshop/edge_cases/changed_builds/models/stg_customers.sql create mode 100644 benchmarks/duckshop/edge_cases/changed_builds/models/stg_orders.sql create mode 100644 benchmarks/duckshop/edge_cases/changed_builds/sources/raw.yml rename crates/sqlbuild-python/src/bindings/_helpers/{consumption/state_io.rs => native_host/consumption_state.rs} (74%) rename crates/sqlbuild-state/src/consumption/{main => _helpers}/event_id.rs (86%) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py create mode 100644 tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py diff --git a/Makefile b/Makefile index b795940c4c..9aae33ca40 100644 --- a/Makefile +++ b/Makefile @@ -145,6 +145,7 @@ E2E_DUCKDB_BUILD_INCREMENTAL_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_append_cursor_build.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_dropped_relation_rebuild.py \ diff --git a/benchmarks/duckshop/edge_cases/changed_builds/models/customer_orders.sql b/benchmarks/duckshop/edge_cases/changed_builds/models/customer_orders.sql new file mode 100644 index 0000000000..730ca07923 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds/models/customer_orders.sql @@ -0,0 +1,5 @@ +MODEL (description "Orders per customer, two models downstream of both sources.", materialized table); +SELECT c.customer_id, count(o.order_id) AS orders +FROM __ref("stg_customers") AS c +LEFT JOIN __ref("stg_orders") AS o USING (customer_id) +GROUP BY c.customer_id diff --git a/benchmarks/duckshop/edge_cases/changed_builds/models/stg_customers.sql b/benchmarks/duckshop/edge_cases/changed_builds/models/stg_customers.sql new file mode 100644 index 0000000000..73f0c12b81 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds/models/stg_customers.sql @@ -0,0 +1,2 @@ +MODEL (description "Staged customers.", materialized table); +SELECT customer_id FROM __source("raw_customers") diff --git a/benchmarks/duckshop/edge_cases/changed_builds/models/stg_orders.sql b/benchmarks/duckshop/edge_cases/changed_builds/models/stg_orders.sql new file mode 100644 index 0000000000..80711320ca --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds/models/stg_orders.sql @@ -0,0 +1,2 @@ +MODEL (description "Staged orders.", materialized table); +SELECT order_id, customer_id FROM __source("raw_orders") diff --git a/benchmarks/duckshop/edge_cases/changed_builds/sources/raw.yml b/benchmarks/duckshop/edge_cases/changed_builds/sources/raw.yml new file mode 100644 index 0000000000..7c84628bd5 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds/sources/raw.yml @@ -0,0 +1,17 @@ +sources: + - name: raw_orders + description: Synthetic orders, numbered by load batch. + schema: raw + table: orders + freshness: + strategy: column + column: load_id + type: integer + - name: raw_customers + description: Synthetic customers, numbered by load batch. + schema: raw + table: customers + freshness: + strategy: column + column: load_id + type: integer diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 11fa400d02..824b6119f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -38,6 +38,7 @@ use crate::bindings::_helpers::cost_telemetry::context::CostContext; use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timestamp}; use crate::bindings::_helpers::freshness::publication::append_freshness_records; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; +use crate::bindings::_helpers::native_host::consumption_state::recorder_callbacks; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::results::ReadResult; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; @@ -387,6 +388,7 @@ pub(crate) fn execute_build_plan<'py>( )?; report_changed_decision(context, &prepared)?; let recorder: &Bound<'py, ConsumptionRecorder> = &prepared.recorder; + let (on_node_complete, before_node_write) = recorder_callbacks(recorder.as_any())?; let connection: &Bound<'py, PyAny> = &preparation.connection_progress; let build_callbacks: Bound<'py, PyAny> = call_with( &host::entry(py, ExecutorHost::BuildCallbacks)?, @@ -396,8 +398,8 @@ pub(crate) fn execute_build_plan<'py>( "on_node_start", keyword_relay(native_field(callbacks, "on_node_start")?, &NODE_START)?, ), - ("on_node_complete", recorder.getattr("on_node_complete")?), - ("before_node_write", recorder.getattr("before_node_write")?), + ("on_node_complete", on_node_complete), + ("before_node_write", before_node_write), ( "on_sub_progress", native_field(callbacks, "on_sub_progress")?, @@ -497,6 +499,14 @@ fn report_changed_decision(context: &BuildContext<'_>, prepared: &Prepared<'_>) let Some(decision) = &prepared.decision else { return Ok(()); }; + if let Some(worker) = &context.worker { + let skipped: Vec = decision + .skipped + .iter() + .map(|key| format!("{}:{}", key.kind, key.name)) + .collect(); + worker.unchanged(context.py(), &skipped)?; + } emit( &context.stream, &format!( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index fe8fa61bb3..21e77b8f61 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -8,11 +8,11 @@ use crate::bindings::_helpers::cli::{ }; use crate::bindings::_helpers::{ audit_command, build_command, build_progress, change_selection, check, clone, command_runtime, - compute_logs, consumption, contract_command, cost_command, cost_report, cost_telemetry, - dbt_command, dbt_events, dbt_interop, debug_command, diff_report, event_exporting, - execution_history, format_command, freshness, init_command, janitor, load, output_capture, - plan_command, presentation_text, query_command, recall, result_output, review_command, - scenario_command, seed_command, sql_test_command, workspace_commands, + compute_logs, contract_command, cost_command, cost_report, cost_telemetry, dbt_command, + dbt_events, dbt_interop, debug_command, diff_report, event_exporting, execution_history, + format_command, freshness, init_command, janitor, load, output_capture, plan_command, + presentation_text, query_command, recall, result_output, review_command, scenario_command, + seed_command, sql_test_command, workspace_commands, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -28,7 +28,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { janitor::registration::register(module)?; plan_command::registration::register(module)?; freshness::registration::register(module)?; - consumption::state_io::register(module)?; + crate::bindings::_helpers::native_host::consumption_state::register(module)?; clone::registration::register(module)?; compute_logs::registration::register(module)?; execution_history::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index ded2b6c2b6..93451ffa40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -93,6 +93,14 @@ impl WorkerBuild { Ok(()) } + /// Nodes `build --changed` leaves unbuilt are up to date, as a planner `no_change` skip is. + pub(crate) fn unchanged(&self, py: Python<'_>, keys: &[String]) -> PyResult<()> { + for key in keys { + self.complete(py, key, (NodeOutcome::Success, None, None))?; + } + Ok(()) + } + fn complete( &self, py: Python<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/mod.rs index ce58c1c1bb..1e68918e23 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/mod.rs @@ -1,4 +1,3 @@ pub(crate) mod facts; pub(crate) mod recorder; pub(crate) mod relation_reads; -pub(crate) mod state_io; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs index 50b7d0d137..e58f8cd66b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs @@ -5,7 +5,7 @@ use std::sync::Mutex; use std::time::{Duration, Instant}; use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::{pyclass, pymethods}; use sqlbuild_state::consumption::constants::RECENT_EXECUTIONS; use sqlbuild_state::consumption::main::changed_run_set::changed_run_set; @@ -20,8 +20,8 @@ use sqlbuild_state::consumption::models::{ use uuid::Uuid; use crate::bindings::_helpers::consumption::facts::{BuildFacts, build_facts}; -use crate::bindings::_helpers::consumption::state_io::{ - Location, append, ensure_table, read_recent, +use crate::bindings::_helpers::native_host::consumption_state::{ + ConsumptionConnection, Location, NodeCompleteHook, }; use crate::bindings::_helpers::records::fields::field as native_field; @@ -33,6 +33,7 @@ const MODEL_RESULT: &str = "model_name"; const SEED_RESULT: &str = "seed_name"; const SUCCESS: &str = "success"; const ERROR_OUTCOME: &str = "error"; +const SEED_KIND: &str = "seed"; /// The mutable side of one build's recording. struct Recording { @@ -44,17 +45,29 @@ struct Recording { warnings: Vec, } +impl Default for Recording { + fn default() -> Self { + Self { + pending: BTreeMap::new(), + pending_executions: 0, + pending_seeds: Vec::new(), + tables: BTreeSet::new(), + last_flush: Instant::now(), + warnings: Vec::new(), + } + } +} + /// Appends `started` rows before a build and `succeeded` rows as its models and seeds complete. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct ConsumptionRecorder { - adapter: Py, - connection: Py, + connection: ConsumptionConnection, run_id: String, nodes: HashMap, history: ConsumptionHistory, executions: HashMap, failed_start: HashMap, - base_on_node_complete: Py, + base_on_node_complete: NodeCompleteHook, recording: Mutex, } @@ -86,36 +99,38 @@ impl ConsumptionRecorder { .map_err(|_| PyRuntimeError::new_err("consumption recorder state is poisoned")) } - /// Queue the succeeded rows of one execution that wrote its relation. - fn queue(&self, recording: &mut Recording, key: &NodeKey) { + /// The succeeded rows of one execution that wrote its relation, with their schema. + fn succeeded(&self, key: &NodeKey) -> Option<(Location, Vec)> { let (Some(node), Some(execution_id)) = (self.nodes.get(key), self.executions.get(key)) else { - return; + return None; }; let inputs = consumed_inputs(node, (&self.nodes, &self.history), &self.executions); + Some(( + Location::of(&node.relation), + succeeded_appends(node, (execution_id, &self.run_id), &inputs), + )) + } + + /// The recording with `key`'s succeeded rows queued. + fn queued(&self, mut recording: Recording, key: &NodeKey) -> Recording { + if let Some((location, rows)) = self.succeeded(key) { + recording.pending.entry(location).or_default().extend(rows); + recording.pending_executions += 1; + } recording - .pending - .entry(Location::of(&node.relation)) - .or_default() - .extend(succeeded_appends( - node, - (execution_id, &self.run_id), - &inputs, - )); - recording.pending_executions += 1; } - /// Append every pending row; a failure is reported once, and its executions simply run again. - fn flush(&self, py: Python<'_>, recording: &mut Recording) { - let adapter: &Bound<'_, PyAny> = self.adapter.bind(py); - let connection: &Bound<'_, PyAny> = self.connection.bind(py); + /// The recording after appending every pending row; a failure is reported, and its executions run again. + fn flushed(&self, py: Python<'_>, mut recording: Recording) -> Recording { for (location, rows) in std::mem::take(&mut recording.pending) { - let written: PyResult<()> = (|| { - if recording.tables.insert(location.clone()) { - ensure_table(adapter, connection, &location)?; - } - append(adapter, connection, &location, &rows) - })(); + let created: bool = !recording.tables.insert(location.clone()); + let written: PyResult<()> = (if created { + Ok(()) + } else { + self.connection.ensure_table(py, &location) + }) + .and_then(|()| self.connection.append(py, &location, &rows)); if let Err(error) = written { recording.warnings.push(format!( "could not record consumed inputs in schema {}: {error}; those models and seeds run again under --changed", @@ -125,6 +140,7 @@ impl ConsumptionRecorder { } recording.pending_executions = 0; recording.last_flush = Instant::now(); + recording } /// Queue the seeds whose error audits passed, flush everything and close the connection. @@ -145,12 +161,11 @@ impl ConsumptionRecorder { let mut recording = self.lock()?; for seed in std::mem::take(&mut recording.pending_seeds) { if !failed_audits.contains(&seed.name) { - self.queue(&mut recording, &seed); + *recording = self.queued(std::mem::take(&mut *recording), &seed); } } - self.flush(py, &mut recording); - let adapter: &Bound<'_, PyAny> = self.adapter.bind(py); - let closed = adapter.call_method1("close", (self.connection.bind(py),)); + *recording = self.flushed(py, std::mem::take(&mut *recording)); + let closed = self.connection.close(py); let warnings: Vec = std::mem::take(&mut recording.warnings); closed?; Ok(warnings) @@ -161,7 +176,7 @@ impl ConsumptionRecorder { impl ConsumptionRecorder { /// Forward a finished node, then queue a successful model, or a seed until its audits are known. fn on_node_complete(&self, py: Python<'_>, node_result: Bound<'_, PyAny>) -> PyResult<()> { - self.base_on_node_complete.bind(py).call1((&node_result,))?; + self.base_on_node_complete.forward(&node_result)?; let Some((kind, name)) = kind_of(&node_result)? else { return Ok(()); }; @@ -173,15 +188,15 @@ impl ConsumptionRecorder { name, }; let mut recording = self.lock()?; - if kind == "seed" { + if kind == SEED_KIND { recording.pending_seeds.push(key); return Ok(()); } - self.queue(&mut recording, &key); + *recording = self.queued(std::mem::take(&mut *recording), &key); if recording.pending_executions >= FLUSH_EXECUTIONS || recording.last_flush.elapsed() >= FLUSH_INTERVAL { - self.flush(py, &mut recording); + *recording = self.flushed(py, std::mem::take(&mut *recording)); } Ok(()) } @@ -213,8 +228,7 @@ impl ConsumptionRecorder { } } -/// The locations whose records this build reads: all of them under `--changed`, else the -/// relations of unselected upstreams whose last execution the selected nodes record. +/// Schemas to read records from: all under `--changed`, else those of unselected upstreams. fn read_locations(facts: &BuildFacts, changed: bool) -> BTreeSet { let by_key: HashMap<&NodeKey, &NodeFacts> = facts.nodes.iter().map(|node| (&node.key, node)).collect(); @@ -254,11 +268,11 @@ pub(crate) fn prepare<'py>( ) -> PyResult> { let py: Python<'py> = plan.py(); let facts: BuildFacts = build_facts((plan, project), (changed, full_refresh))?; - let connection: Bound<'py, PyAny> = adapter.call_method1("connect", (connection_config,))?; + let connection = ConsumptionConnection::open(adapter, connection_config)?; let mut warnings: Vec = Vec::new(); let mut rows: Vec = Vec::new(); for location in read_locations(&facts, changed) { - match read_recent(adapter, &connection, &location) { + match connection.read_recent(py, &location) { Ok(read) => rows.extend(read), Err(error) => warnings.push(format!( "could not read consumed inputs in schema {}: {error}; its models and seeds count as changed", @@ -312,8 +326,9 @@ pub(crate) fn prepare<'py>( let mut failed_start: HashMap = HashMap::new(); let mut tables: BTreeSet = BTreeSet::new(); for (location, rows) in started { - let written: PyResult<()> = ensure_table(adapter, &connection, &location) - .and_then(|()| append(adapter, &connection, &location, &rows)); + let written: PyResult<()> = connection + .ensure_table(py, &location) + .and_then(|()| connection.append(py, &location, &rows)); match written { Ok(()) => { tables.insert(location); @@ -327,21 +342,16 @@ pub(crate) fn prepare<'py>( } } let recorder = ConsumptionRecorder { - adapter: adapter.clone().unbind(), - connection: connection.unbind(), + connection, run_id, nodes, history, executions, failed_start, - base_on_node_complete: base_on_node_complete.unbind(), + base_on_node_complete: NodeCompleteHook::new(&base_on_node_complete), recording: Mutex::new(Recording { - pending: BTreeMap::new(), - pending_executions: 0, - pending_seeds: Vec::new(), tables, - last_flush: Instant::now(), - warnings: Vec::new(), + ..Recording::default() }), }; Ok(Prepared { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs index 7367c56436..5a286a2592 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs @@ -9,7 +9,7 @@ use crate::bindings::_helpers::analysis::relation_names::{ RelationRow, extract_sql_relation_names, }; use crate::bindings::_helpers::compiled_views::{model_reads, project_reads, seed_reads}; -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::fields::{field as native_field, mapping_items}; /// The relations each model, seed and source names, by node key or source name. pub(crate) struct Locations { @@ -57,7 +57,7 @@ impl Locations { } let mut sources: HashMap, Option, String)> = HashMap::new(); for map in ["source_map", "source_read_map"] { - for item in native_field(plan, map)?.call_method0("items")?.try_iter()? { + for item in mapping_items(&native_field(plan, map)?)?.try_iter()? { let (name, entry): (String, Bound<'_, PyAny>) = item?.extract()?; let table: String = optional(native_field(&entry, "table")?)?.unwrap_or_else(|| name.clone()); @@ -126,11 +126,14 @@ pub(crate) fn model_reads_undeclared( else { return Ok(true); }; - Ok(relations.iter().any(|relation| { - !temporary + for relation in &relations { + let created: bool = temporary .iter() - .any(|created| created.eq_ignore_ascii_case(&relation.0)) - && !own.is_some_and(|own| names(relation, own)) - && !inputs.iter().any(|input| locations.allows(input, relation)) - })) + .any(|created| created.eq_ignore_ascii_case(&relation.0)); + let declared: bool = inputs.iter().any(|input| locations.allows(input, relation)); + if !created && !own.is_some_and(|own| names(relation, own)) && !declared { + return Ok(true); + } + } + Ok(false) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/state_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs similarity index 74% rename from crates/sqlbuild-python/src/bindings/_helpers/consumption/state_io.rs rename to crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs index cee3088181..f6a90ddd00 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/state_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs @@ -1,12 +1,13 @@ -//! Consumption table DDL, recent-execution reads and chunked appends through adapter callbacks. +//! The consumption table through the adapter, and the connection and callbacks a recording build holds. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use chrono::{DateTime, NaiveDateTime, Utc}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyFrozenSet, PyInt, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::CONSUMPTION_TABLE_NAME; use sqlbuild_state::consumption::constants::{ CONSUMPTION_COLUMN_TYPES, INTEGER_STATE_VALUE_TYPE, NULLABLE_COLUMNS, RECENT_EXECUTIONS, - RECORDED_AT_COLUMN, }; use sqlbuild_state::consumption::main::prune_executions_sql::prune_executions_sql; use sqlbuild_state::consumption::main::recent_executions_sql::recent_executions_sql; @@ -16,15 +17,17 @@ use sqlbuild_state::consumption::models::{ use sqlbuild_state::freshness::constants::STRING_STATE_VALUE_TYPE; use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; -use crate::bindings::_helpers::boundary::python_calls::{call_with, imported}; +use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::contracts::state_literals::render_state_sql_literal; -use crate::bindings::_helpers::freshness::support::presumed_utc_datetime; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; -use crate::bindings::_helpers::state_rows::timestamps::date_time_typed; /// Rows per `INSERT`; SQL Server accepts at most 1,000 rows in one `VALUES` list. const ROWS_PER_STATEMENT: usize = 500; +/// Warehouse timestamp texts with a UTC offset, as drivers render aware values. +const OFFSET_TIME_FORMATS: [&str; 2] = ["%Y-%m-%dT%H:%M:%S%.f%:z", "%Y-%m-%d %H:%M:%S%.f%:z"]; +/// Warehouse timestamp texts without an offset, which the consumption table stores as UTC. +const NAIVE_TIME_FORMATS: [&str; 2] = ["%Y-%m-%dT%H:%M:%S%.f", "%Y-%m-%d %H:%M:%S%.f"]; /// One destination schema of the consumption table. #[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] @@ -156,14 +159,19 @@ fn execute<'py>( ) } -/// Create the table and its indexes in `location` when missing. -pub(crate) fn ensure_table( +/// Create the schema, table and indexes in `location`; `started` rows precede schema preparation. +fn ensure_table( adapter: &Bound<'_, PyAny>, connection: &Bound<'_, PyAny>, location: &Location, ) -> PyResult<()> { let py: Python<'_> = adapter.py(); let keywords = location.keywords(py); + for statement in + call_with(&native_field(adapter, "render_create_schema")?, &keywords)?.try_iter()? + { + execute(adapter, connection, &statement?.extract::()?)?; + } let create: String = call_with( &native_field(adapter, "render_create_consumption_table_sql")?, &keywords, @@ -197,7 +205,7 @@ fn table_exists( } /// Rows of the newest executions per relation in `location`; none when the table is missing. -pub(crate) fn read_recent( +fn read_recent( adapter: &Bound<'_, PyAny>, connection: &Bound<'_, PyAny>, location: &Location, @@ -213,8 +221,7 @@ pub(crate) fn read_recent( connection, &recent_executions_sql(&table, RECENT_EXECUTIONS), )?; - result - .call_method0("fetchall")? + call_with(&native_field(&result, "fetchall")?, &[])? .try_iter()? .map(|row| read_row(&row?)) .collect() @@ -231,21 +238,23 @@ fn required_text(value: &Bound<'_, PyAny>) -> PyResult { Ok(optional_text(value)?.unwrap_or_default()) } -/// Microseconds since the epoch of a warehouse timestamp, naive values read as UTC. +/// Microseconds since the epoch of a warehouse timestamp's text, naive values read as UTC. fn epoch_micros(value: &Bound<'_, PyAny>) -> PyResult { - let py: Python<'_> = value.py(); - let moment: Bound<'_, PyAny> = if date_time_typed(value)? { - value.clone() - } else { - imported(py, "datetime", "datetime")?.call_method1("fromisoformat", (value.str()?,))? - }; - let utc: Bound<'_, PyAny> = presumed_utc_datetime(&moment)?; - let epoch: Bound<'_, PyAny> = imported(py, "datetime", "datetime")? - .call_method1("fromtimestamp", (0, imported(py, "datetime", "UTC")?))?; - let micros: Bound<'_, PyAny> = imported(py, "datetime", "timedelta")?.call1((0, 0, 1))?; - utc.call_method1("__sub__", (epoch,))? - .call_method1("__floordiv__", (micros,))? - .extract() + let text: String = value.str()?.extract()?; + let text: &str = text.trim(); + for format in OFFSET_TIME_FORMATS { + if let Ok(moment) = DateTime::parse_from_str(text, format) { + return Ok(moment.with_timezone(&Utc).timestamp_micros()); + } + } + for format in NAIVE_TIME_FORMATS { + if let Ok(naive) = NaiveDateTime::parse_from_str(text, format) { + return Ok(naive.and_utc().timestamp_micros()); + } + } + Err(PyValueError::new_err(format!( + "unreadable consumption time {text:?}" + ))) } fn read_row(row: &Bound<'_, PyAny>) -> PyResult { @@ -326,7 +335,7 @@ fn row_cells(py: Python<'_>, row: &ConsumptionAppend, recorded_at: &str) -> PyRe } /// Append `rows` to `location`'s table in statements of whole executions where they fit. -pub(crate) fn append( +fn append( adapter: &Bound<'_, PyAny>, connection: &Bound<'_, PyAny>, location: &Location, @@ -336,14 +345,12 @@ pub(crate) fn append( let Some(table) = table_of(adapter, location)? else { return Ok(()); }; - let recorded_at: String = adapter - .call_method0("render_current_utc_timestamp")? - .extract()?; + let recorded_at: String = + call_with(&native_field(adapter, "render_current_utc_timestamp")?, &[])?.extract()?; let columns: Vec<&str> = CONSUMPTION_COLUMN_TYPES .iter() .map(|(column, _)| *column) .collect(); - debug_assert_eq!(columns.last(), Some(&RECORDED_AT_COLUMN)); for chunk in execution_chunks(rows) { let cells: Vec> = chunk .iter() @@ -384,6 +391,81 @@ fn execution_chunks(rows: &[ConsumptionAppend]) -> Vec<&[ConsumptionAppend]> { chunks } +/// The adapter and the connection one recording build appends through. +pub(crate) struct ConsumptionConnection { + adapter: Py, + connection: Py, +} + +impl ConsumptionConnection { + pub(crate) fn open( + adapter: &Bound<'_, PyAny>, + connection_config: &Bound<'_, PyAny>, + ) -> PyResult { + let connection: Bound<'_, PyAny> = + native_field(adapter, "connect")?.call1((connection_config,))?; + Ok(Self { + adapter: adapter.clone().unbind(), + connection: connection.unbind(), + }) + } + + pub(crate) fn ensure_table(&self, py: Python<'_>, location: &Location) -> PyResult<()> { + ensure_table(self.adapter.bind(py), self.connection.bind(py), location) + } + + pub(crate) fn read_recent( + &self, + py: Python<'_>, + location: &Location, + ) -> PyResult> { + read_recent(self.adapter.bind(py), self.connection.bind(py), location) + } + + pub(crate) fn append( + &self, + py: Python<'_>, + location: &Location, + rows: &[ConsumptionAppend], + ) -> PyResult<()> { + append( + self.adapter.bind(py), + self.connection.bind(py), + location, + rows, + ) + } + + pub(crate) fn close(&self, py: Python<'_>) -> PyResult<()> { + native_field(self.adapter.bind(py), "close")?.call1((self.connection.bind(py),))?; + Ok(()) + } +} + +/// The build's own node-completion callback, which the recorder forwards every result to. +pub(crate) struct NodeCompleteHook(Py); + +impl NodeCompleteHook { + pub(crate) fn new(callback: &Bound<'_, PyAny>) -> Self { + Self(callback.clone().unbind()) + } + + pub(crate) fn forward(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { + self.0.bind(result.py()).call1((result,))?; + Ok(()) + } +} + +/// The recorder's `on_node_complete` and `before_node_write` methods, as build callbacks. +pub(crate) fn recorder_callbacks<'py>( + recorder: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + Ok(( + recorder.getattr("on_node_complete")?, + recorder.getattr("before_node_write")?, + )) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!( build_consumption_create_table_sql, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 849f8bf165..3fb073463c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod connections; pub(crate) mod constants; +pub(crate) mod consumption_state; pub(crate) mod header_projection; pub(crate) mod host; pub(crate) mod interrupt; diff --git a/crates/sqlbuild-state/src/consumption/main/event_id.rs b/crates/sqlbuild-state/src/consumption/_helpers/event_id.rs similarity index 86% rename from crates/sqlbuild-state/src/consumption/main/event_id.rs rename to crates/sqlbuild-state/src/consumption/_helpers/event_id.rs index 4aadd86844..4eeb317f1a 100644 --- a/crates/sqlbuild-state/src/consumption/main/event_id.rs +++ b/crates/sqlbuild-state/src/consumption/_helpers/event_id.rs @@ -6,7 +6,7 @@ use crate::consumption::constants::EVENT_NAMESPACE; use crate::consumption::models::ConsumedInput; /// SHA-256 of `(execution_id, phase, input_kind, input_name, input_version)`, so a retried append repeats it. -pub fn event_id(execution_id: &str, phase: &str, input: Option<&ConsumedInput>) -> String { +pub(crate) fn event_id(execution_id: &str, phase: &str, input: Option<&ConsumedInput>) -> String { let identity = serde_json::json!([ EVENT_NAMESPACE, execution_id, diff --git a/crates/sqlbuild-state/src/consumption/_helpers/mod.rs b/crates/sqlbuild-state/src/consumption/_helpers/mod.rs index d7ef1a72d2..06e3432e89 100644 --- a/crates/sqlbuild-state/src/consumption/_helpers/mod.rs +++ b/crates/sqlbuild-state/src/consumption/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod decision; +pub(crate) mod event_id; pub(crate) mod history; pub(crate) mod rules; diff --git a/crates/sqlbuild-state/src/consumption/main/mod.rs b/crates/sqlbuild-state/src/consumption/main/mod.rs index 2ed00b0cf7..8f097fabfd 100644 --- a/crates/sqlbuild-state/src/consumption/main/mod.rs +++ b/crates/sqlbuild-state/src/consumption/main/mod.rs @@ -1,7 +1,6 @@ pub mod changed_run_set; pub mod consumed_inputs; pub mod consumption_history; -pub mod event_id; pub mod prune_executions_sql; pub mod recent_executions_sql; pub mod started_append; diff --git a/crates/sqlbuild-state/src/consumption/main/started_append.rs b/crates/sqlbuild-state/src/consumption/main/started_append.rs index 70df769893..bcd8554c79 100644 --- a/crates/sqlbuild-state/src/consumption/main/started_append.rs +++ b/crates/sqlbuild-state/src/consumption/main/started_append.rs @@ -1,8 +1,8 @@ //! The row appended before a model or seed writes. use crate::consumption::_helpers::decision::input_key; +use crate::consumption::_helpers::event_id::event_id; use crate::consumption::constants::STARTED_PHASE; -use crate::consumption::main::event_id::event_id; use crate::consumption::models::{ConsumptionAppend, NodeFacts}; /// The `started` row of one execution of `node`; its input count is the node's distinct direct inputs. diff --git a/crates/sqlbuild-state/src/consumption/main/succeeded_appends.rs b/crates/sqlbuild-state/src/consumption/main/succeeded_appends.rs index c68cec364e..092f796cfe 100644 --- a/crates/sqlbuild-state/src/consumption/main/succeeded_appends.rs +++ b/crates/sqlbuild-state/src/consumption/main/succeeded_appends.rs @@ -1,7 +1,7 @@ //! The rows appended after a model or seed's write succeeded. +use crate::consumption::_helpers::event_id::event_id; use crate::consumption::constants::SUCCEEDED_PHASE; -use crate::consumption::main::event_id::event_id; use crate::consumption::models::{ConsumedInput, ConsumptionAppend, NodeFacts}; /// One `succeeded` row per consumed input, or one input-less row when there are none. diff --git a/fensu.toml b/fensu.toml index ecc2f19c8d..fcccc9bfbe 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2623,6 +2623,7 @@ module_layers = [ "compute_logs=output", "contract_command=commands", "contract_records=infra", + "consumption=plan", "contracts=project", "contracts/effective_config=front", "contracts/resource_identities=infra", @@ -2674,6 +2675,7 @@ module_layers = [ "microbatches=plan", "model_config=front", "native_host=infra", + "native_host/consumption_state=front", "native_host/model_lookup=project", "native_host/orchestration_sensors=commands", "native_host/orchestration_triggers=project", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index 7d2eee3bf4..5bd84c5745 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -12,6 +12,7 @@ Online: https://sqlbuild.com/docs/cli/build/ - Global options - Flags - Fast iteration +- Building only what changed - Execution order - Output - Deferred builds @@ -55,6 +56,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | | `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | | `--full-refresh` | Drop and rebuild selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model even without this flag | +| `--changed` | Build only the selected models and seeds whose definition or inputs changed since they last succeeded; see [Building only what changed](#building-only-what-changed) | | `--defer-to` | Resolve unselected model references against another target | | `--defer-sources-to` | Read managed source data from another target | | `--fail-fast` | Stop on first failure and skip remaining nodes | @@ -87,6 +89,57 @@ sqb build --no-tests --no-audits This replaces the former `sqb run` command. The full lifecycle (tests + audits) is always the default; skip flags opt out of specific phases when you need speed. +## Building only what changed + +`--changed` skips each selected model or seed that would read exactly what it read the last time +it built successfully: + +```bash +sqb build --changed +``` + +``` +Changed: 2 of 3 selected models and seeds run; 1 unchanged are skipped. +``` + +Add `--verbose` to print why each one runs, such as `input raw_orders changed`. + +A selected model or seed runs when any of these is true: + +| Reason | Example | +|--------|---------| +| It has no successful build on record | A new model | +| Its last build did not finish | The build was killed or failed after the model started writing | +| Its relation is missing, or `--full-refresh` or a cursor override applies | `sqb build --changed --full-refresh` | +| Its definition changed | Edited SQL, config, hooks, contract or a function it calls | +| A source it reads has a different [freshness](../concepts/sources.md#source-freshness) value | New rows raise `MAX(loaded_at)` | +| An upstream model or seed was rebuilt since, or runs in this build | `stg_orders` ran, so `customer_orders` runs | +| An upstream model or seed needs to run, even when it is not selected | `sqb build customer_orders --changed` runs `customer_orders` on every build until a changed `stg_orders` is built | +| What it reads can't be listed | SQL that reads relations not declared as sources or models, or a Python task or asset upstream | + +A model that runs under `--changed` plans exactly as in a plain build: its +[`replay_on_change`](../concepts/planning/replay-decisions.md) policy still decides how much +history an incremental model reprocesses. A model that runs only because an upstream was rebuilt +takes its normal incremental step. + +In a scheduled job, set `settings={'changed': True}`. Skipped models count as successful nodes of +the run. + +Every build, with or without `--changed`, records what each model and seed read in +`_sqlbuild_consumption` in its target schema, so the build needs permission to create and insert +into that table. [`sqb janitor`](janitor.md) keeps the newest +`direct_state_history_versions` builds of each relation. + +Limitations: + +- A source is compared only through its freshness value. A change that doesn't move it, such as an + update that keeps the old `loaded_at`, isn't seen. A source without freshness configuration counts + as changed on every build. +- SQL that depends on the time, such as `current_date`, or on random values never counts as changed. + Build those models without `--changed`. +- Two builds of one model less than 5 seconds apart can't be ordered, so its downstream models run + on the next `--changed` build. + ## Execution order 1. Managed sources are loaded (unless `--no-load`, which audits their existing data instead) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md index 0f416c4451..998306878c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md @@ -160,8 +160,9 @@ like a review schema is left alone. ### State history pruning The janitor prunes old rows from `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness`, keeping -the latest `direct_state_history_versions` rows per identity (default `20`). This is separate from -relation retention. +the latest `direct_state_history_versions` rows per identity (default `20`), and from +`_sqlbuild_consumption`, keeping the latest `direct_state_history_versions` builds of each model and +seed. This is separate from relation retention. ## Safety diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 9a67ec2de1..9fa72d06df 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -809,7 +809,7 @@ keep_in_use = false | `archive_retention_days` | `14` | How many days an archive is kept before it is deleted. `0` deletes it in the same run. | | `delete_tracked_only` | `true` | Only clean relations that appear in fingerprint tracking | | `exclude_patterns` | `[]` | Glob patterns for relations to skip | -| `direct_state_history_versions` | `20` | How many state-history rows to keep per identity in `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness` | +| `direct_state_history_versions` | `20` | How many state-history rows to keep per identity in `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness`, and how many builds of each model and seed to keep in `_sqlbuild_consumption` | | `in_use_window` | `"7d"` | How far back the janitor looks for reads of each relation it would archive. A fixed duration such as `"7d"` or `"36h"`. See [Relations in use](../cli/janitor.md#relations-in-use) | | `keep_in_use` | `false` | Keep relations read within `in_use_window` instead of archiving them. `--keep-in-use` turns it on for one run | diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index dd0a9ca300..ac7ed14d08 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -27,6 +27,7 @@ _typed_collection_items, _typed_scalar_payload, ) +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.historical_check_snapshot_sql import ( HistoricalCheckSnapshotSql, ) @@ -36,7 +37,6 @@ from sqlbuild.adapter.contract.classes.historical_timestamp_snapshot_sql import ( HistoricalTimestampSnapshotSql, ) -from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 3a33dd0088..4155286972 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -29,8 +29,8 @@ _typed_scalar_payload, _validate_rectangular_typed_array, ) -from sqlbuild.adapter.contract.classes.historical_snapshot_sql import render_is_distinct_from from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin +from sqlbuild.adapter.contract.classes.historical_snapshot_sql import render_is_distinct_from from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder @@ -1921,7 +1921,6 @@ def render_create_source_freshness_table_sql(self, *, database: str | None, sche super().render_create_source_freshness_table_sql(database=database, schema=schema) ) - def render_create_consumption_table_sql(self, *, database: str | None, schema: str) -> str: return _state_table_ddl( super().render_create_consumption_table_sql(database=database, schema=schema) diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 44c79bf7d6..59c4438f80 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -26,6 +26,7 @@ _render_ansi_typed_scalar, _render_typed_value_list, ) +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.historical_check_snapshot_sql import ( HistoricalCheckSnapshotSql, ) @@ -35,7 +36,6 @@ from sqlbuild.adapter.contract.classes.historical_timestamp_snapshot_sql import ( HistoricalTimestampSnapshotSql, ) -from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder diff --git a/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py b/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py index 7f9e86563e..faf2a9effb 100644 --- a/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py +++ b/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py @@ -7,8 +7,8 @@ from uuid import uuid4 from sqlbuild.adapter.contract.classes.base_adapter import _quote_sql_string -from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin +from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.microbatches.constants import MICROBATCH_GENERATION_COMMENT_PREFIX diff --git a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py index 27b289c9ff..c3ef1b029d 100644 --- a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py +++ b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py @@ -8,8 +8,8 @@ from typing import Any, ClassVar from urllib.parse import urlencode -from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin +from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.types import BuiltinAdapter, LoaderLogicalType diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 5a08f97b33..918ea98ffb 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -28,6 +28,7 @@ _render_typed_value_list, _validate_rectangular_typed_array, ) +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.historical_check_snapshot_sql import ( HistoricalCheckSnapshotSql, ) @@ -37,7 +38,6 @@ from sqlbuild.adapter.contract.classes.historical_timestamp_snapshot_sql import ( HistoricalTimestampSnapshotSql, ) -from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder @@ -1710,7 +1710,6 @@ def render_create_source_freshness_index_sqls( "observed_at DESC, run_id DESC)", ) - def render_create_consumption_index_sqls( self, *, database: str | None, schema: str ) -> tuple[str, ...]: diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 5e6d84b86f..5a5990ea81 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -29,6 +29,7 @@ _render_ansi_typed_scalar, _render_typed_value_list, ) +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.historical_check_snapshot_sql import ( HistoricalCheckSnapshotSql, ) @@ -38,7 +39,6 @@ from sqlbuild.adapter.contract.classes.historical_timestamp_snapshot_sql import ( HistoricalTimestampSnapshotSql, ) -from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder @@ -650,7 +650,6 @@ def render_create_source_freshness_index_sqls( del database, schema return () - def render_create_consumption_table_sql(self, *, database: str | None, schema: str) -> str: return self._with_state_table_retention( create_sql=super().render_create_consumption_table_sql( diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py index a437c23000..f52330e32f 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py @@ -33,6 +33,7 @@ _render_explicit_source_relation_cast_subquery, _render_typed_value_list, ) +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.historical_check_snapshot_sql import ( HistoricalCheckSnapshotSql, ) @@ -42,7 +43,6 @@ from sqlbuild.adapter.contract.classes.historical_timestamp_snapshot_sql import ( HistoricalTimestampSnapshotSql, ) -from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index afeac8d5de..2a6dab69f1 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -28,6 +28,7 @@ _render_typed_value_list, _typed_scalar_payload, ) +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.historical_check_snapshot_sql import ( HistoricalCheckSnapshotSql, ) @@ -37,7 +38,6 @@ from sqlbuild.adapter.contract.classes.historical_timestamp_snapshot_sql import ( HistoricalTimestampSnapshotSql, ) -from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder @@ -529,7 +529,6 @@ def render_create_source_freshness_index_sqls( "observed_at DESC, run_id DESC)", ) - def render_create_consumption_table_sql(self, *, database: str | None, schema: str) -> str: from sqlbuild._native import CONSUMPTION_TABLE_NAME diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index aef671c86b..a6152e378e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -5,6 +5,8 @@ import subprocess from collections.abc import Callable from dataclasses import dataclass, field +from pathlib import Path +from typing import NamedTuple @dataclass(frozen=True) @@ -1524,3 +1526,41 @@ class QuotedConfiguredColumnsE2ETestCase: cursor: str expected_history_rows: list[tuple[object, ...]] expected_activity_rows: list[tuple[object, ...]] + + +class ChangedWarehouse(NamedTuple): + """One adapter's changed-builds project: configuration, raw tables and how to query it.""" + + name: str + project_toml: str + warehouse_file: str + raw_tables_sql: tuple[str, ...] + consumption_table: str + orders_table: str + customers_table: str + slow_scan_cte: str + execute: Callable[[Path, str], list[tuple[object, ...]]] + + +class ChangedBuildStep(NamedTuple): + """One mutation of the warehouse or project followed by one build.""" + + description: str + warehouse_sql: tuple[str, ...] + project_files: dict[str, str] + arguments: tuple[str, ...] + + +@dataclass(frozen=True) +class ChangedBuildE2ETestCase: + description: str + warehouse: ChangedWarehouse + expected_ran: tuple[tuple[str, frozenset[str]], ...] + + +@dataclass(frozen=True) +class ChangedCrashE2ETestCase: + description: str + warehouse: ChangedWarehouse + expected_ran: frozenset[str] + expected_killed_rows: list[tuple[object, ...]] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index a71e643d6e..f75ccf0029 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -1,15 +1,26 @@ from __future__ import annotations import json +import re +import signal +import sqlite3 import subprocess +import time +from contextlib import closing +from itertools import takewhile from pathlib import Path from textwrap import dedent from typing import Any +import duckdb + from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + ChangedBuildStep, + ChangedWarehouse, DeferCloneBuildE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + REPO_ROOT, execute_duckdb, prepare_inline_project, query_duckdb, @@ -1234,3 +1245,186 @@ def target_concurrency_microbatch_files() -> dict[str, str]: "WHERE event_time >= __cursor_start() AND event_time < __cursor_end()\n" ), } + + +CHANGED_BUILDS_FIXTURE: Path = REPO_ROOT / "benchmarks/duckshop/edge_cases/changed_builds" +CHANGED_ORDER_SKEW_SECONDS: float = 5.2 +_CHANGED_PROGRESS_PATTERN: re.Pattern[str] = re.compile( + r"^\s+\d+/\d+\s+\S+\s+(\S+)\s+OK\b", re.MULTILINE +) +_CHANGED_SLOW_MODEL: str = ( + 'MODEL (description "Order totals over a long synthetic scan.", materialized table);\n' + "WITH {scan}\n" + "SELECT o.customer_id, count(*) AS weight\n" + 'FROM __ref("stg_orders") AS o\n' + "CROSS JOIN scan\n" + "GROUP BY o.customer_id\n" +) + + +def _duckdb_rows(path: Path, sql: str) -> list[tuple[object, ...]]: + with duckdb.connect(str(path)) as connection: + return connection.execute(sql).fetchall() + + +def _sqlite_rows(path: Path, sql: str) -> list[tuple[object, ...]]: + with closing(sqlite3.connect(path)) as connection, connection: + return connection.execute(sql).fetchall() + + +DUCKDB_CHANGED: ChangedWarehouse = ChangedWarehouse( + name="duckdb", + project_toml=( + 'name = "changed_builds"\nadapter = "duckdb"\n\n[connection]\ndatabase = "shop.duckdb"\n' + ), + warehouse_file="shop.duckdb", + raw_tables_sql=( + "CREATE SCHEMA raw", + "CREATE TABLE raw.orders AS SELECT 1 AS order_id, 1 AS customer_id, 1 AS load_id", + "CREATE TABLE raw.customers AS SELECT 1 AS customer_id, 1 AS load_id", + ), + consumption_table="main._sqlbuild_consumption", + orders_table="raw.orders", + customers_table="raw.customers", + slow_scan_cte="scan AS (SELECT r.i FROM range(4000000000) AS r(i))", + execute=_duckdb_rows, +) +SQLITE_CHANGED: ChangedWarehouse = ChangedWarehouse( + name="sqlite", + project_toml=( + 'name = "changed_builds"\nadapter = "sqlite"\n\n' + '[connection]\npath = "shop.sqlite"\n\n[targets.dev]\nschema = "main"\n' + ), + warehouse_file="shop.sqlite", + raw_tables_sql=( + "CREATE TABLE raw__orders (order_id INTEGER, customer_id INTEGER, load_id INTEGER)", + "INSERT INTO raw__orders VALUES (1, 1, 1)", + "CREATE TABLE raw__customers (customer_id INTEGER, load_id INTEGER)", + "INSERT INTO raw__customers VALUES (1, 1)", + ), + consumption_table="_sqlbuild_consumption", + orders_table="raw__orders", + customers_table="raw__customers", + slow_scan_cte=( + "RECURSIVE scan(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM scan WHERE i < 30000000)" + ), + execute=_sqlite_rows, +) + + +def prepare_changed_project(*, tmp_path: Path, warehouse: ChangedWarehouse) -> Path: + """The DuckShop changed-builds edge case on `warehouse`, with its raw tables loaded.""" + + files: dict[str, str] = { + str(path.relative_to(CHANGED_BUILDS_FIXTURE)): path.read_text(encoding="utf-8") + for path in sorted(CHANGED_BUILDS_FIXTURE.glob("*/*")) + } + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="changed_builds", + repo_files=files | {"sqlbuild_project.toml": warehouse.project_toml}, + ) + for sql in warehouse.raw_tables_sql: + warehouse.execute(project_dir / warehouse.warehouse_file, sql) + return project_dir + + +def run_changed_build(project_dir: Path, arguments: tuple[str, ...]) -> frozenset[str]: + """Wait out the order skew, build successfully, and return the models and seeds written.""" + + time.sleep(CHANGED_ORDER_SKEW_SECONDS) + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", *arguments), project_dir=project_dir + ) + assert result.returncode == 0, result.stdout + result.stderr + return frozenset(_CHANGED_PROGRESS_PATTERN.findall(result.stdout)) + + +def apply_changed_step( + project_dir: Path, warehouse: ChangedWarehouse, step: ChangedBuildStep +) -> None: + """Apply one step's warehouse statements and project edits.""" + + for sql in step.warehouse_sql: + warehouse.execute(project_dir / warehouse.warehouse_file, sql) + for relative_path, contents in step.project_files.items(): + (project_dir / relative_path).write_text(contents, encoding="utf-8") + + +def kill_changed_build_mid_write( + project_dir: Path, warehouse: ChangedWarehouse +) -> tuple[frozenset[str], list[tuple[object, ...]]]: + """Kill a build while a slow model writes; return what it finished and the slow model's rows.""" + + slow_model: Path = project_dir / "models/order_weights.sql" + slow_model.write_text( + _CHANGED_SLOW_MODEL.format(scan=warehouse.slow_scan_cte), encoding="utf-8" + ) + time.sleep(CHANGED_ORDER_SKEW_SECONDS) + build: subprocess.Popen[str] = subprocess.Popen( + ( + str(REPO_ROOT / ".venv/bin/sqb"), + "--no-color", + "--project-dir", + str(project_dir), + "build", + "--concurrency", + "1", + ), + cwd=REPO_ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + assert build.stdout is not None + output: str = "".join(takewhile(lambda line: "order_weights START" not in line, build.stdout)) + time.sleep(0.5) + build.send_signal(signal.SIGKILL) + build.wait(timeout=30) + build.stdout.close() + slow_model.unlink() + rows: list[tuple[object, ...]] = warehouse.execute( + project_dir / warehouse.warehouse_file, + f"SELECT node_name, phase FROM {warehouse.consumption_table} " + "WHERE node_name = 'order_weights'", + ) + return frozenset(_CHANGED_PROGRESS_PATTERN.findall(output)), rows + + +def changed_build_steps(warehouse: ChangedWarehouse) -> tuple[ChangedBuildStep, ...]: + """Mutations and builds whose `--changed` run sets follow from consumed inputs alone.""" + + return ( + ChangedBuildStep("first --changed build", (), {}, ("--changed",)), + ChangedBuildStep("nothing changed", (), {}, ("--changed",)), + ChangedBuildStep( + "a new order", + (f"INSERT INTO {warehouse.orders_table} VALUES (2, 1, 2)",), + {}, + ("--changed",), + ), + ChangedBuildStep( + "a definition change", + (), + { + "models/stg_customers.sql": ( + 'MODEL (description "Staged customers.", materialized table);\n' + 'SELECT customer_id FROM __source("raw_customers") WHERE customer_id > 0\n' + ) + }, + ("--changed",), + ), + ChangedBuildStep( + "a stale unselected upstream", + (f"INSERT INTO {warehouse.customers_table} VALUES (2, 2)",), + {}, + ("--changed", "--select", "customer_orders"), + ), + ChangedBuildStep( + "the upstream is still stale", (), {}, ("--changed", "--select", "customer_orders") + ), + ChangedBuildStep("building the whole selection", (), {}, ("--changed",)), + ChangedBuildStep("a plain build", (), {}, ()), + ChangedBuildStep("after the plain build", (), {}, ("--changed",)), + ChangedBuildStep("full refresh", (), {}, ("--changed", "--full-refresh")), + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py new file mode 100644 index 0000000000..3e49d83520 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py @@ -0,0 +1,85 @@ +"""`sqb build --changed` runs exactly what consumed changed inputs, across crashes, on DuckDB and SQLite.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + ChangedBuildE2ETestCase, + ChangedCrashE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( + DUCKDB_CHANGED, + SQLITE_CHANGED, + apply_changed_step, + changed_build_steps, + kill_changed_build_mid_write, + prepare_changed_project, + run_changed_build, +) + +EVERY_MODEL: frozenset[str] = frozenset({"stg_orders", "stg_customers", "customer_orders"}) +EXPECTED_CHANGED_RUNS: tuple[tuple[str, frozenset[str]], ...] = ( + ("first --changed build", EVERY_MODEL), + ("nothing changed", frozenset()), + ("a new order", frozenset({"stg_orders", "customer_orders"})), + ("a definition change", frozenset({"stg_customers", "customer_orders"})), + ("a stale unselected upstream", frozenset({"customer_orders"})), + ("the upstream is still stale", frozenset({"customer_orders"})), + ("building the whole selection", frozenset({"stg_customers", "customer_orders"})), + ("a plain build", EVERY_MODEL), + ("after the plain build", frozenset()), + ("full refresh", EVERY_MODEL), +) + + +@pytest.mark.parametrize( + "test_case", + [ + ChangedBuildE2ETestCase( + description=warehouse.name, warehouse=warehouse, expected_ran=EXPECTED_CHANGED_RUNS + ) + for warehouse in (DUCKDB_CHANGED, SQLITE_CHANGED) + ], + ids=lambda case: case.description, +) +def test_given_input_changes_when_building_with_changed_then_exactly_their_consumers_run( + test_case: ChangedBuildE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_changed_project(tmp_path=tmp_path, warehouse=test_case.warehouse) + observed: list[tuple[str, frozenset[str]]] = [] + for step in changed_build_steps(test_case.warehouse): + apply_changed_step(project_dir, test_case.warehouse, step) + observed.append((step.description, run_changed_build(project_dir, step.arguments))) + + assert tuple(observed) == test_case.expected_ran + + +@pytest.mark.parametrize( + "test_case", + [ + ChangedCrashE2ETestCase( + description=warehouse.name, + warehouse=warehouse, + expected_ran=EVERY_MODEL, + expected_killed_rows=[("order_weights", "started")], + ) + for warehouse in (DUCKDB_CHANGED, SQLITE_CHANGED) + ], + ids=lambda case: case.description, +) +def test_given_a_build_killed_mid_write_when_building_with_changed_then_every_interrupted_model_runs( + test_case: ChangedCrashE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_changed_project(tmp_path=tmp_path, warehouse=test_case.warehouse) + first: frozenset[str] = run_changed_build(project_dir, ("--changed",)) + settled: frozenset[str] = run_changed_build(project_dir, ("--changed",)) + finished, killed_rows = kill_changed_build_mid_write(project_dir, test_case.warehouse) + + assert (first, settled) == (test_case.expected_ran, frozenset()) + assert "stg_orders" in finished + assert killed_rows == test_case.expected_killed_rows + assert run_changed_build(project_dir, ("--changed",)) == test_case.expected_ran + assert run_changed_build(project_dir, ("--changed",)) == frozenset() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index c9866865b7..03c0519a35 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -325,3 +325,15 @@ class SchedulerCrashTestCase: fixture: str expected_anomalies: dict[str, int] expected_minimum_kills: int + + +@dataclass(frozen=True, kw_only=True) +class ChangedJobTestCase(WorkerExecutionTestCase): + description: str + project_toml: str + warehouse_file: str + consumption_table: str + raw_tables_sql: tuple[str, ...] + new_order_sql: str + execute: Callable[[Path, str], list[tuple[Any, ...]]] + expected_runs: tuple[tuple[str, frozenset[str], dict[str, int]], ...] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 28fb6f4797..8aaeb52516 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -1221,3 +1221,59 @@ def scheduler_crash_history(directory: Path, fixture: dict[str, Any]) -> dict[st "ticks_before_survivor": interrupted, "ticks": len(ticks), } + + +CHANGED_FIXTURE: Path = Path("benchmarks/duckshop/edge_cases/changed_builds") +CHANGED_JOB: str = ( + "from sqlbuild.orchestration import job\n" + "changed_orders = job('changed_orders', description='Build what changed.', " + "select='stg_orders stg_customers customer_orders', settings={'changed': True})\n" +) + + +def duckdb_rows(path: Path, sql: str) -> list[tuple[Any, ...]]: + """Run `sql` on a DuckDB warehouse file and return its rows.""" + with duckdb.connect(str(path)) as connection: + return connection.execute(sql).fetchall() + + +def sqlite_rows(path: Path, sql: str) -> list[tuple[Any, ...]]: + """Run and commit `sql` on a SQLite warehouse file and return its rows.""" + with closing(sqlite3.connect(path)) as connection, connection: + return connection.execute(sql).fetchall() + + +def changed_job_files(project_toml: str) -> dict[str, str]: + """The DuckShop changed-builds edge case with one scheduled `changed` job.""" + return { + str(path.relative_to(CHANGED_FIXTURE)): path.read_text() + for path in sorted(CHANGED_FIXTURE.glob("*/*")) + } | {"orchestration/orders.py": CHANGED_JOB, "sqlbuild_project.toml": project_toml} + + +def request_job_run(worker: ScheduledWorker, job: str) -> dict[str, Any]: + """Queue `job` through privileged administration and wait for its terminal run.""" + run_id: str = str(uuid4()) + requested: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "request-run", + worker.api.location_id, + job, + "--target", + "development", + "--run-id", + run_id, + "--reason", + "Build what changed", + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert requested.returncode == 0, requested.stderr + return wait_for_run(worker.api, worker.process, worker.log, run_id=run_id) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py new file mode 100644 index 0000000000..bdae8b9e69 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py @@ -0,0 +1,116 @@ +"""Scheduled jobs with `changed` run only models whose consumed inputs changed, on DuckDB and SQLite.""" + +import time +from dataclasses import replace +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ChangedJobTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + changed_job_files, + duckdb_rows, + request_job_run, + scheduled_worker, + sqlite_rows, +) + +ORDER_SKEW_SECONDS: float = 5.2 +EXPECTED_CHANGED_RUNS: tuple[tuple[str, frozenset[str], dict[str, int]], ...] = ( + ("first run", frozenset({"stg_orders", "stg_customers", "customer_orders"}), {"success": 3}), + ("nothing changed", frozenset(), {"success": 3}), + ("a new order", frozenset({"stg_orders", "customer_orders"}), {"success": 3}), +) + + +@pytest.mark.parametrize( + "test_case", + [ + ChangedJobTestCase( + description="DuckDB job with changed runs only consumers of new orders", + model_sql="SELECT 1 AS amount", + warehouse_query="", + expected_status="success", + expected_rows=[], + project_toml=( + 'name = "orders"\nadapter = "duckdb"\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n' + '[targets.development]\nschema = "main"\n' + ), + warehouse_file="warehouse.duckdb", + consumption_table="main._sqlbuild_consumption", + raw_tables_sql=( + "CREATE SCHEMA raw", + "CREATE TABLE raw.orders AS SELECT 1 AS order_id, 1 AS customer_id, 1 AS load_id", + "CREATE TABLE raw.customers AS SELECT 1 AS customer_id, 1 AS load_id", + ), + new_order_sql="INSERT INTO raw.orders VALUES (2, 1, 2)", + execute=duckdb_rows, + expected_runs=EXPECTED_CHANGED_RUNS, + ), + ChangedJobTestCase( + description="SQLite job with changed runs only consumers of new orders", + model_sql="SELECT 1 AS amount", + warehouse_query="", + expected_status="success", + expected_rows=[], + project_toml=( + 'name = "orders"\nadapter = "sqlite"\n' + '[connections.local]\npath = "warehouse.sqlite"\n' + '[targets.development]\nschema = "main"\n' + ), + warehouse_file="warehouse.sqlite", + consumption_table="_sqlbuild_consumption", + raw_tables_sql=( + "CREATE TABLE raw__orders (order_id INTEGER, customer_id INTEGER, load_id INTEGER)", + "INSERT INTO raw__orders VALUES (1, 1, 1)", + "CREATE TABLE raw__customers (customer_id INTEGER, load_id INTEGER)", + "INSERT INTO raw__customers VALUES (1, 1)", + ), + new_order_sql="INSERT INTO raw__orders VALUES (2, 1, 2)", + execute=sqlite_rows, + expected_runs=EXPECTED_CHANGED_RUNS, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_changed_job_when_inputs_change_between_runs_then_only_their_consumers_run( + test_case: ChangedJobTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + test_case = replace(test_case, files=changed_job_files(test_case.project_toml)) + observed: list[tuple[str, frozenset[str], dict[str, int]]] = [] + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + warehouse: Path = worker.project / test_case.warehouse_file + for sql in test_case.raw_tables_sql: + test_case.execute(warehouse, sql) + recorded: set[tuple[Any, ...]] = set() + for description, mutation in [ + ("first run", ()), + ("nothing changed", ()), + ("a new order", (test_case.new_order_sql,)), + ]: + time.sleep(ORDER_SKEW_SECONDS) + for sql in mutation: + test_case.execute(warehouse, sql) + run: dict[str, Any] = request_job_run(worker, "changed_orders") + assert run["status"] == "success", worker.log.read_text() + started: set[tuple[Any, ...]] = set( + test_case.execute( + warehouse, + f"SELECT event_id, node_name FROM {test_case.consumption_table} " + "WHERE phase = 'started'", + ) + ) + observed.append( + ( + description, + frozenset(row[1] for row in started - recorded), + run["node_counts_json"], + ) + ) + recorded |= started + + assert tuple(observed) == test_case.expected_runs diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py index 2437fb74f4..764ad8b30d 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py @@ -20,6 +20,7 @@ PlanOutput, PlanReason, PythonHookEntry, + SeedPlanEntry, SourceEntry, SourceLoadPlanEntry, SourceWriteStrategy, @@ -218,7 +219,7 @@ def test_given_model_materialize_hook_when_build_runs_then_it_prepares_or_fails_ ), ) - def before_node_write(entry: ModelPlanEntry, *, connection: object) -> None: + def before_node_write(entry: ModelPlanEntry | SeedPlanEntry, *, connection: object) -> None: events.append(f"hook:{entry.name}") hook_actions: dict[bool, Callable[[], object]] = { True: lambda: (_ for _ in ()).throw(RuntimeError("hook failed")), diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index 8640a900ea..a9a901c12b 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -41,6 +41,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | | `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | | `--full-refresh` | Drop and rebuild selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model even without this flag | +| `--changed` | Build only the selected models and seeds whose definition or inputs changed since they last succeeded; see [Building only what changed](#building-only-what-changed) | | `--defer-to` | Resolve unselected model references against another target | | `--defer-sources-to` | Read managed source data from another target | | `--fail-fast` | Stop on first failure and skip remaining nodes | @@ -73,6 +74,57 @@ sqb build --no-tests --no-audits This replaces the former `sqb run` command. The full lifecycle (tests + audits) is always the default; skip flags opt out of specific phases when you need speed. +## Building only what changed + +`--changed` skips each selected model or seed that would read exactly what it read the last time +it built successfully: + +```bash +sqb build --changed +``` + +``` +Changed: 2 of 3 selected models and seeds run; 1 unchanged are skipped. +``` + +Add `--verbose` to print why each one runs, such as `input raw_orders changed`. + +A selected model or seed runs when any of these is true: + +| Reason | Example | +|--------|---------| +| It has no successful build on record | A new model | +| Its last build did not finish | The build was killed or failed after the model started writing | +| Its relation is missing, or `--full-refresh` or a cursor override applies | `sqb build --changed --full-refresh` | +| Its definition changed | Edited SQL, config, hooks, contract or a function it calls | +| A source it reads has a different [freshness](/docs/concepts/sources#source-freshness) value | New rows raise `MAX(loaded_at)` | +| An upstream model or seed was rebuilt since, or runs in this build | `stg_orders` ran, so `customer_orders` runs | +| An upstream model or seed needs to run, even when it is not selected | `sqb build customer_orders --changed` runs `customer_orders` on every build until a changed `stg_orders` is built | +| What it reads can't be listed | SQL that reads relations not declared as sources or models, or a Python task or asset upstream | + +A model that runs under `--changed` plans exactly as in a plain build: its +[`replay_on_change`](/docs/concepts/planning/replay-decisions) policy still decides how much +history an incremental model reprocesses. A model that runs only because an upstream was rebuilt +takes its normal incremental step. + +In a scheduled job, set `settings={'changed': True}`. Skipped models count as successful nodes of +the run. + +Every build, with or without `--changed`, records what each model and seed read in +`_sqlbuild_consumption` in its target schema, so the build needs permission to create and insert +into that table. [`sqb janitor`](/docs/cli/janitor) keeps the newest +`direct_state_history_versions` builds of each relation. + +Limitations: + +- A source is compared only through its freshness value. A change that doesn't move it, such as an + update that keeps the old `loaded_at`, isn't seen. A source without freshness configuration counts + as changed on every build. +- SQL that depends on the time, such as `current_date`, or on random values never counts as changed. + Build those models without `--changed`. +- Two builds of one model less than 5 seconds apart can't be ordered, so its downstream models run + on the next `--changed` build. + ## Execution order 1. Managed sources are loaded (unless `--no-load`, which audits their existing data instead) diff --git a/website/src/content/docs/docs/cli/janitor.mdx b/website/src/content/docs/docs/cli/janitor.mdx index 9314df64f4..5c5e080ab6 100644 --- a/website/src/content/docs/docs/cli/janitor.mdx +++ b/website/src/content/docs/docs/cli/janitor.mdx @@ -147,8 +147,9 @@ like a review schema is left alone. ### State history pruning The janitor prunes old rows from `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness`, keeping -the latest `direct_state_history_versions` rows per identity (default `20`). This is separate from -relation retention. +the latest `direct_state_history_versions` rows per identity (default `20`), and from +`_sqlbuild_consumption`, keeping the latest `direct_state_history_versions` builds of each model and +seed. This is separate from relation retention. ## Safety diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 4a521e186d..e987399e32 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -784,7 +784,7 @@ keep_in_use = false | `archive_retention_days` | `14` | How many days an archive is kept before it is deleted. `0` deletes it in the same run. | | `delete_tracked_only` | `true` | Only clean relations that appear in fingerprint tracking | | `exclude_patterns` | `[]` | Glob patterns for relations to skip | -| `direct_state_history_versions` | `20` | How many state-history rows to keep per identity in `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness` | +| `direct_state_history_versions` | `20` | How many state-history rows to keep per identity in `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness`, and how many builds of each model and seed to keep in `_sqlbuild_consumption` | | `in_use_window` | `"7d"` | How far back the janitor looks for reads of each relation it would archive. A fixed duration such as `"7d"` or `"36h"`. See [Relations in use](/docs/cli/janitor#relations-in-use) | | `keep_in_use` | `false` | Keep relations read within `in_use_window` instead of archiving them. `--keep-in-use` turns it on for one run | From 0876c4f7bfeed48472be0a72a60d0a8b9528beef Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 4 Nov 2026 20:47:59 +0000 Subject: [PATCH 4/8] wip: mirror consumption history and release the recorder connection between phases Mirror _sqlbuild_consumption into the control store through the existing warehouse-history sync (new history kind, ingest, route, client and an additive migration). --changed never reads the mirror. The recorder now connects only for its read and append phases instead of holding a warehouse connection for the whole build, so ingestr loads and in-process crash/retry paths can open a file-backed DuckDB warehouse. Janitor e2e expectations and the planner differential masks account for the new always-written table. Validated locally: make check-ci, Fensu, hygiene scan; janitor e2e 43/43; ingestr e2e 6/6; migration crash/retry integration 56/56; planner differential; changed-builds e2e, scheduler --changed with mirror, and warehouse-metadata integration 43/43; orchestration native tests for the mirror on SQLite and PostgreSQL 7/7; TypeScript client check and test. Timings and independent review still pending. --- .../orchestration/consumption_history.json | 70 ++++++++++++++ clients/typescript/src/schema.d.ts | 96 ++++++++++++++++++- clients/typescript/tests/types.ts | 8 ++ .../src/native_commands/_helpers/execution.rs | 9 ++ .../src/api/_helpers/resources/router.rs | 4 + .../_helpers/resources/warehouse_metadata.rs | 24 ++++- .../sqlbuild-orchestration/src/api/models.rs | 1 + .../src/platform/_helpers/work_client.rs | 14 ++- .../src/store/_helpers/migration.rs | 2 + .../src/store/warehouse_consumption.sql | 25 +++++ .../main/ingest_consumption.rs | 94 ++++++++++++++++++ .../src/warehouse_metadata/main/mod.rs | 1 + .../src/warehouse_metadata/models.rs | 41 +++++++- .../tests/api/test_external_sessions.rs | 1 + .../tests/api/test_warehouse_metadata.rs | 5 + .../tests/credentials/test_audit.rs | 1 + .../tests/sensors/test_sources.rs | 1 + .../tests/store/helpers.rs | 1 + .../bindings/_helpers/consumption/recorder.rs | 19 ++-- .../_helpers/native_host/consumption_state.rs | 58 +++++++---- .../native_host/warehouse_metadata.rs | 24 ++++- fensu.toml | 2 +- scripts/planner_differential/constants.py | 3 +- .../cli/commands/main/janitor/test_janitor.py | 11 ++- .../main/janitor/test_janitor_progress.py | 18 ++-- .../main/janitor/test_janitor_targets.py | 4 +- .../main/janitor/test_janitor_usage.py | 12 +-- .../compiled_project/publication/helpers.py | 26 +++++ .../publication/test_changed.py | 43 +++++++++ .../publication/test_warehouse_metadata.py | 33 +++++++ 30 files changed, 590 insertions(+), 61 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/consumption_history.json create mode 100644 crates/sqlbuild-orchestration/src/store/warehouse_consumption.sql create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_consumption.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/consumption_history.json b/benchmarks/duckshop/edge_cases/orchestration/consumption_history.json new file mode 100644 index 0000000000..20e85c5bef --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/consumption_history.json @@ -0,0 +1,70 @@ +{ + "record": { + "state_database": null, + "state_schema": "main", + "event_id": "consumption-000", + "execution_id": "0190a1b2-0000-7000-8000-000000000001", + "phase": "succeeded", + "run_id": "external-build", + "node_type": "model", + "node_name": "customer_orders", + "target_database": null, + "target_schema": "main", + "target_name": "customer_orders_🧾", + "version_hash": "customer-orders-version", + "input_count": 2, + "input_kind": "source", + "input_name": "shop.orders", + "input_version": "load 'orders' 🧾 #7", + "recorded_at": "2026-02-03T04:05:06Z" + }, + "source_rows": 104, + "duplicate_rows": 3, + "later_time": "2026-02-03T04:05:06.001Z", + "retained_result": "consumption-103", + "kind": "consumption", + "source_key": "event_id", + "timestamp_key": "recorded_at", + "id_prefix": "consumption", + "page_lengths": [ + 100, + 7 + ], + "later_index": 106, + "later_record": { + "event_id": "consumption-103", + "phase": "started", + "execution_id": "0190a1b2-0000-7000-8000-000000000002", + "input_kind": null, + "input_name": null, + "input_version": null, + "recorded_at": "2026-02-03T04:05:06.001Z" + }, + "last_record": { + "event_id": "consumption-103", + "phase": "started", + "execution_id": "0190a1b2-0000-7000-8000-000000000002", + "input_kind": null, + "input_name": null, + "input_version": null, + "recorded_at": "2026-02-03T04:05:06.001Z" + }, + "record_overrides": { + "consumption-102": { + "input_kind": "node", + "input_name": "model.stg_customers", + "input_version": "0190a1b2-0000-7000-8000-0000000000aa" + }, + "consumption-103": { + "event_id": "consumption-103", + "phase": "started", + "execution_id": "0190a1b2-0000-7000-8000-000000000002", + "input_kind": null, + "input_name": null, + "input_version": null, + "recorded_at": "2026-02-03T04:05:06.001Z" + } + }, + "source_count": 107, + "model": "MODEL (description 'Orders.', materialized table); SELECT 1 AS order_id" +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index d48b740008..4af98f4730 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -499,6 +499,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/locations/{location_id}/targets/{target}/metadata/consumption": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["report_consumption"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/locations/{location_id}/targets/{target}/metadata/fingerprints": { parameters: { query?: never; @@ -997,6 +1013,14 @@ export interface components { /** Format: uuid */ request_id: string; }; + ConsumptionBatch: { + /** Format: int64 */ + generation: number; + reconciliation?: null | components["schemas"]["WarehouseReconciliationPage"]; + records: components["schemas"]["WarehouseConsumptionRecord"][]; + /** Format: uuid */ + session_id: string; + }; /** @description The current successful snapshot, read with a consistent generation and change set. */ CurrentSnapshot: { changes: components["schemas"]["SnapshotChange"][]; @@ -1929,6 +1953,28 @@ export interface components { state_database?: string | null; state_schema: string; }; + /** @description What one execution consumed, as history; `--changed` reads the warehouse table, never this mirror. */ + WarehouseConsumptionRecord: { + event_id: string; + execution_id: string; + /** Format: int64 */ + input_count: number; + input_kind?: string | null; + input_name?: string | null; + input_version?: string | null; + node_name: string; + node_type: string; + phase: string; + /** Format: date-time */ + recorded_at: string; + run_id: string; + state_database?: string | null; + state_schema: string; + target_database?: string | null; + target_name: string; + target_schema: string; + version_hash: string; + }; /** @description Fingerprints remain history here; latest-row selection belongs to the existing planner reader. */ WarehouseFingerprintRecord: { definition: string; @@ -1969,7 +2015,7 @@ export interface components { * @description Catalog observations require full scans; only immutable history supports a watermark. * @enum {string} */ - WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches" | "catalog" | "statistics"; + WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches" | "consumption" | "catalog" | "statistics"; /** @description Warehouse cleanup facts do not authorize control-plane deletion or worker recovery. */ WarehouseJanitorRecord: { archive_name: string; @@ -3437,6 +3483,54 @@ export interface operations { }; }; }; + report_consumption: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ConsumptionBatch"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WarehouseSyncResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; report_fingerprints: { parameters: { query?: never; diff --git a/clients/typescript/tests/types.ts b/clients/typescript/tests/types.ts index 73c6560b93..a720ad4c93 100644 --- a/clients/typescript/tests/types.ts +++ b/clients/typescript/tests/types.ts @@ -67,6 +67,14 @@ void client.POST("/api/locations/{location_id}/targets/{target}/metadata/microba params: { path: { location_id: "synthetic-location", target: "development" } }, body: { session_id: "synthetic-session", generation: 1, records: [] }, }); +const consumptionHistory: components["schemas"]["WarehouseHistoryKind"] = "consumption"; +const startedInput: components["schemas"]["WarehouseConsumptionRecord"]["input_version"] = null; +void consumptionHistory; +void startedInput; +void client.POST("/api/locations/{location_id}/targets/{target}/metadata/consumption", { + params: { path: { location_id: "synthetic-location", target: "development" } }, + body: { session_id: "synthetic-session", generation: 1, records: [] }, +}); const catalogHistory: components["schemas"]["WarehouseHistoryKind"] = "catalog"; const catalogColumn: components["schemas"]["WarehouseCatalogColumn"] = { name: "order_id", data_type: "INTEGER", diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 043b0b9682..599fce8e15 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -521,6 +521,15 @@ pub(crate) async fn deliver_metadata_page( ) .await } + WarehouseHistoryKind::Consumption => { + client + .sync_consumption( + target, + &serde_json::from_value(body.clone()) + .map_err(|_| ClientError::Configuration("invalid consumption batch"))?, + ) + .await + } } } diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs index 1143bfe060..d5d7f24755 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs @@ -59,6 +59,10 @@ pub(crate) fn router(store: &ControlStore, feed: ChangeFeed) -> Router, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(batch): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json( + ingest_consumption(&store, (location_id, &target), &batch).await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index ba9d2139b1..e3416f511c 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -68,6 +68,7 @@ use crate::api::_helpers::resources::{ warehouse_metadata::report_migrations, warehouse_metadata::report_old_name_views, warehouse_metadata::report_microbatches, + warehouse_metadata::report_consumption, warehouse_metadata::report_catalog, warehouse_metadata::report_statistics, warehouse_metadata::statistics_due, diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs index f530db781e..dde7ba2d2d 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -10,9 +10,9 @@ use crate::platform::models::{ LocationWorkResult, }; use crate::warehouse_metadata::models::{ - AuditResultsBatch, CatalogBatch, FingerprintBatch, JanitorEventsBatch, MicrobatchesBatch, - MigrationsBatch, OldNameViewsBatch, SourceFreshnessBatch, StatisticsBatch, StatisticsDueRead, - StatisticsRead, WarehouseSyncResponse, WarehouseWatermarkRead, + AuditResultsBatch, CatalogBatch, ConsumptionBatch, FingerprintBatch, JanitorEventsBatch, + MicrobatchesBatch, MigrationsBatch, OldNameViewsBatch, SourceFreshnessBatch, StatisticsBatch, + StatisticsDueRead, StatisticsRead, WarehouseSyncResponse, WarehouseWatermarkRead, }; impl LocationWorkClient { @@ -109,6 +109,14 @@ impl LocationWorkClient { self.sync_metadata((target, "microbatches"), batch).await } + pub async fn sync_consumption( + &self, + target: &str, + batch: &ConsumptionBatch, + ) -> Result { + self.sync_metadata((target, "consumption"), batch).await + } + /// Metadata shares the location's scoped pool, not the scheduler's execution authority. pub async fn sync_source_freshness( &self, diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 0aae15d12e..b7169a441e 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -44,6 +44,7 @@ pub(crate) const TRIGGER_PROGRESS: &str = include_str!("../trigger_progress.sql" pub(crate) const CLI_LOGIN_FLOWS: &str = include_str!("../cli_login_flows.sql"); pub(crate) const RETENTION_INDEXES: &str = include_str!("../retention_indexes.sql"); pub(crate) const CUSTOM_ROLES: &str = include_str!("../custom_roles.sql"); +pub(crate) const WAREHOUSE_CONSUMPTION: &str = include_str!("../warehouse_consumption.sql"); pub(crate) const ADDITIONS: &[(&str, &str)] = &[ ("location_instances", LOCATION_INSTANCES), @@ -80,6 +81,7 @@ pub(crate) const ADDITIONS: &[(&str, &str)] = &[ ("cli_login_flows", CLI_LOGIN_FLOWS), ("retention_indexes", RETENTION_INDEXES), ("custom_roles", CUSTOM_ROLES), + ("warehouse_consumption", WAREHOUSE_CONSUMPTION), ]; pub(crate) fn migration_digests(store: &ControlStore) -> Vec { diff --git a/crates/sqlbuild-orchestration/src/store/warehouse_consumption.sql b/crates/sqlbuild-orchestration/src/store/warehouse_consumption.sql new file mode 100644 index 0000000000..08e4db7f24 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/warehouse_consumption.sql @@ -0,0 +1,25 @@ +CREATE TABLE IF NOT EXISTS sqb_warehouse_consumption ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at {time} NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + execution_id TEXT NOT NULL, + phase TEXT NOT NULL, + run_id TEXT NOT NULL, + node_name TEXT NOT NULL, + target_database TEXT, + target_schema TEXT NOT NULL, + target_name TEXT NOT NULL, + version_hash TEXT NOT NULL, + input_count BIGINT NOT NULL, + input_kind TEXT, + input_name TEXT, + input_version TEXT, + record_json {json} NOT NULL, + received_at {time} NOT NULL +); +CREATE INDEX IF NOT EXISTS sqb_consumption_reconciliation ON sqb_warehouse_consumption (location_id,target,state_database,state_schema,observed_at,event_id); +CREATE INDEX IF NOT EXISTS sqb_consumption_relation ON sqb_warehouse_consumption (location_id,target_schema,target_name,observed_at); diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_consumption.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_consumption.rs new file mode 100644 index 0000000000..890057a4a2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_consumption.rs @@ -0,0 +1,94 @@ +//! Mirror consumed-input history for inspection; run decisions never read this mirror. + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::events::ingest_events; +use crate::warehouse_metadata::_helpers::ingestion::validate_fields; +use crate::warehouse_metadata::models::{ + ConsumptionBatch, StateEventImport, StateEventsImport, WarehouseHistoryKind, + WarehouseStateLocation, WarehouseSyncResponse, +}; + +pub async fn ingest_consumption( + store: &ControlStore, + scope: (Uuid, &str), + batch: &ConsumptionBatch, +) -> Result { + validate_fields( + ("consumption", batch.generation, batch.records.len()), + false, + false, + )?; + ingest_events( + store, + scope, + StateEventsImport { + kind: WarehouseHistoryKind::Consumption, + session_id: batch.session_id, + generation: batch.generation, + invalid_identity: batch + .records + .iter() + .flat_map(|record| { + [ + &record.event_id, + &record.state_schema, + &record.execution_id, + &record.phase, + &record.node_name, + &record.target_schema, + &record.target_name, + ] + }) + .any(|value| value.trim().is_empty()), + records: batch + .records + .iter() + .map(|record| StateEventImport { + source_id: record.event_id.clone(), + location: WarehouseStateLocation { + database: record.state_database.clone(), + schema: record.state_schema.clone(), + }, + observed_at: record.recorded_at, + columns: vec![ + ("execution_id", Parameter::Text(record.execution_id.clone())), + ("phase", Parameter::Text(record.phase.clone())), + ("run_id", Parameter::Text(record.run_id.clone())), + ("node_name", Parameter::Text(record.node_name.clone())), + ( + "target_database", + Parameter::OptionalText(record.target_database.clone()), + ), + ( + "target_schema", + Parameter::Text(record.target_schema.clone()), + ), + ("target_name", Parameter::Text(record.target_name.clone())), + ("version_hash", Parameter::Text(record.version_hash.clone())), + ("input_count", Parameter::Integer(record.input_count)), + ( + "input_kind", + Parameter::OptionalText(record.input_kind.clone()), + ), + ( + "input_name", + Parameter::OptionalText(record.input_name.clone()), + ), + ( + "input_version", + Parameter::OptionalText(record.input_version.clone()), + ), + ], + record_json: json!(record), + }) + .collect(), + reconciliation: batch.reconciliation.as_ref(), + payload: json!(batch), + }, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index 8fac735025..f5f60dcd9d 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod bootstrap_source_observations; pub mod ingest_audit_results; pub mod ingest_catalog; +pub mod ingest_consumption; pub mod ingest_fingerprints; pub mod ingest_janitor_events; pub mod ingest_microbatches; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 8ea8b3988e..5565b4907a 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -271,6 +271,39 @@ pub struct MicrobatchesBatch { pub reconciliation: Option, } +/// What one execution consumed, as history; `--changed` reads the warehouse table, never this mirror. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseConsumptionRecord { + pub state_database: Option, + pub state_schema: String, + pub event_id: String, + pub execution_id: String, + pub phase: String, + pub run_id: String, + pub node_type: String, + pub node_name: String, + pub target_database: Option, + pub target_schema: String, + pub target_name: String, + pub version_hash: String, + pub input_count: i64, + pub input_kind: Option, + pub input_name: Option, + pub input_version: Option, + pub recorded_at: DateTime, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct ConsumptionBatch { + pub session_id: Uuid, + pub generation: i64, + pub records: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reconciliation: Option, +} + /// Physical namespace of SQLBuild's source-freshness state table, never a source query. #[derive( Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, utoipa::ToSchema, @@ -292,12 +325,13 @@ pub enum WarehouseHistoryKind { Migrations, OldNameViews, Microbatches, + Consumption, Catalog, Statistics, } impl WarehouseHistoryKind { - pub const ALL: [Self; 9] = [ + pub const ALL: [Self; 10] = [ Self::Fingerprints, Self::SourceFreshness, Self::AuditResults, @@ -305,6 +339,7 @@ impl WarehouseHistoryKind { Self::Migrations, Self::OldNameViews, Self::Microbatches, + Self::Consumption, Self::Catalog, Self::Statistics, ]; @@ -319,6 +354,7 @@ impl WarehouseHistoryKind { Self::Migrations => "migrations", Self::OldNameViews => "old_name_views", Self::Microbatches => "microbatches", + Self::Consumption => "consumption", Self::Catalog => "catalog", Self::Statistics => "statistics", } @@ -334,6 +370,7 @@ impl WarehouseHistoryKind { Self::Migrations => "sqb_warehouse_migrations", Self::OldNameViews => "sqb_warehouse_old_name_views", Self::Microbatches => "sqb_warehouse_microbatches", + Self::Consumption => "sqb_warehouse_consumption", Self::Catalog => "sqb_warehouse_catalog", Self::Statistics => "sqb_warehouse_statistics", } @@ -451,6 +488,7 @@ pub enum WarehouseHistoryRead { Migrations(StateEventsRead), OldNameViews(StateEventsRead), Microbatches(StateEventsRead), + Consumption(StateEventsRead), Catalog(CatalogRead), Statistics(StatisticsRead), } @@ -465,6 +503,7 @@ pub enum WarehouseHistoryRecord { Migration(Box), OldNameView(Box), Microbatch(Box), + Consumption(Box), Catalog(WarehouseCatalogRecord), Statistics(WarehouseStatisticsRecord), } diff --git a/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs index 1ad9760fca..e67beec5b8 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs @@ -94,6 +94,7 @@ fn given_federated_people_when_administrators_and_sessions_change_then_live_auth .expect("populated predecessor"); for sql in [ "DROP TABLE sqb_external_sessions", + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_consumption'", "DELETE FROM sqb_schema_migrations WHERE name='custom_roles'", "DELETE FROM sqb_schema_migrations WHERE name='retention_indexes'", "DELETE FROM sqb_schema_migrations WHERE name='external_sessions'", diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index 8ab85c3b95..453c5e41d7 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -348,6 +348,11 @@ fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicit "microbatch_history", "event_id", ), + ( + WarehouseHistoryKind::Consumption, + "consumption_history", + "event_id", + ), ] { crate::api::tests::warehouse_events_helpers::event_history( store, diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs index 5cdb5276a7..38bfa64c85 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs @@ -139,6 +139,7 @@ fn given_security_audit_pages_when_clock_and_authority_change_then_commit_order_ .expect("predecessor"); for sql in [ "DROP TABLE sqb_warehouse_catalog", + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_consumption'", "DELETE FROM sqb_schema_migrations WHERE name='custom_roles'", "DELETE FROM sqb_schema_migrations WHERE name='retention_indexes'", "DELETE FROM sqb_schema_migrations WHERE name='warehouse_catalog'", diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs index 87a83c4487..89666c4294 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs @@ -175,6 +175,7 @@ fn given_imported_source_versions_when_consuming_then_late_facts_queue_once_and_ .expect("pre-sensor predecessor"); for sql in [ "DROP TABLE sqb_warehouse_catalog", + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_consumption'", "DELETE FROM sqb_schema_migrations WHERE name='custom_roles'", "DELETE FROM sqb_schema_migrations WHERE name='retention_indexes'", "DELETE FROM sqb_schema_migrations WHERE name='warehouse_catalog'", diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index baa49874ed..e1d893e6d2 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -92,6 +92,7 @@ pub(crate) async fn reconstruct_fingerprint_predecessor(store: &ControlStore) { .await .expect("fingerprint predecessor"); for sql in [ + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_consumption'", "DELETE FROM sqb_schema_migrations WHERE name='custom_roles'", "DELETE FROM sqb_schema_migrations WHERE name='retention_indexes'", "DROP TABLE sqb_cli_login_flows", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs index e58f8cd66b..89389b1be8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs @@ -123,7 +123,9 @@ impl ConsumptionRecorder { /// The recording after appending every pending row; a failure is reported, and its executions run again. fn flushed(&self, py: Python<'_>, mut recording: Recording) -> Recording { - for (location, rows) in std::mem::take(&mut recording.pending) { + let pending = std::mem::take(&mut recording.pending); + let wrote: bool = !pending.is_empty(); + for (location, rows) in pending { let created: bool = !recording.tables.insert(location.clone()); let written: PyResult<()> = (if created { Ok(()) @@ -138,12 +140,17 @@ impl ConsumptionRecorder { )); } } + if wrote && let Err(error) = self.connection.close(py) { + recording.warnings.push(format!( + "could not close the consumed-input connection: {error}" + )); + } recording.pending_executions = 0; recording.last_flush = Instant::now(); recording } - /// Queue the seeds whose error audits passed, flush everything and close the connection. + /// Queue the seeds whose error audits passed and flush everything. pub(crate) fn finish( &self, py: Python<'_>, @@ -165,10 +172,7 @@ impl ConsumptionRecorder { } } *recording = self.flushed(py, std::mem::take(&mut *recording)); - let closed = self.connection.close(py); - let warnings: Vec = std::mem::take(&mut recording.warnings); - closed?; - Ok(warnings) + Ok(std::mem::take(&mut recording.warnings)) } } @@ -268,7 +272,7 @@ pub(crate) fn prepare<'py>( ) -> PyResult> { let py: Python<'py> = plan.py(); let facts: BuildFacts = build_facts((plan, project), (changed, full_refresh))?; - let connection = ConsumptionConnection::open(adapter, connection_config)?; + let connection = ConsumptionConnection::new(adapter, connection_config); let mut warnings: Vec = Vec::new(); let mut rows: Vec = Vec::new(); for location in read_locations(&facts, changed) { @@ -341,6 +345,7 @@ pub(crate) fn prepare<'py>( } } } + connection.close(py)?; let recorder = ConsumptionRecorder { connection, run_id, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs index f6a90ddd00..3579c5959f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs @@ -1,7 +1,9 @@ //! The consumption table through the adapter, and the connection and callbacks a recording build holds. use chrono::{DateTime, NaiveDateTime, Utc}; -use pyo3::exceptions::PyValueError; +use std::sync::Mutex; + +use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyFrozenSet, PyInt, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -391,27 +393,38 @@ fn execution_chunks(rows: &[ConsumptionAppend]) -> Vec<&[ConsumptionAppend]> { chunks } -/// The adapter and the connection one recording build appends through. +/// One build's consumption I/O, connected only per phase so loaders can lock file warehouses. pub(crate) struct ConsumptionConnection { adapter: Py, - connection: Py, + connection_config: Py, + connection: Mutex>>, } impl ConsumptionConnection { - pub(crate) fn open( - adapter: &Bound<'_, PyAny>, - connection_config: &Bound<'_, PyAny>, - ) -> PyResult { - let connection: Bound<'_, PyAny> = - native_field(adapter, "connect")?.call1((connection_config,))?; - Ok(Self { + pub(crate) fn new(adapter: &Bound<'_, PyAny>, connection_config: &Bound<'_, PyAny>) -> Self { + Self { adapter: adapter.clone().unbind(), - connection: connection.unbind(), - }) + connection_config: connection_config.clone().unbind(), + connection: Mutex::new(None), + } + } + + fn connected<'py>(&self, py: Python<'py>) -> PyResult> { + let mut open = self + .connection + .lock() + .map_err(|_| PyRuntimeError::new_err("consumption connection lock poisoned"))?; + if let Some(connection) = open.as_ref() { + return Ok(connection.bind(py).clone()); + } + let connection: Bound<'py, PyAny> = native_field(self.adapter.bind(py), "connect")? + .call1((self.connection_config.bind(py),))?; + *open = Some(connection.clone().unbind()); + Ok(connection) } pub(crate) fn ensure_table(&self, py: Python<'_>, location: &Location) -> PyResult<()> { - ensure_table(self.adapter.bind(py), self.connection.bind(py), location) + ensure_table(self.adapter.bind(py), &self.connected(py)?, location) } pub(crate) fn read_recent( @@ -419,7 +432,7 @@ impl ConsumptionConnection { py: Python<'_>, location: &Location, ) -> PyResult> { - read_recent(self.adapter.bind(py), self.connection.bind(py), location) + read_recent(self.adapter.bind(py), &self.connected(py)?, location) } pub(crate) fn append( @@ -428,16 +441,19 @@ impl ConsumptionConnection { location: &Location, rows: &[ConsumptionAppend], ) -> PyResult<()> { - append( - self.adapter.bind(py), - self.connection.bind(py), - location, - rows, - ) + append(self.adapter.bind(py), &self.connected(py)?, location, rows) } + /// Close the connection if one is open; the next operation connects again. pub(crate) fn close(&self, py: Python<'_>) -> PyResult<()> { - native_field(self.adapter.bind(py), "close")?.call1((self.connection.bind(py),))?; + let open: Option> = self + .connection + .lock() + .map_err(|_| PyRuntimeError::new_err("consumption connection lock poisoned"))? + .take(); + if let Some(connection) = open { + native_field(self.adapter.bind(py), "close")?.call1((connection.bind(py),))?; + } Ok(()) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index a5c265e81e..240edbdff5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -8,12 +8,15 @@ use sqlbuild_analysis::fingerprint_storage::constants::{COLUMN_TIMESTAMP, FINGER use sqlbuild_config::command_connection::constants::{ DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, LOCAL_DATABASE_PATH_KEYS, }; -use sqlbuild_core::constants::{FINGERPRINT_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME}; +use sqlbuild_core::constants::{ + CONSUMPTION_TABLE_NAME, FINGERPRINT_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, +}; use sqlbuild_core::models::SqlValue; use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use sqlbuild_orchestration::warehouse_metadata::models::{ WarehouseHistoryPage, WarehouseHistoryRead, WarehouseHistoryRecord, WarehouseStateLocation, }; +use sqlbuild_state::consumption::constants::{CONSUMPTION_COLUMN_TYPES, RECORDED_AT_COLUMN}; use sqlbuild_state::events::constants::{MIGRATION_COLUMNS, OLD_NAME_VIEW_COLUMNS}; use sqlbuild_state::freshness::constants::{ OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES, SQLITE_ADAPTER, @@ -50,7 +53,8 @@ pub(crate) fn read_history( | WarehouseHistoryRead::JanitorEvents(read) | WarehouseHistoryRead::Migrations(read) | WarehouseHistoryRead::OldNameViews(read) - | WarehouseHistoryRead::Microbatches(read) => &read.location, + | WarehouseHistoryRead::Microbatches(read) + | WarehouseHistoryRead::Consumption(read) => &read.location, }; if location.schema.trim().is_empty() { return Err(PyValueError::new_err( @@ -110,6 +114,7 @@ fn read_page( request: &WarehouseHistoryRead, ) -> PyResult { let freshness_columns = SOURCE_FRESHNESS_COLUMN_TYPES.map(|(name, _)| name); + let consumption_columns = CONSUMPTION_COLUMN_TYPES.map(|(name, _)| name); let event_columns: Vec = match request { WarehouseHistoryRead::AuditResults(_) => py .import("sqlbuild.executor.audit_results.constants")? @@ -196,6 +201,15 @@ fn read_page( event_columns.as_slice(), "created_at", ), + WarehouseHistoryRead::Consumption(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + read.limit, + CONSUMPTION_TABLE_NAME, + consumption_columns.as_slice(), + RECORDED_AT_COLUMN, + ), }; let limit = limit.unwrap_or(MAX_HISTORY_RECORDS); if limit == 0 || limit > MAX_HISTORY_RECORDS { @@ -269,6 +283,7 @@ fn read_page( WarehouseHistoryRead::Migrations(_) | WarehouseHistoryRead::OldNameViews(_) | WarehouseHistoryRead::Microbatches(_) => Some(&["event_id", "created_at"]), + WarehouseHistoryRead::Consumption(_) => Some(&["event_id", RECORDED_AT_COLUMN]), _ => None, }; let mut cursor_indexes: Vec = columns @@ -389,6 +404,7 @@ fn read_page( WarehouseHistoryRecord::Migration(record) => record.created_at, WarehouseHistoryRecord::OldNameView(record) => record.created_at, WarehouseHistoryRecord::Microbatch(record) => record.created_at, + WarehouseHistoryRecord::Consumption(record) => record.recorded_at, }; if since.is_none_or(|since| observed >= since) { records.push(record); @@ -427,7 +443,9 @@ fn decode_record( qualified, )?, ), - WarehouseHistoryRead::AuditResults(read) | WarehouseHistoryRead::Migrations(read) => { + WarehouseHistoryRead::AuditResults(read) + | WarehouseHistoryRead::Migrations(read) + | WarehouseHistoryRead::Consumption(read) => { return decode_event_record(&read.location, row, (columns, &[timestamp])); } WarehouseHistoryRead::JanitorEvents(read) => { diff --git a/fensu.toml b/fensu.toml index fcccc9bfbe..d70b715043 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1673,7 +1673,7 @@ paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerpr reason = "Typed observation field projections intentionally mirror provenance checks while preserving different required identity fields and timestamp names. Bounds, rejection rules and reconciliation completion share the ingestion owner." [[dupes.allowlist]] -paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_janitor_events.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_microbatches.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_migrations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_old_name_views.rs"] +paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_consumption.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_janitor_events.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_microbatches.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_migrations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_old_name_views.rs"] reason = "Typed event projections retain distinct required identities and indexed payload columns. Event persistence, fencing, atomic retries and reconciliation share the events ingestion owner; no per-table lifecycle is copied." [[targets.python.rule_exceptions]] diff --git a/scripts/planner_differential/constants.py b/scripts/planner_differential/constants.py index 58d1c11d62..498c0b447b 100644 --- a/scripts/planner_differential/constants.py +++ b/scripts/planner_differential/constants.py @@ -121,7 +121,7 @@ RUN_RELATIVE_PAST_DAYS: int = 7 RUN_RELATIVE_FUTURE_DAYS: int = 400 MAPPING_MARKER: str = "__mapping__" -VOLATILE_KEY_SUFFIXES: tuple[str, ...] = ("event_id",) +VOLATILE_KEY_SUFFIXES: tuple[str, ...] = ("event_id", "execution_id") VOLATILE_KEYS: frozenset[str] = frozenset( { "invocation_time", @@ -140,6 +140,7 @@ "created_at", "updated_at", "recorded_at", + "input_version", } ) BASE64_COLUMN_SUFFIX: str = "_b64" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index f4896cd156..a7c21d7385 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -77,12 +77,13 @@ def test_given_default_config_when_running_janitor_then_it_reports_disabled( expected_stdout_fragments=( "relations to archive 1", "archives to delete 0", - "objects skipped 4", + "objects skipped 5", "main.janitor_tracked_extra -> main._sqb_archive__", "main.janitor_untracked_extra relation is not tracked by SQLBuild", "main.partition_state relation matches exclude pattern 'partition_*'", "main._sqlbuild_fingerprints relation matches exclude pattern", "main._sqlbuild_microbatches relation matches exclude pattern", + "main._sqlbuild_consumption relation matches exclude pattern", "Archived 1 relation.", ), expected_existing_tables=( @@ -92,6 +93,7 @@ def test_given_default_config_when_running_janitor_then_it_reports_disabled( "_sqlbuild_fingerprints", "_sqlbuild_microbatches", "_sqlbuild_janitor_events", + "_sqlbuild_consumption", ), expected_missing_tables=("janitor_tracked_extra",), expected_archived_original_names=("janitor_tracked_extra",), @@ -315,7 +317,7 @@ def test_given_source_in_managed_schema_when_running_direct_janitor_then_blocks_ expected_exit_code=0, expected_stdout_fragments=( "relations to archive 2", - "objects skipped 2", + "objects skipped 3", "main.__sqb_a13f09c2e7b8__model__daily_revenue -> main._sqb_archive__", "main.__sqb_a13f09c2e7b8__source__raw_orders -> main._sqb_archive__", "main.__sqb_a13f09c2e7b__model__daily_revenue relation is not tracked by SQLBuild", @@ -391,11 +393,12 @@ def test_given_scenario_artifacts_when_running_direct_janitor_then_it_archives_t plan_command=("--no-color", "plan"), expected_exit_code=0, expected_stdout_fragments=( - "direct state pruned 2", + "direct state pruned 3", "Eligible direct state pruning", "main._sqlbuild_fingerprints keep latest 2", "main._sqlbuild_source_freshness keep latest 2", - "Pruned 2 direct state tables.", + "main._sqlbuild_consumption keep latest 2", + "Pruned 3 direct state tables.", ), expected_fingerprint_count_before=5, expected_fingerprint_count_after=3, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py index 22d535943f..f7f1c0d7f6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py @@ -62,10 +62,11 @@ "Deleting 1 expired archive...", " 1/1 deleted main._sqb_archive__20200101t000000z__old_products OK", "Deleted 1 expired archive. (", - "Pruning 1 direct state table...", - " 1/1 pruned main._sqlbuild_fingerprints OK", - "Pruned 1 direct state table. (", - "\u2713 Janitor completed archived 12, deleted 1, pruned 1 (", + "Pruning 2 direct state tables...", + " 1/2 pruned main._sqlbuild_fingerprints OK", + " 2/2 pruned main._sqlbuild_consumption OK", + "Pruned 2 direct state tables. (", + "\u2713 Janitor completed archived 12, deleted 1, pruned 2 (", ), expected_archive_events=12, unexpected_fragments=( @@ -80,8 +81,9 @@ expected_exit_code=0, expected_ordered_lines=( " main.old_orders_011 -> main._sqb_archive__", - "Skipped objects\n main._sqlbuild_fingerprints relation matches exclude pattern", - "\u2713 Janitor completed archived 12, deleted 1, pruned 1 (", + "Skipped objects\n main._sqlbuild_consumption relation matches exclude pattern", + " main._sqlbuild_fingerprints relation matches exclude pattern", + "\u2713 Janitor completed archived 12, deleted 1, pruned 2 (", ), expected_archive_events=12, unexpected_fragments=("... and", "Skipped objects ("), @@ -146,12 +148,12 @@ def test_given_stale_relations_when_running_janitor_then_progress_streams_in_ord stale_tables=3, expected_status="completed", expected_archived=3, - expected_item_statuses=("ok",) * 5, + expected_item_statuses=("ok",) * 6, expected_stderr_fragments=( "Janitor preview default", "Archiving 3 relations...", " 1/3 archived main.old_orders_000 -> main._sqb_archive__", - "\u2713 Janitor completed archived 3, deleted 1, pruned 1", + "\u2713 Janitor completed archived 3, deleted 1, pruned 2", ), ), ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py index 75a481aa83..ba85d4120e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py @@ -162,7 +162,7 @@ def test_given_prod_old_name_view_when_previewing_early_drop_as_prod_then_view_i expected_stdout_fragments=( "Janitor preview prod", "prod.customers -> prod._sqb_archive__", - "\u2713 Janitor completed archived 1, deleted 1, pruned 2", + "\u2713 Janitor completed archived 1, deleted 1, pruned 3", ), expected_deleted_names=(EXPIRED_ARCHIVE_NAME, "customers"), expected_archived_original_names=("customers",), @@ -176,7 +176,7 @@ def test_given_prod_old_name_view_when_previewing_early_drop_as_prod_then_view_i expected_stdout_fragments=( "Janitor preview dev", "dev.customers -> dev._sqb_archive__", - "\u2713 Janitor completed archived 1, deleted 1, pruned 2", + "\u2713 Janitor completed archived 1, deleted 1, pruned 3", ), expected_deleted_names=(EXPIRED_ARCHIVE_NAME, "customers"), expected_archived_original_names=("customers",), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py index a2935276c8..2d8d724049 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py @@ -50,7 +50,7 @@ " relations to archive 3\n", "\n usage not checked: unsupported adapter\n", "Relations to archive", - "\u2713 Janitor completed archived 3, deleted 1, pruned 1", + "\u2713 Janitor completed archived 3, deleted 1, pruned 2", ), expected_archived=3, ), @@ -61,7 +61,7 @@ " usage not checked: unsupported adapter\n" " nothing kept in use: no usage could be checked, so relations in use are " "archived too\n", - "\u2713 Janitor completed archived 3, deleted 1, pruned 1", + "\u2713 Janitor completed archived 3, deleted 1, pruned 2", ), expected_archived=3, ), @@ -151,7 +151,7 @@ def test_given_duckdb_when_running_janitor_json_then_usage_check_is_not_checked( "(40 queries in 7 days); frozen since 2026-09-01, still in use\n", " main.old_orders_view queried 2026-10-21 by svc_reporting " "(14 queries in 7 days)\n", - "\u2713 Janitor completed archived 4, deleted 1, pruned 1", + "\u2713 Janitor completed archived 4, deleted 1, pruned 2", ), unexpected_fragments=( "Kept (in use)", @@ -174,7 +174,7 @@ def test_given_duckdb_when_running_janitor_json_then_usage_check_is_not_checked( "Checked relation usage: 2 of 4 in use. (", " relations to archive 2\n kept (in use) 2\n", "\nKept (in use)\n main.old_orders_000 queried 2026-10-21", - "\u2713 Janitor completed archived 2, deleted 1, pruned 1", + "\u2713 Janitor completed archived 2, deleted 1, pruned 2", ), unexpected_fragments=( "\nIn use\n", @@ -210,7 +210,7 @@ def test_given_duckdb_when_running_janitor_json_then_usage_check_is_not_checked( " kept (in use) 1\n", " main.old_orders_001 queried 2026-10-21 by alice (1 query in 36 hours); " "frozen since 2026-09-01, still in use\n", - "\u2713 Janitor completed archived 3, deleted 1, pruned 1", + "\u2713 Janitor completed archived 3, deleted 1, pruned 2", ), unexpected_fragments=("main.old_orders_001 ->",), expected_remaining=("old_orders_001",), @@ -332,7 +332,7 @@ def test_given_usage_source_when_janitor_prints_json_then_candidates_carry_in_us reads={"old_orders_000": (("alice",), 3)}, unavailable=None, expected_fragments=( - "Janitor will archive 3 and delete 2 items from default.\r\n" + "Janitor will archive 3 and delete 3 items from default.\r\n" "3 relations will be archived; 1 of them is in use.\r\n", "Janitor cancelled.", ), diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 8aaeb52516..6a3de302be 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -217,6 +217,32 @@ def append_microbatch_history(adapter: Any, connection: Any, fixture: dict[str, assert repeated.inserted == 0 +def append_consumption_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: + """Real consumption DDL with duplicated physical appends, as an ambiguous retry leaves.""" + _ = adapter.execute( + connection=connection, + sql=adapter.render_create_consumption_table_sql(database=None, schema="main"), + ) + columns: tuple[str, ...] = tuple(fixture["record"])[2:] + rows: list[dict[str, Any]] = [ + fixture["record"] + | {"event_id": f"{fixture['id_prefix']}-{index:03}"} + | fixture["record_overrides"].get(f"{fixture['id_prefix']}-{index:03}", {}) + for index in range(fixture["source_rows"]) + ] + [fixture["record"]] * fixture["duplicate_rows"] + literals: dict[type, Callable[[Any], str]] = { + type(None): lambda _value: "NULL", + int: str, + str: lambda value: "'" + value.removesuffix("Z").replace("'", "''") + "'", + } + for row in rows: + _ = adapter.execute( + connection=connection, + sql=f"INSERT INTO _sqlbuild_consumption ({', '.join(columns)}) VALUES " + f"({', '.join(literals[type(row[column])](row[column]) for column in columns)})", + ) + + def append_migration_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: """Real model-move facts, including idempotent duplicate attempts.""" data: dict[str, Any] = migration_history_data(fixture, ("origin", "destination")) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py index bdae8b9e69..92f74bf4fe 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py @@ -1,6 +1,10 @@ """Scheduled jobs with `changed` run only models whose consumed inputs changed, on DuckDB and SQLite.""" +import os +import sqlite3 +import sys import time +from contextlib import closing from dataclasses import replace from pathlib import Path from typing import Any @@ -13,6 +17,7 @@ from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( changed_job_files, duckdb_rows, + location_process, request_job_run, scheduled_worker, sqlite_rows, @@ -112,5 +117,43 @@ def test_given_a_changed_job_when_inputs_change_between_runs_then_only_their_con ) ) recorded |= started + warehouse_events: set[str] = { + row[0] + for row in test_case.execute( + warehouse, f"SELECT event_id FROM {test_case.consumption_table}" + ) + } + log: Path = tmp_path / "location.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(worker.project), + "location", + "serve", + "--server", + worker.api.url, + "--location-id", + worker.api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": worker.api.bearer}, + ) as location: + deadline: float = time.monotonic() + 45 + mirrored: set[str] = set() + while mirrored != warehouse_events: + assert location.poll() is None, log.read_text() + assert time.monotonic() < deadline, (warehouse_events - mirrored, log.read_text()) + with closing(sqlite3.connect(worker.control_store)) as control: + mirrored = { + row[0] + for row in control.execute( + "SELECT source_event_id FROM sqb_warehouse_consumption" + ) + } + time.sleep(0.2) assert tuple(observed) == test_case.expected_runs diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index 8e5ce46011..d64538b569 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -44,6 +44,7 @@ ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( append_audit_history, + append_consumption_history, append_contended_fingerprints, append_janitor_history, append_legacy_history, @@ -103,6 +104,22 @@ ("sqlite", "path", "janitor_event_history", append_janitor_history, "cleanup-run", ()), # SQLite has no supported concurrent-microbatch coordination writer. ("duckdb", "database", "microbatch_history", append_microbatch_history, "", ()), + ( + "duckdb", + "database", + "consumption_history", + append_consumption_history, + "external-build", + (), + ), + ( + "sqlite", + "path", + "consumption_history", + append_consumption_history, + "external-build", + (), + ), ( "duckdb", "database", @@ -240,6 +257,22 @@ def test_given_external_state_events_when_reading_pages_then_identities_text_and ), ("sqlite", "path", "janitor_event_history", append_janitor_history, "cleanup-run", ()), ("duckdb", "database", "microbatch_history", append_microbatch_history, "", ()), + ( + "duckdb", + "database", + "consumption_history", + append_consumption_history, + "external-build", + (), + ), + ( + "sqlite", + "path", + "consumption_history", + append_consumption_history, + "external-build", + (), + ), ( "duckdb", "database", From 835099fe0377b15c75663290975b330d850ea1b3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 4 Nov 2026 23:32:04 +0000 Subject: [PATCH 5/8] fix: let --changed skip models that call functions The undeclared-read guard parsed model SQL with function reference calls still in it. The parser cannot read __udf("name")(...), so every model calling a SQL or table function, and everything downstream, ran on each --changed build. Function calls now parse as plain calls; what a function reads is already a declared input. Also drops the two slow served-location consumption cases from the warehouse metadata integration test, which pushed the CI job past its timeout. The mirror stays covered by the native orchestration test and the scheduler --changed test. WIP: validated locally with make check-ci, the hygiene scan, the changed-builds e2e, warehouse metadata and scheduler --changed tests, and a 3,000-model --changed timing run. --- .../sql/table_fn__customer_order_keys.sql | 9 +++++ .../functions/sql/udf__order_key.sql | 7 ++++ .../models/first_customer_order_keys.sql | 3 ++ .../changed_builds_udf/models/stg_orders.sql | 3 ++ .../_helpers/consumption/relation_reads.rs | 10 ++++++ .../cli/commands/main/build/helpers.py | 10 ++++++ .../main/build/test_changed_builds.py | 35 +++++++++++++++++++ .../publication/test_warehouse_metadata.py | 16 --------- 8 files changed, 77 insertions(+), 16 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/table_fn__customer_order_keys.sql create mode 100644 benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/udf__order_key.sql create mode 100644 benchmarks/duckshop/edge_cases/changed_builds_udf/models/first_customer_order_keys.sql create mode 100644 benchmarks/duckshop/edge_cases/changed_builds_udf/models/stg_orders.sql diff --git a/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/table_fn__customer_order_keys.sql b/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/table_fn__customer_order_keys.sql new file mode 100644 index 0000000000..c42ac12221 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/table_fn__customer_order_keys.sql @@ -0,0 +1,9 @@ +FUNCTION ( + description "Order keys of one customer, read from staged orders", + arguments (p_customer_id INTEGER), + returns table (order_key INTEGER, customer_id INTEGER) +); + +SELECT o.order_key, o.customer_id +FROM __ref("stg_orders") AS o +WHERE o.customer_id = p_customer_id diff --git a/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/udf__order_key.sql b/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/udf__order_key.sql new file mode 100644 index 0000000000..53fa6ea33a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/udf__order_key.sql @@ -0,0 +1,7 @@ +FUNCTION ( + description "Display key of an order, ten times its number", + arguments (p_order_id INTEGER), + returns INTEGER, +); + +p_order_id * 10 diff --git a/benchmarks/duckshop/edge_cases/changed_builds_udf/models/first_customer_order_keys.sql b/benchmarks/duckshop/edge_cases/changed_builds_udf/models/first_customer_order_keys.sql new file mode 100644 index 0000000000..71ec3bee57 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_udf/models/first_customer_order_keys.sql @@ -0,0 +1,3 @@ +MODEL (description "Order keys of the first customer, read only through a table function.", materialized table); +SELECT k.order_key +FROM __table_fn("table_fn__customer_order_keys")(1) AS k diff --git a/benchmarks/duckshop/edge_cases/changed_builds_udf/models/stg_orders.sql b/benchmarks/duckshop/edge_cases/changed_builds_udf/models/stg_orders.sql new file mode 100644 index 0000000000..21434bec39 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_udf/models/stg_orders.sql @@ -0,0 +1,3 @@ +MODEL (description "Staged orders with a display key from a SQL function.", materialized table); +SELECT order_id, customer_id, __udf("udf__order_key")(order_id) AS order_key +FROM __source("raw_orders") diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs index 5a286a2592..9ff28d6f0e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::auditing::main::replace_reference_calls::replace_reference_calls; use sqlbuild_state::consumption::models::{InputFact, NodeKey, RelationKey}; use crate::bindings::_helpers::analysis::relation_names::{ @@ -11,6 +12,9 @@ use crate::bindings::_helpers::analysis::relation_names::{ use crate::bindings::_helpers::compiled_views::{model_reads, project_reads, seed_reads}; use crate::bindings::_helpers::records::fields::{field as native_field, mapping_items}; +/// Function calls, `__udf("name")(...)`, parsed as plain calls; their reads are declared inputs. +const FUNCTION_CALLS: [&str; 2] = ["__udf", "__table_fn"]; + /// The relations each model, seed and source names, by node key or source name. pub(crate) struct Locations { nodes: HashMap, @@ -122,6 +126,12 @@ pub(crate) fn model_reads_undeclared( dialect: Option<&str>, ) -> PyResult { let sql: String = model_reads::query_sql(model)?.extract()?; + let sql: String = FUNCTION_CALLS.iter().fold(sql, |sql, function| { + replace_reference_calls(&sql, function, |_| { + Ok::<_, std::convert::Infallible>(Some((*function).to_owned())) + }) + .unwrap_or_else(|never| match never {}) + }); let Some((relations, temporary)) = extract_sql_relation_names(model.py(), &sql, dialect)? else { return Ok(true); diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index f75ccf0029..f62950760a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -1248,6 +1248,7 @@ def target_concurrency_microbatch_files() -> dict[str, str]: CHANGED_BUILDS_FIXTURE: Path = REPO_ROOT / "benchmarks/duckshop/edge_cases/changed_builds" +CHANGED_BUILDS_UDF_FIXTURE: Path = REPO_ROOT / "benchmarks/duckshop/edge_cases/changed_builds_udf" CHANGED_ORDER_SKEW_SECONDS: float = 5.2 _CHANGED_PROGRESS_PATTERN: re.Pattern[str] = re.compile( r"^\s+\d+/\d+\s+\S+\s+(\S+)\s+OK\b", re.MULTILINE @@ -1329,6 +1330,15 @@ def prepare_changed_project(*, tmp_path: Path, warehouse: ChangedWarehouse) -> P return project_dir +def add_changed_function_calls(project_dir: Path) -> None: + """Overlay the DuckShop case whose models call a SQL function and a table function.""" + + for path in CHANGED_BUILDS_UDF_FIXTURE.rglob("*.sql"): + destination: Path = project_dir / path.relative_to(CHANGED_BUILDS_UDF_FIXTURE) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(path.read_text(encoding="utf-8"), encoding="utf-8") + + def run_changed_build(project_dir: Path, arguments: tuple[str, ...]) -> frozenset[str]: """Wait out the order skew, build successfully, and return the models and seeds written.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py index 3e49d83520..80e0a9fce2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py @@ -13,6 +13,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( DUCKDB_CHANGED, SQLITE_CHANGED, + add_changed_function_calls, apply_changed_step, changed_build_steps, kill_changed_build_mid_write, @@ -21,6 +22,8 @@ ) EVERY_MODEL: frozenset[str] = frozenset({"stg_orders", "stg_customers", "customer_orders"}) +# Functions deploy on every build; `--changed` decides models and seeds only. +FUNCTIONS: frozenset[str] = frozenset({"udf__order_key", "table_fn__customer_order_keys"}) EXPECTED_CHANGED_RUNS: tuple[tuple[str, frozenset[str]], ...] = ( ("first --changed build", EVERY_MODEL), ("nothing changed", frozenset()), @@ -57,6 +60,38 @@ def test_given_input_changes_when_building_with_changed_then_exactly_their_consu assert tuple(observed) == test_case.expected_ran +@pytest.mark.parametrize( + "test_case", + [ + ChangedBuildE2ETestCase( + description="duckdb", + warehouse=DUCKDB_CHANGED, + expected_ran=( + ("first --changed build", EVERY_MODEL | {"first_customer_order_keys"}), + ("nothing changed", frozenset()), + ( + "a new order", + frozenset({"stg_orders", "customer_orders", "first_customer_order_keys"}), + ), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_models_calling_functions_when_building_with_changed_then_only_changed_inputs_run( + test_case: ChangedBuildE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_changed_project(tmp_path=tmp_path, warehouse=test_case.warehouse) + add_changed_function_calls(project_dir) + observed: list[tuple[str, frozenset[str]]] = [] + for step in changed_build_steps(test_case.warehouse)[: len(test_case.expected_ran)]: + apply_changed_step(project_dir, test_case.warehouse, step) + ran: frozenset[str] = run_changed_build(project_dir, step.arguments) + observed.append((step.description, ran - FUNCTIONS)) + + assert tuple(observed) == test_case.expected_ran + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index d64538b569..90310c9bab 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -257,22 +257,6 @@ def test_given_external_state_events_when_reading_pages_then_identities_text_and ), ("sqlite", "path", "janitor_event_history", append_janitor_history, "cleanup-run", ()), ("duckdb", "database", "microbatch_history", append_microbatch_history, "", ()), - ( - "duckdb", - "database", - "consumption_history", - append_consumption_history, - "external-build", - (), - ), - ( - "sqlite", - "path", - "consumption_history", - append_consumption_history, - "external-build", - (), - ), ( "duckdb", "database", From ae6059f8b6d0fe97782f83c0727c550f0a748ab3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 5 Nov 2026 00:07:30 +0000 Subject: [PATCH 6/8] fix: run --changed models whose functions read undeclared relations Function call stubbing let the undeclared-read guard parse models that call SQL functions, but it only checked the model's own SQL. A table function or SQL function whose body read a relation it does not declare was invisible, so the model calling it could be skipped while stale. The guard now also checks the body of every SQL function a model calls, transitively, against that function's declared inputs. A body that reads anything else, fails to parse, or belongs to an unknown function forces the model to run. Python functions read no relations. The DuckShop changed_builds_udf case gains a table function and a SQL function that read raw.customers undeclared, and the DuckDB e2e test checks that their models run on every --changed build and pick up a new customer. WIP: validated locally with make check-ci, the hygiene scan, and the changed-builds e2e and scheduler --changed tests. --- .../sql/table_fn__raw_customers_from.sql | 9 ++ .../functions/sql/udf__raw_customer_count.sql | 7 ++ .../models/customer_count_via_udf.sql | 2 + .../models/customers_via_table_fn.sql | 3 + .../bindings/_helpers/consumption/facts.rs | 108 ++++++++++++++++-- .../_helpers/consumption/relation_reads.rs | 16 ++- .../main/build/test_changed_builds.py | 32 +++++- 7 files changed, 155 insertions(+), 22 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/table_fn__raw_customers_from.sql create mode 100644 benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/udf__raw_customer_count.sql create mode 100644 benchmarks/duckshop/edge_cases/changed_builds_udf/models/customer_count_via_udf.sql create mode 100644 benchmarks/duckshop/edge_cases/changed_builds_udf/models/customers_via_table_fn.sql diff --git a/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/table_fn__raw_customers_from.sql b/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/table_fn__raw_customers_from.sql new file mode 100644 index 0000000000..8a4ce1e23b --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/table_fn__raw_customers_from.sql @@ -0,0 +1,9 @@ +FUNCTION ( + description "Customers from an id on, read straight from the raw table without declaring it", + arguments (p_customer_id INTEGER), + returns table (customer_id INTEGER) +); + +SELECT c.customer_id +FROM raw.customers AS c +WHERE c.customer_id >= p_customer_id diff --git a/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/udf__raw_customer_count.sql b/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/udf__raw_customer_count.sql new file mode 100644 index 0000000000..21187a1561 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/udf__raw_customer_count.sql @@ -0,0 +1,7 @@ +FUNCTION ( + description "Number of customers from an id on, counted straight from the raw table without declaring it", + arguments (p_customer_id INTEGER), + returns BIGINT, +); + +(SELECT count(*) FROM raw.customers AS c WHERE c.customer_id >= p_customer_id) diff --git a/benchmarks/duckshop/edge_cases/changed_builds_udf/models/customer_count_via_udf.sql b/benchmarks/duckshop/edge_cases/changed_builds_udf/models/customer_count_via_udf.sql new file mode 100644 index 0000000000..5e3db8ed37 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_udf/models/customer_count_via_udf.sql @@ -0,0 +1,2 @@ +MODEL (description "Customer count read only through a SQL function whose body reads an undeclared raw table.", materialized table); +SELECT __udf("udf__raw_customer_count")(1) AS customer_count diff --git a/benchmarks/duckshop/edge_cases/changed_builds_udf/models/customers_via_table_fn.sql b/benchmarks/duckshop/edge_cases/changed_builds_udf/models/customers_via_table_fn.sql new file mode 100644 index 0000000000..32bb6b2a05 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_udf/models/customers_via_table_fn.sql @@ -0,0 +1,3 @@ +MODEL (description "Customers read only through a table function whose body reads an undeclared raw table.", materialized table); +SELECT k.customer_id +FROM __table_fn("table_fn__raw_customers_from")(1) AS k diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs index 2ad56b6031..61e987e7e1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs @@ -10,7 +10,7 @@ use sqlbuild_state::consumption::models::{ForcedRun, InputFact, NodeFacts, NodeK use crate::bindings::_helpers::compiled_views::{ function_reads, model_reads, project_reads, seed_reads, }; -use crate::bindings::_helpers::consumption::relation_reads::{Locations, model_reads_undeclared}; +use crate::bindings::_helpers::consumption::relation_reads::{Locations, sql_reads_undeclared}; use crate::bindings::_helpers::planner_identity::hashing::{graph_key, model_query_hash}; use crate::bindings::_helpers::planner_identity::identities::hook_version_hashes; use crate::bindings::_helpers::planner_identity::metadata::{ @@ -27,6 +27,8 @@ const UNCHANGED_REASONS: [&str; 2] = ["no_change", "normal_incremental"]; const FIRST_RUN: &str = "first_run"; const FULL_REFRESH: &str = "full_refresh"; const SKIP_ACTION: &str = "skip"; +const SQL_LANGUAGE: &str = "sql"; +const UDF: &str = "udf"; /// The selection's nodes and their ancestry, and the selected ones that execute. pub(crate) struct BuildFacts { @@ -121,6 +123,71 @@ fn inputs( facts } +/// Checks that functions a model calls, transitively, read only their declared inputs. +struct FunctionReads<'a> { + /// Each function's SQL to check, or `None` for a Python function, which reads no relation. + bodies: &'a HashMap<(String, String), Option>, + function_deps: &'a HashMap<(String, String), Vec<(String, String)>>, + sources: &'a HashMap>, + locations: &'a Locations, + dialect: Option<&'a str>, + checked: HashMap<(String, String), bool>, +} + +impl FunctionReads<'_> { + fn undeclared(&mut self, py: Python<'_>, deps: &[(String, String)]) -> PyResult { + let mut pending: VecDeque<(String, String)> = deps + .iter() + .filter(|(kind, _)| FUNCTION_KINDS.contains(&kind.as_str())) + .cloned() + .collect(); + let mut seen: BTreeSet<(String, String)> = BTreeSet::new(); + while let Some(key) = pending.pop_front() { + if !seen.insert(key.clone()) { + continue; + } + let own_deps: Vec<(String, String)> = + self.function_deps.get(&key).cloned().unwrap_or_default(); + if self.function_undeclared(py, &key, &own_deps)? { + return Ok(true); + } + pending.extend( + own_deps + .into_iter() + .filter(|(kind, _)| FUNCTION_KINDS.contains(&kind.as_str())), + ); + } + Ok(false) + } + + fn function_undeclared( + &mut self, + py: Python<'_>, + key: &(String, String), + own_deps: &[(String, String)], + ) -> PyResult { + if let Some(undeclared) = self.checked.get(key) { + return Ok(*undeclared); + } + let undeclared: bool = match self.bodies.get(key) { + None => true, + Some(None) => false, + Some(Some(sql)) => { + let allowed: Vec = + inputs(own_deps.to_vec(), self.function_deps, self.sources); + sql_reads_undeclared( + py, + sql.clone(), + (&allowed, None), + (self.locations, self.dialect), + )? + } + }; + self.checked.insert(key.clone(), undeclared); + Ok(undeclared) + } +} + fn key_rows(keys: &Bound<'_, PyAny>) -> PyResult> { keys.try_iter()? .map(|key| { @@ -173,14 +240,27 @@ pub(crate) fn build_facts( ); } let mut function_deps: HashMap<(String, String), Vec<(String, String)>> = HashMap::new(); + let mut function_bodies: HashMap<(String, String), Option> = HashMap::new(); let functions: Bound<'_, PyAny> = project_reads::functions(project)?; for function in functions.try_iter()? { let function: Bound<'_, PyAny> = function?; let key = graph_key(&function_reads::key(&function)?)?; function_deps.insert( - (key.kind, key.name), + (key.kind.clone(), key.name.clone()), key_rows(&function_reads::deps(&function)?)?, ); + let body: Option = + if enum_text(&function_reads::language(&function)?)? == SQL_LANGUAGE { + let body: String = function_reads::body_sql(&function)?.extract()?; + Some(if key.kind == UDF { + format!("SELECT ({body})") + } else { + body + }) + } else { + None + }; + function_bodies.insert((key.kind, key.name), body); } let mut selected: BTreeMap> = BTreeMap::new(); let mut runnable: BTreeSet = BTreeSet::new(); @@ -213,6 +293,15 @@ pub(crate) fn build_facts( } else { None }; + let mut function_checks: Option> = + locations.as_ref().map(|locations| FunctionReads { + bodies: &function_bodies, + function_deps: &function_deps, + sources: &sources, + locations, + dialect: dialect.as_deref(), + checked: HashMap::new(), + }); let mut nodes: Vec = Vec::new(); let mut pending: VecDeque = selected.keys().cloned().collect(); let mut visited: BTreeSet = BTreeSet::new(); @@ -249,7 +338,7 @@ pub(crate) fn build_facts( seed_identity(seed)?.0, ), }; - let node_inputs: Vec = inputs(deps, &function_deps, &sources); + let node_inputs: Vec = inputs(deps.clone(), &function_deps, &sources); for input in &node_inputs { if let InputFact::Node(upstream) = input { pending.push_back(upstream.clone()); @@ -263,13 +352,14 @@ pub(crate) fn build_facts( None => Vec::new(), }; let relation: Option = relation(&destination)?; - if let (Compiled::Model(model), Some(locations)) = (node, &locations) - && model_reads_undeclared( - model, + if let (Compiled::Model(model), Some(locations), Some(function_checks)) = + (node, &locations, &mut function_checks) + && (sql_reads_undeclared( + py, + model_reads::query_sql(model)?.extract()?, (&node_inputs, relation.as_ref()), - locations, - dialect.as_deref(), - )? + (locations, dialect.as_deref()), + )? || function_checks.undeclared(py, &deps)?) { forced.push(ForcedRun::InputsNotEnumerable); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs index 9ff28d6f0e..04513e209c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_analysis::auditing::main::replace_reference_calls::replace_reference_calls; use sqlbuild_state::consumption::models::{InputFact, NodeKey, RelationKey}; @@ -118,22 +118,20 @@ fn names((name, schema, database): &RelationRow, location: &RelationKey) -> bool }) } -/// True when the SQL cannot be parsed or names a relation no input or the model itself writes. -pub(crate) fn model_reads_undeclared( - model: &Bound<'_, PyAny>, +/// True when `sql` cannot be parsed or names a relation no input or its own relation writes. +pub(crate) fn sql_reads_undeclared( + py: Python<'_>, + sql: String, (inputs, own): (&[InputFact], Option<&RelationKey>), - locations: &Locations, - dialect: Option<&str>, + (locations, dialect): (&Locations, Option<&str>), ) -> PyResult { - let sql: String = model_reads::query_sql(model)?.extract()?; let sql: String = FUNCTION_CALLS.iter().fold(sql, |sql, function| { replace_reference_calls(&sql, function, |_| { Ok::<_, std::convert::Infallible>(Some((*function).to_owned())) }) .unwrap_or_else(|never| match never {}) }); - let Some((relations, temporary)) = extract_sql_relation_names(model.py(), &sql, dialect)? - else { + let Some((relations, temporary)) = extract_sql_relation_names(py, &sql, dialect)? else { return Ok(true); }; for relation in &relations { diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py index 80e0a9fce2..f3cee88112 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py @@ -23,7 +23,16 @@ EVERY_MODEL: frozenset[str] = frozenset({"stg_orders", "stg_customers", "customer_orders"}) # Functions deploy on every build; `--changed` decides models and seeds only. -FUNCTIONS: frozenset[str] = frozenset({"udf__order_key", "table_fn__customer_order_keys"}) +FUNCTIONS: frozenset[str] = frozenset( + { + "udf__order_key", + "table_fn__customer_order_keys", + "udf__raw_customer_count", + "table_fn__raw_customers_from", + } +) +# Their functions read raw.customers without declaring it, so they run on every build. +UNDECLARED_READERS: frozenset[str] = frozenset({"customers_via_table_fn", "customer_count_via_udf"}) EXPECTED_CHANGED_RUNS: tuple[tuple[str, frozenset[str]], ...] = ( ("first --changed build", EVERY_MODEL), ("nothing changed", frozenset()), @@ -67,11 +76,15 @@ def test_given_input_changes_when_building_with_changed_then_exactly_their_consu description="duckdb", warehouse=DUCKDB_CHANGED, expected_ran=( - ("first --changed build", EVERY_MODEL | {"first_customer_order_keys"}), - ("nothing changed", frozenset()), + ( + "first --changed build", + EVERY_MODEL | UNDECLARED_READERS | {"first_customer_order_keys"}, + ), + ("nothing changed", UNDECLARED_READERS), ( "a new order", - frozenset({"stg_orders", "customer_orders", "first_customer_order_keys"}), + UNDECLARED_READERS + | {"stg_orders", "customer_orders", "first_customer_order_keys"}, ), ), ) @@ -88,8 +101,19 @@ def test_given_models_calling_functions_when_building_with_changed_then_only_cha apply_changed_step(project_dir, test_case.warehouse, step) ran: frozenset[str] = run_changed_build(project_dir, step.arguments) observed.append((step.description, ran - FUNCTIONS)) + warehouse_file: Path = project_dir / test_case.warehouse.warehouse_file + test_case.warehouse.execute( + warehouse_file, f"INSERT INTO {test_case.warehouse.customers_table} VALUES (2, 3)" + ) + run_changed_build(project_dir, ("--changed",)) + function_reads: list[tuple[object, ...]] = test_case.warehouse.execute( + warehouse_file, + "SELECT (SELECT count(*) FROM main.customers_via_table_fn), " + "(SELECT customer_count FROM main.customer_count_via_udf)", + ) assert tuple(observed) == test_case.expected_ran + assert function_reads == [(2, 2)] @pytest.mark.parametrize( From 53d071675f58da2adaef5a735d5f2921a9b882ea Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 5 Nov 2026 00:18:57 +0000 Subject: [PATCH 7/8] fix: run --changed models whose unqualified names are ambiguous The undeclared-read guard accepted a relation name the SQL spells without its schema or database whenever a declared input had that name. When two project relations share the name across schemas, the warehouse resolves the omitted qualifier from its session default and can read the one that is not declared, so --changed skipped a stale model: a model declaring the raw customers source while reading the customers model unqualified did not rerun after that model changed. A name that matches more than one project relation now forces the model to run. Project relations are indexed by name, so each check stays constant time. The DuckShop changed_builds_unqualified case reproduces it, and a DuckDB e2e test checks that the reader runs and picks up the change. WIP: validated with make check-ci, the hygiene scan, the changed-builds e2e and scheduler --changed tests, and paired 3,000-model timings. --- .../models/customers.sql | 2 + .../models/unqualified_customer_reader.sql | 5 ++ .../_helpers/consumption/relation_reads.rs | 42 ++++++++++++++-- .../cli/commands/main/build/helpers.py | 11 +++-- .../main/build/test_changed_builds.py | 48 ++++++++++++++++++- 5 files changed, 98 insertions(+), 10 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/changed_builds_unqualified/models/customers.sql create mode 100644 benchmarks/duckshop/edge_cases/changed_builds_unqualified/models/unqualified_customer_reader.sql diff --git a/benchmarks/duckshop/edge_cases/changed_builds_unqualified/models/customers.sql b/benchmarks/duckshop/edge_cases/changed_builds_unqualified/models/customers.sql new file mode 100644 index 0000000000..37e05da290 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_unqualified/models/customers.sql @@ -0,0 +1,2 @@ +MODEL (description "Customers in the target schema, sharing a table name with the raw customers source.", materialized table); +SELECT customer_id FROM __ref("stg_customers") diff --git a/benchmarks/duckshop/edge_cases/changed_builds_unqualified/models/unqualified_customer_reader.sql b/benchmarks/duckshop/edge_cases/changed_builds_unqualified/models/unqualified_customer_reader.sql new file mode 100644 index 0000000000..c6c8a98df9 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_unqualified/models/unqualified_customer_reader.sql @@ -0,0 +1,5 @@ +MODEL (description "Declares the raw customers source but also reads the customers model by its unqualified name.", materialized table); +SELECT r.customer_id, count(c.customer_id) AS modelled +FROM __source("raw_customers") AS r +LEFT JOIN customers AS c USING (customer_id) +GROUP BY r.customer_id diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs index 04513e209c..ea22765f74 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs @@ -1,6 +1,6 @@ //! Whether a model's compiled SQL reads a relation that none of its declared inputs writes. -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_analysis::auditing::main::replace_reference_calls::replace_reference_calls; @@ -19,6 +19,8 @@ const FUNCTION_CALLS: [&str; 2] = ["__udf", "__table_fn"]; pub(crate) struct Locations { nodes: HashMap, sources: HashMap, Option, String)>, + /// Every project relation, distinct and lower-cased, by lower-cased relation name. + by_name: HashMap>, } fn optional(value: Bound<'_, PyAny>) -> PyResult> { @@ -75,7 +77,34 @@ impl Locations { ); } } - Ok(Self { nodes, sources }) + let mut by_name: HashMap> = HashMap::new(); + let source_keys = sources + .values() + .map(|(database, schema, table)| RelationKey { + database: database.clone(), + schema: schema.clone().unwrap_or_default(), + name: table.clone(), + }); + for key in nodes.values().cloned().chain(source_keys) { + let key: RelationKey = RelationKey { + database: key.database.map(|database| database.to_lowercase()), + schema: key.schema.to_lowercase(), + name: key.name.to_lowercase(), + }; + by_name.entry(key.name.clone()).or_default().insert(key); + } + Ok(Self { + nodes, + sources, + by_name, + }) + } + + /// Whether the SQL relation, with the qualifiers it spells, can name several project relations. + fn ambiguous(&self, relation: &RelationRow) -> bool { + self.by_name + .get(&relation.0.to_lowercase()) + .is_some_and(|keys| keys.iter().filter(|key| names(relation, key)).count() > 1) } fn allows(&self, input: &InputFact, relation: &RelationRow) -> bool { @@ -118,7 +147,7 @@ fn names((name, schema, database): &RelationRow, location: &RelationKey) -> bool }) } -/// True when `sql` cannot be parsed or names a relation no input or its own relation writes. +/// True when `sql` cannot be parsed or names an ambiguous relation or one no input writes. pub(crate) fn sql_reads_undeclared( py: Python<'_>, sql: String, @@ -139,7 +168,12 @@ pub(crate) fn sql_reads_undeclared( .iter() .any(|created| created.eq_ignore_ascii_case(&relation.0)); let declared: bool = inputs.iter().any(|input| locations.allows(input, relation)); - if !created && !own.is_some_and(|own| names(relation, own)) && !declared { + if created { + continue; + } + if locations.ambiguous(relation) + || (!own.is_some_and(|own| names(relation, own)) && !declared) + { return Ok(true); } } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index f62950760a..29dad07049 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -1249,6 +1249,9 @@ def target_concurrency_microbatch_files() -> dict[str, str]: CHANGED_BUILDS_FIXTURE: Path = REPO_ROOT / "benchmarks/duckshop/edge_cases/changed_builds" CHANGED_BUILDS_UDF_FIXTURE: Path = REPO_ROOT / "benchmarks/duckshop/edge_cases/changed_builds_udf" +CHANGED_BUILDS_UNQUALIFIED_FIXTURE: Path = ( + REPO_ROOT / "benchmarks/duckshop/edge_cases/changed_builds_unqualified" +) CHANGED_ORDER_SKEW_SECONDS: float = 5.2 _CHANGED_PROGRESS_PATTERN: re.Pattern[str] = re.compile( r"^\s+\d+/\d+\s+\S+\s+(\S+)\s+OK\b", re.MULTILINE @@ -1330,11 +1333,11 @@ def prepare_changed_project(*, tmp_path: Path, warehouse: ChangedWarehouse) -> P return project_dir -def add_changed_function_calls(project_dir: Path) -> None: - """Overlay the DuckShop case whose models call a SQL function and a table function.""" +def add_changed_overlay(project_dir: Path, fixture: Path) -> None: + """Overlay one of the DuckShop changed-builds cases on the base case.""" - for path in CHANGED_BUILDS_UDF_FIXTURE.rglob("*.sql"): - destination: Path = project_dir / path.relative_to(CHANGED_BUILDS_UDF_FIXTURE) + for path in fixture.rglob("*.sql"): + destination: Path = project_dir / path.relative_to(fixture) destination.parent.mkdir(parents=True, exist_ok=True) destination.write_text(path.read_text(encoding="utf-8"), encoding="utf-8") diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py index f3cee88112..f9d396497f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py @@ -11,9 +11,11 @@ ChangedCrashE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( + CHANGED_BUILDS_UDF_FIXTURE, + CHANGED_BUILDS_UNQUALIFIED_FIXTURE, DUCKDB_CHANGED, SQLITE_CHANGED, - add_changed_function_calls, + add_changed_overlay, apply_changed_step, changed_build_steps, kill_changed_build_mid_write, @@ -95,7 +97,7 @@ def test_given_models_calling_functions_when_building_with_changed_then_only_cha test_case: ChangedBuildE2ETestCase, tmp_path: Path ) -> None: project_dir: Path = prepare_changed_project(tmp_path=tmp_path, warehouse=test_case.warehouse) - add_changed_function_calls(project_dir) + add_changed_overlay(project_dir, CHANGED_BUILDS_UDF_FIXTURE) observed: list[tuple[str, frozenset[str]]] = [] for step in changed_build_steps(test_case.warehouse)[: len(test_case.expected_ran)]: apply_changed_step(project_dir, test_case.warehouse, step) @@ -116,6 +118,48 @@ def test_given_models_calling_functions_when_building_with_changed_then_only_cha assert function_reads == [(2, 2)] +@pytest.mark.parametrize( + "test_case", + [ + ChangedBuildE2ETestCase( + description="duckdb", + warehouse=DUCKDB_CHANGED, + expected_ran=( + ("nothing changed", frozenset({"unqualified_customer_reader"})), + ("customers redefined", frozenset({"customers", "unqualified_customer_reader"})), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_an_unqualified_name_matching_two_relations_when_building_with_changed_then_it_runs( + test_case: ChangedBuildE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_changed_project(tmp_path=tmp_path, warehouse=test_case.warehouse) + add_changed_overlay(project_dir, CHANGED_BUILDS_UNQUALIFIED_FIXTURE) + customers_model: Path = project_dir / "models/customers.sql" + run_changed_build(project_dir, ("--select", "+customers")) + run_changed_build(project_dir, ("--changed",)) + nothing_changed: frozenset[str] = run_changed_build(project_dir, ("--changed",)) + customers_model.write_text( + customers_model.read_text(encoding="utf-8").replace( + '__ref("stg_customers")', '__ref("stg_customers") WHERE customer_id > 100' + ), + encoding="utf-8", + ) + redefined: frozenset[str] = run_changed_build(project_dir, ("--changed",)) + modelled: list[tuple[object, ...]] = test_case.warehouse.execute( + project_dir / test_case.warehouse.warehouse_file, + "SELECT modelled FROM main.unqualified_customer_reader", + ) + + assert ( + ("nothing changed", nothing_changed), + ("customers redefined", redefined), + ) == test_case.expected_ran + assert modelled == [(0,)] + + @pytest.mark.parametrize( "test_case", [ From 50ebc84a23445e54b8437a5ca72438d4c1335139 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 5 Nov 2026 01:09:48 +0000 Subject: [PATCH 8/8] test: read the unqualified reader after its upstream settles The reader does not declare the customers model, so nothing orders it after that model's rebuild and it can read the old rows, as it did on CI. It runs on every --changed build, so the test now checks its data after the next build, once customers has settled. WIP: validated with make check-ci, the hygiene scan and the changed-builds e2e tests. --- .../sqlbuild/cli/commands/main/build/test_changed_builds.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py index f9d396497f..0578a7d699 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py @@ -127,6 +127,7 @@ def test_given_models_calling_functions_when_building_with_changed_then_only_cha expected_ran=( ("nothing changed", frozenset({"unqualified_customer_reader"})), ("customers redefined", frozenset({"customers", "unqualified_customer_reader"})), + ("customers settled", frozenset({"unqualified_customer_reader"})), ), ) ], @@ -148,6 +149,9 @@ def test_given_an_unqualified_name_matching_two_relations_when_building_with_cha encoding="utf-8", ) redefined: frozenset[str] = run_changed_build(project_dir, ("--changed",)) + # Nothing orders the reader after the undeclared `customers`, so it may read it before the + # rebuild; it runs on every `--changed` build, so the next one reads the settled model. + settled: frozenset[str] = run_changed_build(project_dir, ("--changed",)) modelled: list[tuple[object, ...]] = test_case.warehouse.execute( project_dir / test_case.warehouse.warehouse_file, "SELECT modelled FROM main.unqualified_customer_reader", @@ -156,6 +160,7 @@ def test_given_an_unqualified_name_matching_two_relations_when_building_with_cha assert ( ("nothing changed", nothing_changed), ("customers redefined", redefined), + ("customers settled", settled), ) == test_case.expected_ran assert modelled == [(0,)]