diff --git a/Makefile b/Makefile index b795940c4c..9aae33ca40 100644 --- a/Makefile +++ b/Makefile @@ -145,6 +145,7 @@ E2E_DUCKDB_BUILD_INCREMENTAL_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_append_cursor_build.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_dropped_relation_rebuild.py \ diff --git a/benchmarks/duckshop/edge_cases/changed_builds/models/customer_orders.sql b/benchmarks/duckshop/edge_cases/changed_builds/models/customer_orders.sql new file mode 100644 index 0000000000..730ca07923 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds/models/customer_orders.sql @@ -0,0 +1,5 @@ +MODEL (description "Orders per customer, two models downstream of both sources.", materialized table); +SELECT c.customer_id, count(o.order_id) AS orders +FROM __ref("stg_customers") AS c +LEFT JOIN __ref("stg_orders") AS o USING (customer_id) +GROUP BY c.customer_id diff --git a/benchmarks/duckshop/edge_cases/changed_builds/models/stg_customers.sql b/benchmarks/duckshop/edge_cases/changed_builds/models/stg_customers.sql new file mode 100644 index 0000000000..73f0c12b81 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds/models/stg_customers.sql @@ -0,0 +1,2 @@ +MODEL (description "Staged customers.", materialized table); +SELECT customer_id FROM __source("raw_customers") diff --git a/benchmarks/duckshop/edge_cases/changed_builds/models/stg_orders.sql b/benchmarks/duckshop/edge_cases/changed_builds/models/stg_orders.sql new file mode 100644 index 0000000000..80711320ca --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds/models/stg_orders.sql @@ -0,0 +1,2 @@ +MODEL (description "Staged orders.", materialized table); +SELECT order_id, customer_id FROM __source("raw_orders") diff --git a/benchmarks/duckshop/edge_cases/changed_builds/sources/raw.yml b/benchmarks/duckshop/edge_cases/changed_builds/sources/raw.yml new file mode 100644 index 0000000000..7c84628bd5 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds/sources/raw.yml @@ -0,0 +1,17 @@ +sources: + - name: raw_orders + description: Synthetic orders, numbered by load batch. + schema: raw + table: orders + freshness: + strategy: column + column: load_id + type: integer + - name: raw_customers + description: Synthetic customers, numbered by load batch. + schema: raw + table: customers + freshness: + strategy: column + column: load_id + type: integer diff --git a/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/table_fn__customer_order_keys.sql b/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/table_fn__customer_order_keys.sql new file mode 100644 index 0000000000..c42ac12221 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/table_fn__customer_order_keys.sql @@ -0,0 +1,9 @@ +FUNCTION ( + description "Order keys of one customer, read from staged orders", + arguments (p_customer_id INTEGER), + returns table (order_key INTEGER, customer_id INTEGER) +); + +SELECT o.order_key, o.customer_id +FROM __ref("stg_orders") AS o +WHERE o.customer_id = p_customer_id diff --git a/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/table_fn__raw_customers_from.sql b/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/table_fn__raw_customers_from.sql new file mode 100644 index 0000000000..8a4ce1e23b --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/table_fn__raw_customers_from.sql @@ -0,0 +1,9 @@ +FUNCTION ( + description "Customers from an id on, read straight from the raw table without declaring it", + arguments (p_customer_id INTEGER), + returns table (customer_id INTEGER) +); + +SELECT c.customer_id +FROM raw.customers AS c +WHERE c.customer_id >= p_customer_id diff --git a/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/udf__order_key.sql b/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/udf__order_key.sql new file mode 100644 index 0000000000..53fa6ea33a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/udf__order_key.sql @@ -0,0 +1,7 @@ +FUNCTION ( + description "Display key of an order, ten times its number", + arguments (p_order_id INTEGER), + returns INTEGER, +); + +p_order_id * 10 diff --git a/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/udf__raw_customer_count.sql b/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/udf__raw_customer_count.sql new file mode 100644 index 0000000000..21187a1561 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_udf/functions/sql/udf__raw_customer_count.sql @@ -0,0 +1,7 @@ +FUNCTION ( + description "Number of customers from an id on, counted straight from the raw table without declaring it", + arguments (p_customer_id INTEGER), + returns BIGINT, +); + +(SELECT count(*) FROM raw.customers AS c WHERE c.customer_id >= p_customer_id) diff --git a/benchmarks/duckshop/edge_cases/changed_builds_udf/models/customer_count_via_udf.sql b/benchmarks/duckshop/edge_cases/changed_builds_udf/models/customer_count_via_udf.sql new file mode 100644 index 0000000000..5e3db8ed37 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_udf/models/customer_count_via_udf.sql @@ -0,0 +1,2 @@ +MODEL (description "Customer count read only through a SQL function whose body reads an undeclared raw table.", materialized table); +SELECT __udf("udf__raw_customer_count")(1) AS customer_count diff --git a/benchmarks/duckshop/edge_cases/changed_builds_udf/models/customers_via_table_fn.sql b/benchmarks/duckshop/edge_cases/changed_builds_udf/models/customers_via_table_fn.sql new file mode 100644 index 0000000000..32bb6b2a05 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_udf/models/customers_via_table_fn.sql @@ -0,0 +1,3 @@ +MODEL (description "Customers read only through a table function whose body reads an undeclared raw table.", materialized table); +SELECT k.customer_id +FROM __table_fn("table_fn__raw_customers_from")(1) AS k diff --git a/benchmarks/duckshop/edge_cases/changed_builds_udf/models/first_customer_order_keys.sql b/benchmarks/duckshop/edge_cases/changed_builds_udf/models/first_customer_order_keys.sql new file mode 100644 index 0000000000..71ec3bee57 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_udf/models/first_customer_order_keys.sql @@ -0,0 +1,3 @@ +MODEL (description "Order keys of the first customer, read only through a table function.", materialized table); +SELECT k.order_key +FROM __table_fn("table_fn__customer_order_keys")(1) AS k diff --git a/benchmarks/duckshop/edge_cases/changed_builds_udf/models/stg_orders.sql b/benchmarks/duckshop/edge_cases/changed_builds_udf/models/stg_orders.sql new file mode 100644 index 0000000000..21434bec39 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_udf/models/stg_orders.sql @@ -0,0 +1,3 @@ +MODEL (description "Staged orders with a display key from a SQL function.", materialized table); +SELECT order_id, customer_id, __udf("udf__order_key")(order_id) AS order_key +FROM __source("raw_orders") diff --git a/benchmarks/duckshop/edge_cases/changed_builds_unqualified/models/customers.sql b/benchmarks/duckshop/edge_cases/changed_builds_unqualified/models/customers.sql new file mode 100644 index 0000000000..37e05da290 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_unqualified/models/customers.sql @@ -0,0 +1,2 @@ +MODEL (description "Customers in the target schema, sharing a table name with the raw customers source.", materialized table); +SELECT customer_id FROM __ref("stg_customers") diff --git a/benchmarks/duckshop/edge_cases/changed_builds_unqualified/models/unqualified_customer_reader.sql b/benchmarks/duckshop/edge_cases/changed_builds_unqualified/models/unqualified_customer_reader.sql new file mode 100644 index 0000000000..c6c8a98df9 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_unqualified/models/unqualified_customer_reader.sql @@ -0,0 +1,5 @@ +MODEL (description "Declares the raw customers source but also reads the customers model by its unqualified name.", materialized table); +SELECT r.customer_id, count(c.customer_id) AS modelled +FROM __source("raw_customers") AS r +LEFT JOIN customers AS c USING (customer_id) +GROUP BY r.customer_id diff --git a/benchmarks/duckshop/edge_cases/orchestration/consumption_history.json b/benchmarks/duckshop/edge_cases/orchestration/consumption_history.json new file mode 100644 index 0000000000..20e85c5bef --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/consumption_history.json @@ -0,0 +1,70 @@ +{ + "record": { + "state_database": null, + "state_schema": "main", + "event_id": "consumption-000", + "execution_id": "0190a1b2-0000-7000-8000-000000000001", + "phase": "succeeded", + "run_id": "external-build", + "node_type": "model", + "node_name": "customer_orders", + "target_database": null, + "target_schema": "main", + "target_name": "customer_orders_🧾", + "version_hash": "customer-orders-version", + "input_count": 2, + "input_kind": "source", + "input_name": "shop.orders", + "input_version": "load 'orders' 🧾 #7", + "recorded_at": "2026-02-03T04:05:06Z" + }, + "source_rows": 104, + "duplicate_rows": 3, + "later_time": "2026-02-03T04:05:06.001Z", + "retained_result": "consumption-103", + "kind": "consumption", + "source_key": "event_id", + "timestamp_key": "recorded_at", + "id_prefix": "consumption", + "page_lengths": [ + 100, + 7 + ], + "later_index": 106, + "later_record": { + "event_id": "consumption-103", + "phase": "started", + "execution_id": "0190a1b2-0000-7000-8000-000000000002", + "input_kind": null, + "input_name": null, + "input_version": null, + "recorded_at": "2026-02-03T04:05:06.001Z" + }, + "last_record": { + "event_id": "consumption-103", + "phase": "started", + "execution_id": "0190a1b2-0000-7000-8000-000000000002", + "input_kind": null, + "input_name": null, + "input_version": null, + "recorded_at": "2026-02-03T04:05:06.001Z" + }, + "record_overrides": { + "consumption-102": { + "input_kind": "node", + "input_name": "model.stg_customers", + "input_version": "0190a1b2-0000-7000-8000-0000000000aa" + }, + "consumption-103": { + "event_id": "consumption-103", + "phase": "started", + "execution_id": "0190a1b2-0000-7000-8000-000000000002", + "input_kind": null, + "input_name": null, + "input_version": null, + "recorded_at": "2026-02-03T04:05:06.001Z" + } + }, + "source_count": 107, + "model": "MODEL (description 'Orders.', materialized table); SELECT 1 AS order_id" +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index d48b740008..4af98f4730 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -499,6 +499,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/locations/{location_id}/targets/{target}/metadata/consumption": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["report_consumption"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/locations/{location_id}/targets/{target}/metadata/fingerprints": { parameters: { query?: never; @@ -997,6 +1013,14 @@ export interface components { /** Format: uuid */ request_id: string; }; + ConsumptionBatch: { + /** Format: int64 */ + generation: number; + reconciliation?: null | components["schemas"]["WarehouseReconciliationPage"]; + records: components["schemas"]["WarehouseConsumptionRecord"][]; + /** Format: uuid */ + session_id: string; + }; /** @description The current successful snapshot, read with a consistent generation and change set. */ CurrentSnapshot: { changes: components["schemas"]["SnapshotChange"][]; @@ -1929,6 +1953,28 @@ export interface components { state_database?: string | null; state_schema: string; }; + /** @description What one execution consumed, as history; `--changed` reads the warehouse table, never this mirror. */ + WarehouseConsumptionRecord: { + event_id: string; + execution_id: string; + /** Format: int64 */ + input_count: number; + input_kind?: string | null; + input_name?: string | null; + input_version?: string | null; + node_name: string; + node_type: string; + phase: string; + /** Format: date-time */ + recorded_at: string; + run_id: string; + state_database?: string | null; + state_schema: string; + target_database?: string | null; + target_name: string; + target_schema: string; + version_hash: string; + }; /** @description Fingerprints remain history here; latest-row selection belongs to the existing planner reader. */ WarehouseFingerprintRecord: { definition: string; @@ -1969,7 +2015,7 @@ export interface components { * @description Catalog observations require full scans; only immutable history supports a watermark. * @enum {string} */ - WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches" | "catalog" | "statistics"; + WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches" | "consumption" | "catalog" | "statistics"; /** @description Warehouse cleanup facts do not authorize control-plane deletion or worker recovery. */ WarehouseJanitorRecord: { archive_name: string; @@ -3437,6 +3483,54 @@ export interface operations { }; }; }; + report_consumption: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ConsumptionBatch"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WarehouseSyncResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Invalid credential */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Location not authorized */ + 403: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; report_fingerprints: { parameters: { query?: never; diff --git a/clients/typescript/tests/types.ts b/clients/typescript/tests/types.ts index 73c6560b93..a720ad4c93 100644 --- a/clients/typescript/tests/types.ts +++ b/clients/typescript/tests/types.ts @@ -67,6 +67,14 @@ void client.POST("/api/locations/{location_id}/targets/{target}/metadata/microba params: { path: { location_id: "synthetic-location", target: "development" } }, body: { session_id: "synthetic-session", generation: 1, records: [] }, }); +const consumptionHistory: components["schemas"]["WarehouseHistoryKind"] = "consumption"; +const startedInput: components["schemas"]["WarehouseConsumptionRecord"]["input_version"] = null; +void consumptionHistory; +void startedInput; +void client.POST("/api/locations/{location_id}/targets/{target}/metadata/consumption", { + params: { path: { location_id: "synthetic-location", target: "development" } }, + body: { session_id: "synthetic-session", generation: 1, records: [] }, +}); const catalogHistory: components["schemas"]["WarehouseHistoryKind"] = "catalog"; const catalogColumn: components["schemas"]["WarehouseCatalogColumn"] = { name: "order_id", data_type: "INTEGER", diff --git a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs index 6f0b096877..29ab3e1097 100644 --- a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs +++ b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs @@ -1,16 +1,18 @@ //! Janitor planning names, prefixes and skip reasons. use sqlbuild_core::constants::COLUMN_MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::CONSUMPTION_TABLE_NAME; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use sqlbuild_core::constants::MIGRATION_TABLE_NAME; use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; /// SQLBuild state tables the janitor never acts on. -pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 9] = [ +pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 10] = [ FINGERPRINT_TABLE_NAME, "_sqlbuild_microbatches", SOURCE_FRESHNESS_TABLE_NAME, + CONSUMPTION_TABLE_NAME, "_sqlbuild_node_results", "_sqlbuild_audit_results", "_sqlbuild_janitor_events", diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index d352ae8208..4ca62b994c 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -2360,6 +2360,17 @@ "default": false, "group": 1 }, + { + "kind": "store_true", + "options": [ + "--changed" + ], + "dest": "changed", + "nargs": 0, + "const": true, + "default": false, + "group": 1 + }, { "kind": "store_true", "options": [ diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 3c079c4086..e4a1b23bfc 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -71,7 +71,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 184] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 185] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), @@ -126,6 +126,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 184] = [ ("no_color", NamespaceDefault::Bool(false)), ("fail_fast", NamespaceDefault::Bool(false)), ("full_refresh", NamespaceDefault::Bool(false)), + ("changed", NamespaceDefault::Bool(false)), ("as_target", NamespaceDefault::None), ("include_python", NamespaceDefault::Bool(true)), ("load_sources", NamespaceDefault::None), diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 247242fd45..9e50dee08d 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -139,7 +139,7 @@ pub(crate) const PLAN_ROUTE: Route = routed( REQUEST_KEYWORD_SHAPE, ); -const BUILD_FIELDS: [Field; 35] = [ +const BUILD_FIELDS: [Field; 36] = [ PROJECT_DIR, NO_SQL_VALIDATION, same("no_cache"), @@ -151,6 +151,7 @@ const BUILD_FIELDS: [Field; 35] = [ NO_COLOR, same("fail_fast"), same("full_refresh"), + same("changed"), same("load_sources"), attr("reload_sources", "reload"), same("include_python"), diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 043b0b9682..599fce8e15 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -521,6 +521,15 @@ pub(crate) async fn deliver_metadata_page( ) .await } + WarehouseHistoryKind::Consumption => { + client + .sync_consumption( + target, + &serde_json::from_value(body.clone()) + .map_err(|_| ClientError::Configuration("invalid consumption batch"))?, + ) + .await + } } } diff --git a/crates/sqlbuild-core/src/constants.rs b/crates/sqlbuild-core/src/constants.rs index 75a4f1cdc8..7f5096db14 100644 --- a/crates/sqlbuild-core/src/constants.rs +++ b/crates/sqlbuild-core/src/constants.rs @@ -22,6 +22,8 @@ pub const PYTHON_INIT_MODULE_STEM: &str = "__init__"; pub const FINGERPRINT_TABLE_NAME: &str = "_sqlbuild_fingerprints"; /// State table of observed source freshness. pub const SOURCE_FRESHNESS_TABLE_NAME: &str = "_sqlbuild_source_freshness"; +/// State table of the inputs each model and seed execution consumed. +pub const CONSUMPTION_TABLE_NAME: &str = "_sqlbuild_consumption"; /// State table of model migration events. pub const MIGRATION_TABLE_NAME: &str = "_sqlbuild_migrations"; /// State table of column migration events. diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs index 1143bfe060..d5d7f24755 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/router.rs @@ -59,6 +59,10 @@ pub(crate) fn router(store: &ControlStore, feed: ChangeFeed) -> Router, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(batch): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json( + ingest_consumption(&store, (location_id, &target), &batch).await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index ba9d2139b1..e3416f511c 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -68,6 +68,7 @@ use crate::api::_helpers::resources::{ warehouse_metadata::report_migrations, warehouse_metadata::report_old_name_views, warehouse_metadata::report_microbatches, + warehouse_metadata::report_consumption, warehouse_metadata::report_catalog, warehouse_metadata::report_statistics, warehouse_metadata::statistics_due, diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs index f530db781e..dde7ba2d2d 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -10,9 +10,9 @@ use crate::platform::models::{ LocationWorkResult, }; use crate::warehouse_metadata::models::{ - AuditResultsBatch, CatalogBatch, FingerprintBatch, JanitorEventsBatch, MicrobatchesBatch, - MigrationsBatch, OldNameViewsBatch, SourceFreshnessBatch, StatisticsBatch, StatisticsDueRead, - StatisticsRead, WarehouseSyncResponse, WarehouseWatermarkRead, + AuditResultsBatch, CatalogBatch, ConsumptionBatch, FingerprintBatch, JanitorEventsBatch, + MicrobatchesBatch, MigrationsBatch, OldNameViewsBatch, SourceFreshnessBatch, StatisticsBatch, + StatisticsDueRead, StatisticsRead, WarehouseSyncResponse, WarehouseWatermarkRead, }; impl LocationWorkClient { @@ -109,6 +109,14 @@ impl LocationWorkClient { self.sync_metadata((target, "microbatches"), batch).await } + pub async fn sync_consumption( + &self, + target: &str, + batch: &ConsumptionBatch, + ) -> Result { + self.sync_metadata((target, "consumption"), batch).await + } + /// Metadata shares the location's scoped pool, not the scheduler's execution authority. pub async fn sync_source_freshness( &self, diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs index 0aae15d12e..b7169a441e 100644 --- a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -44,6 +44,7 @@ pub(crate) const TRIGGER_PROGRESS: &str = include_str!("../trigger_progress.sql" pub(crate) const CLI_LOGIN_FLOWS: &str = include_str!("../cli_login_flows.sql"); pub(crate) const RETENTION_INDEXES: &str = include_str!("../retention_indexes.sql"); pub(crate) const CUSTOM_ROLES: &str = include_str!("../custom_roles.sql"); +pub(crate) const WAREHOUSE_CONSUMPTION: &str = include_str!("../warehouse_consumption.sql"); pub(crate) const ADDITIONS: &[(&str, &str)] = &[ ("location_instances", LOCATION_INSTANCES), @@ -80,6 +81,7 @@ pub(crate) const ADDITIONS: &[(&str, &str)] = &[ ("cli_login_flows", CLI_LOGIN_FLOWS), ("retention_indexes", RETENTION_INDEXES), ("custom_roles", CUSTOM_ROLES), + ("warehouse_consumption", WAREHOUSE_CONSUMPTION), ]; pub(crate) fn migration_digests(store: &ControlStore) -> Vec { diff --git a/crates/sqlbuild-orchestration/src/store/warehouse_consumption.sql b/crates/sqlbuild-orchestration/src/store/warehouse_consumption.sql new file mode 100644 index 0000000000..08e4db7f24 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/warehouse_consumption.sql @@ -0,0 +1,25 @@ +CREATE TABLE IF NOT EXISTS sqb_warehouse_consumption ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at {time} NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + execution_id TEXT NOT NULL, + phase TEXT NOT NULL, + run_id TEXT NOT NULL, + node_name TEXT NOT NULL, + target_database TEXT, + target_schema TEXT NOT NULL, + target_name TEXT NOT NULL, + version_hash TEXT NOT NULL, + input_count BIGINT NOT NULL, + input_kind TEXT, + input_name TEXT, + input_version TEXT, + record_json {json} NOT NULL, + received_at {time} NOT NULL +); +CREATE INDEX IF NOT EXISTS sqb_consumption_reconciliation ON sqb_warehouse_consumption (location_id,target,state_database,state_schema,observed_at,event_id); +CREATE INDEX IF NOT EXISTS sqb_consumption_relation ON sqb_warehouse_consumption (location_id,target_schema,target_name,observed_at); diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_consumption.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_consumption.rs new file mode 100644 index 0000000000..890057a4a2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_consumption.rs @@ -0,0 +1,94 @@ +//! Mirror consumed-input history for inspection; run decisions never read this mirror. + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::events::ingest_events; +use crate::warehouse_metadata::_helpers::ingestion::validate_fields; +use crate::warehouse_metadata::models::{ + ConsumptionBatch, StateEventImport, StateEventsImport, WarehouseHistoryKind, + WarehouseStateLocation, WarehouseSyncResponse, +}; + +pub async fn ingest_consumption( + store: &ControlStore, + scope: (Uuid, &str), + batch: &ConsumptionBatch, +) -> Result { + validate_fields( + ("consumption", batch.generation, batch.records.len()), + false, + false, + )?; + ingest_events( + store, + scope, + StateEventsImport { + kind: WarehouseHistoryKind::Consumption, + session_id: batch.session_id, + generation: batch.generation, + invalid_identity: batch + .records + .iter() + .flat_map(|record| { + [ + &record.event_id, + &record.state_schema, + &record.execution_id, + &record.phase, + &record.node_name, + &record.target_schema, + &record.target_name, + ] + }) + .any(|value| value.trim().is_empty()), + records: batch + .records + .iter() + .map(|record| StateEventImport { + source_id: record.event_id.clone(), + location: WarehouseStateLocation { + database: record.state_database.clone(), + schema: record.state_schema.clone(), + }, + observed_at: record.recorded_at, + columns: vec![ + ("execution_id", Parameter::Text(record.execution_id.clone())), + ("phase", Parameter::Text(record.phase.clone())), + ("run_id", Parameter::Text(record.run_id.clone())), + ("node_name", Parameter::Text(record.node_name.clone())), + ( + "target_database", + Parameter::OptionalText(record.target_database.clone()), + ), + ( + "target_schema", + Parameter::Text(record.target_schema.clone()), + ), + ("target_name", Parameter::Text(record.target_name.clone())), + ("version_hash", Parameter::Text(record.version_hash.clone())), + ("input_count", Parameter::Integer(record.input_count)), + ( + "input_kind", + Parameter::OptionalText(record.input_kind.clone()), + ), + ( + "input_name", + Parameter::OptionalText(record.input_name.clone()), + ), + ( + "input_version", + Parameter::OptionalText(record.input_version.clone()), + ), + ], + record_json: json!(record), + }) + .collect(), + reconciliation: batch.reconciliation.as_ref(), + payload: json!(batch), + }, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index 8fac735025..f5f60dcd9d 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod bootstrap_source_observations; pub mod ingest_audit_results; pub mod ingest_catalog; +pub mod ingest_consumption; pub mod ingest_fingerprints; pub mod ingest_janitor_events; pub mod ingest_microbatches; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 8ea8b3988e..5565b4907a 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -271,6 +271,39 @@ pub struct MicrobatchesBatch { pub reconciliation: Option, } +/// What one execution consumed, as history; `--changed` reads the warehouse table, never this mirror. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseConsumptionRecord { + pub state_database: Option, + pub state_schema: String, + pub event_id: String, + pub execution_id: String, + pub phase: String, + pub run_id: String, + pub node_type: String, + pub node_name: String, + pub target_database: Option, + pub target_schema: String, + pub target_name: String, + pub version_hash: String, + pub input_count: i64, + pub input_kind: Option, + pub input_name: Option, + pub input_version: Option, + pub recorded_at: DateTime, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct ConsumptionBatch { + pub session_id: Uuid, + pub generation: i64, + pub records: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reconciliation: Option, +} + /// Physical namespace of SQLBuild's source-freshness state table, never a source query. #[derive( Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, utoipa::ToSchema, @@ -292,12 +325,13 @@ pub enum WarehouseHistoryKind { Migrations, OldNameViews, Microbatches, + Consumption, Catalog, Statistics, } impl WarehouseHistoryKind { - pub const ALL: [Self; 9] = [ + pub const ALL: [Self; 10] = [ Self::Fingerprints, Self::SourceFreshness, Self::AuditResults, @@ -305,6 +339,7 @@ impl WarehouseHistoryKind { Self::Migrations, Self::OldNameViews, Self::Microbatches, + Self::Consumption, Self::Catalog, Self::Statistics, ]; @@ -319,6 +354,7 @@ impl WarehouseHistoryKind { Self::Migrations => "migrations", Self::OldNameViews => "old_name_views", Self::Microbatches => "microbatches", + Self::Consumption => "consumption", Self::Catalog => "catalog", Self::Statistics => "statistics", } @@ -334,6 +370,7 @@ impl WarehouseHistoryKind { Self::Migrations => "sqb_warehouse_migrations", Self::OldNameViews => "sqb_warehouse_old_name_views", Self::Microbatches => "sqb_warehouse_microbatches", + Self::Consumption => "sqb_warehouse_consumption", Self::Catalog => "sqb_warehouse_catalog", Self::Statistics => "sqb_warehouse_statistics", } @@ -451,6 +488,7 @@ pub enum WarehouseHistoryRead { Migrations(StateEventsRead), OldNameViews(StateEventsRead), Microbatches(StateEventsRead), + Consumption(StateEventsRead), Catalog(CatalogRead), Statistics(StatisticsRead), } @@ -465,6 +503,7 @@ pub enum WarehouseHistoryRecord { Migration(Box), OldNameView(Box), Microbatch(Box), + Consumption(Box), Catalog(WarehouseCatalogRecord), Statistics(WarehouseStatisticsRecord), } diff --git a/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs index 1ad9760fca..e67beec5b8 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_external_sessions.rs @@ -94,6 +94,7 @@ fn given_federated_people_when_administrators_and_sessions_change_then_live_auth .expect("populated predecessor"); for sql in [ "DROP TABLE sqb_external_sessions", + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_consumption'", "DELETE FROM sqb_schema_migrations WHERE name='custom_roles'", "DELETE FROM sqb_schema_migrations WHERE name='retention_indexes'", "DELETE FROM sqb_schema_migrations WHERE name='external_sessions'", diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index 8ab85c3b95..453c5e41d7 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -348,6 +348,11 @@ fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicit "microbatch_history", "event_id", ), + ( + WarehouseHistoryKind::Consumption, + "consumption_history", + "event_id", + ), ] { crate::api::tests::warehouse_events_helpers::event_history( store, diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs index 5cdb5276a7..38bfa64c85 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit.rs @@ -139,6 +139,7 @@ fn given_security_audit_pages_when_clock_and_authority_change_then_commit_order_ .expect("predecessor"); for sql in [ "DROP TABLE sqb_warehouse_catalog", + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_consumption'", "DELETE FROM sqb_schema_migrations WHERE name='custom_roles'", "DELETE FROM sqb_schema_migrations WHERE name='retention_indexes'", "DELETE FROM sqb_schema_migrations WHERE name='warehouse_catalog'", diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs index 87a83c4487..89666c4294 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_sources.rs @@ -175,6 +175,7 @@ fn given_imported_source_versions_when_consuming_then_late_facts_queue_once_and_ .expect("pre-sensor predecessor"); for sql in [ "DROP TABLE sqb_warehouse_catalog", + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_consumption'", "DELETE FROM sqb_schema_migrations WHERE name='custom_roles'", "DELETE FROM sqb_schema_migrations WHERE name='retention_indexes'", "DELETE FROM sqb_schema_migrations WHERE name='warehouse_catalog'", diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs index baa49874ed..e1d893e6d2 100644 --- a/crates/sqlbuild-orchestration/tests/store/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -92,6 +92,7 @@ pub(crate) async fn reconstruct_fingerprint_predecessor(store: &ControlStore) { .await .expect("fingerprint predecessor"); for sql in [ + "DELETE FROM sqb_schema_migrations WHERE name='warehouse_consumption'", "DELETE FROM sqb_schema_migrations WHERE name='custom_roles'", "DELETE FROM sqb_schema_migrations WHERE name='retention_indexes'", "DROP TABLE sqb_cli_login_flows", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index e92e997318..824b6119f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -31,10 +31,14 @@ use crate::bindings::_helpers::check::check_core::{ use crate::bindings::_helpers::check::check_results::write_check_results; use crate::bindings::_helpers::command_runtime::warehouses::describe_connection_warehouse; use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; +use crate::bindings::_helpers::consumption::recorder::{ + ConsumptionRecorder, Prepared, prepare as prepare_consumption, run_reason_text, +}; use crate::bindings::_helpers::cost_telemetry::context::CostContext; use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timestamp}; use crate::bindings::_helpers::freshness::publication::append_freshness_records; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; +use crate::bindings::_helpers::native_host::consumption_state::recorder_callbacks; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::results::ReadResult; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; @@ -370,6 +374,21 @@ pub(crate) fn execute_build_plan<'py>( ), ], )?; + let prepared: Prepared<'py> = prepare_consumption( + (&plan, &project), + (&adapter, &connection_config), + ( + context.request_flag("changed")?, + context.request_flag("full_refresh")?, + ), + ( + native_field(&project, "run_id")?.str()?.extract()?, + native_field(&preparation.lifecycle, "on_node_complete")?, + ), + )?; + report_changed_decision(context, &prepared)?; + let recorder: &Bound<'py, ConsumptionRecorder> = &prepared.recorder; + let (on_node_complete, before_node_write) = recorder_callbacks(recorder.as_any())?; let connection: &Bound<'py, PyAny> = &preparation.connection_progress; let build_callbacks: Bound<'py, PyAny> = call_with( &host::entry(py, ExecutorHost::BuildCallbacks)?, @@ -379,10 +398,8 @@ pub(crate) fn execute_build_plan<'py>( "on_node_start", keyword_relay(native_field(callbacks, "on_node_start")?, &NODE_START)?, ), - ( - "on_node_complete", - native_field(&preparation.lifecycle, "on_node_complete")?, - ), + ("on_node_complete", on_node_complete), + ("before_node_write", before_node_write), ( "on_sub_progress", native_field(callbacks, "on_sub_progress")?, @@ -429,7 +446,10 @@ pub(crate) fn execute_build_plan<'py>( let initial_state: Bound<'py, PyAny> = call_with( &host::entry(py, ExecutorHost::BuildInitialState)?, &[ - ("precompleted_keys", lifecycle.precompleted_keys(py)?), + ( + "precompleted_keys", + with_skipped_keys(&plan, lifecycle.precompleted_keys(py)?, &prepared)?, + ), ("initial_load_results", lifecycle.ingress_load_results(py)), ("initial_failed_keys", lifecycle.blocked_keys(py)?), ], @@ -447,6 +467,9 @@ pub(crate) fn execute_build_plan<'py>( ("initial_state", initial_state), ], )?; + for warning in recorder.get().finish(py, &result)? { + emit(&context.stream, &format!("warning: {warning}\n"))?; + } let native_result = BuildExecutionResult::read(&result)?; let successful_models: Vec = native_result .model_results @@ -468,6 +491,74 @@ pub(crate) fn execute_build_plan<'py>( }) } +/// Report what `--changed` runs and skips, and any warnings from reading records. +fn report_changed_decision(context: &BuildContext<'_>, prepared: &Prepared<'_>) -> PyResult<()> { + for warning in &prepared.warnings { + emit(&context.stream, &format!("warning: {warning}\n"))?; + } + let Some(decision) = &prepared.decision else { + return Ok(()); + }; + if let Some(worker) = &context.worker { + let skipped: Vec = decision + .skipped + .iter() + .map(|key| format!("{}:{}", key.kind, key.name)) + .collect(); + worker.unchanged(context.py(), &skipped)?; + } + emit( + &context.stream, + &format!( + "Changed: {} of {} selected models and seeds run; {} unchanged are skipped.\n", + decision.run, + decision.run + decision.skipped.len(), + decision.skipped.len() + ), + )?; + if context.request_flag("verbose")? { + for (key, reason) in &decision.reasons { + emit( + &context.stream, + &format!( + " runs {} {}: {}\n", + key.kind, + key.name, + run_reason_text(reason) + ), + )?; + } + } + Ok(()) +} + +/// `precompleted` with the plan keys of the models and seeds `--changed` skips. +fn with_skipped_keys<'py>( + plan: &Bound<'py, PyAny>, + precompleted: Bound<'py, PyAny>, + prepared: &Prepared<'py>, +) -> PyResult> { + let Some(decision) = &prepared.decision else { + return Ok(precompleted); + }; + let py: Python<'py> = plan.py(); + let mut keys: Vec> = precompleted.try_iter()?.collect::>()?; + for (entries, kind) in [("model_entries", "model"), ("seed_entries", "seed")] { + for entry in native_field(plan, entries)?.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + let name: String = native_field(&entry, "name")?.extract()?; + if decision + .skipped + .iter() + .any(|key| key.kind == kind && key.name == name) + { + keys.push(native_field(&entry, "key")?); + } + } + } + frozenset(py, &keys) +} + fn check_runtime<'py>( context: &BuildContext<'py>, pipeline: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 7c15b912eb..21e77b8f61 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -28,6 +28,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { janitor::registration::register(module)?; plan_command::registration::register(module)?; freshness::registration::register(module)?; + crate::bindings::_helpers::native_host::consumption_state::register(module)?; clone::registration::register(module)?; compute_logs::registration::register(module)?; execution_history::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index ded2b6c2b6..93451ffa40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -93,6 +93,14 @@ impl WorkerBuild { Ok(()) } + /// Nodes `build --changed` leaves unbuilt are up to date, as a planner `no_change` skip is. + pub(crate) fn unchanged(&self, py: Python<'_>, keys: &[String]) -> PyResult<()> { + for key in keys { + self.complete(py, key, (NodeOutcome::Success, None, None))?; + } + Ok(()) + } + fn complete( &self, py: Python<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs new file mode 100644 index 0000000000..61e987e7e1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs @@ -0,0 +1,393 @@ +//! The facts `--changed` and the consumption recorder read from the plan and compiled project. + +use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_core::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; +use sqlbuild_state::consumption::models::{ForcedRun, InputFact, NodeFacts, NodeKey, RelationKey}; + +use crate::bindings::_helpers::compiled_views::{ + function_reads, model_reads, project_reads, seed_reads, +}; +use crate::bindings::_helpers::consumption::relation_reads::{Locations, sql_reads_undeclared}; +use crate::bindings::_helpers::planner_identity::hashing::{graph_key, model_query_hash}; +use crate::bindings::_helpers::planner_identity::identities::hook_version_hashes; +use crate::bindings::_helpers::planner_identity::metadata::{ + function_local_hashes, model_version_identity_metadata_json, seed_identity, +}; +use crate::bindings::_helpers::records::fields::field as native_field; + +const MODEL: &str = "model"; +const SEED: &str = "seed"; +const SOURCE: &str = "source"; +const FUNCTION_KINDS: [&str; 2] = ["udf", "table_fn"]; +/// Plan reasons that mean the planner sees no change of the model itself. +const UNCHANGED_REASONS: [&str; 2] = ["no_change", "normal_incremental"]; +const FIRST_RUN: &str = "first_run"; +const FULL_REFRESH: &str = "full_refresh"; +const SKIP_ACTION: &str = "skip"; +const SQL_LANGUAGE: &str = "sql"; +const UDF: &str = "udf"; + +/// The selection's nodes and their ancestry, and the selected ones that execute. +pub(crate) struct BuildFacts { + pub(crate) nodes: Vec, + pub(crate) runnable: BTreeSet, +} + +/// One compiled model or seed of the ancestry walk. +enum Compiled<'py> { + Model(Bound<'py, PyAny>), + Seed(Bound<'py, PyAny>), +} + +fn node_key(kind: &str, name: &str) -> NodeKey { + NodeKey { + kind: kind.to_owned(), + name: name.to_owned(), + } +} + +fn enum_text(value: &Bound<'_, PyAny>) -> PyResult { + let value: Bound<'_, PyAny> = value.getattr_opt("value")?.unwrap_or_else(|| value.clone()); + value.str()?.extract() +} + +fn relation(destination: &Bound<'_, PyAny>) -> PyResult> { + let schema: Bound<'_, PyAny> = native_field(destination, "schema")?; + if schema.is_none() { + return Ok(None); + } + let database: Bound<'_, PyAny> = native_field(destination, "database")?; + Ok(Some(RelationKey { + database: if database.is_none() { + None + } else { + Some(database.extract()?) + }, + schema: schema.extract()?, + name: native_field(destination, "name")?.extract()?, + })) +} + +/// Each observed source's `data_version_hash`; sources loaded in this build are left out. +fn source_versions(plan: &Bound<'_, PyAny>) -> PyResult>> { + let mut versions: HashMap> = HashMap::new(); + let freshness: Bound<'_, PyAny> = native_field(plan, "source_freshness")?; + if !freshness.is_none() { + for record in native_field(&freshness, "observed_records")?.try_iter()? { + let record: Bound<'_, PyAny> = record?; + let name: String = native_field(&record, "source_name")?.extract()?; + let hash: String = native_field(&record, "data_version_hash")?.extract()?; + let entry = versions.entry(name).or_insert_with(|| Some(hash.clone())); + if entry.as_deref() != Some(hash.as_str()) { + *entry = None; + } + } + } + for load in native_field(plan, "source_load_entries")?.try_iter()? { + versions.insert(native_field(&load?, "name")?.extract()?, None); + } + Ok(versions) +} + +/// The direct inputs of a node, through functions to what they read. +fn inputs( + deps: Vec<(String, String)>, + function_deps: &HashMap<(String, String), Vec<(String, String)>>, + sources: &HashMap>, +) -> Vec { + let mut seen: BTreeSet<(String, String)> = BTreeSet::new(); + let mut pending: VecDeque<(String, String)> = deps.into(); + let mut facts: Vec = Vec::new(); + while let Some((kind, name)) = pending.pop_front() { + if !seen.insert((kind.clone(), name.clone())) { + continue; + } + match kind.as_str() { + MODEL | SEED => facts.push(InputFact::Node(node_key(&kind, &name))), + SOURCE => facts.push(InputFact::Source { + version: sources.get(&name).cloned().flatten(), + name, + }), + function if FUNCTION_KINDS.contains(&function) => pending.extend( + function_deps + .get(&(kind.clone(), name.clone())) + .cloned() + .unwrap_or_default(), + ), + _ => facts.push(InputFact::Opaque(format!("{kind}:{name}"))), + } + } + facts +} + +/// Checks that functions a model calls, transitively, read only their declared inputs. +struct FunctionReads<'a> { + /// Each function's SQL to check, or `None` for a Python function, which reads no relation. + bodies: &'a HashMap<(String, String), Option>, + function_deps: &'a HashMap<(String, String), Vec<(String, String)>>, + sources: &'a HashMap>, + locations: &'a Locations, + dialect: Option<&'a str>, + checked: HashMap<(String, String), bool>, +} + +impl FunctionReads<'_> { + fn undeclared(&mut self, py: Python<'_>, deps: &[(String, String)]) -> PyResult { + let mut pending: VecDeque<(String, String)> = deps + .iter() + .filter(|(kind, _)| FUNCTION_KINDS.contains(&kind.as_str())) + .cloned() + .collect(); + let mut seen: BTreeSet<(String, String)> = BTreeSet::new(); + while let Some(key) = pending.pop_front() { + if !seen.insert(key.clone()) { + continue; + } + let own_deps: Vec<(String, String)> = + self.function_deps.get(&key).cloned().unwrap_or_default(); + if self.function_undeclared(py, &key, &own_deps)? { + return Ok(true); + } + pending.extend( + own_deps + .into_iter() + .filter(|(kind, _)| FUNCTION_KINDS.contains(&kind.as_str())), + ); + } + Ok(false) + } + + fn function_undeclared( + &mut self, + py: Python<'_>, + key: &(String, String), + own_deps: &[(String, String)], + ) -> PyResult { + if let Some(undeclared) = self.checked.get(key) { + return Ok(*undeclared); + } + let undeclared: bool = match self.bodies.get(key) { + None => true, + Some(None) => false, + Some(Some(sql)) => { + let allowed: Vec = + inputs(own_deps.to_vec(), self.function_deps, self.sources); + sql_reads_undeclared( + py, + sql.clone(), + (&allowed, None), + (self.locations, self.dialect), + )? + } + }; + self.checked.insert(key.clone(), undeclared); + Ok(undeclared) + } +} + +fn key_rows(keys: &Bound<'_, PyAny>) -> PyResult> { + keys.try_iter()? + .map(|key| { + let key = graph_key(&key?)?; + Ok((key.kind, key.name)) + }) + .collect() +} + +/// Why the planner runs a selected entry regardless of its records. +fn entry_forced(entry: &Bound<'_, PyAny>, full_refresh: bool) -> PyResult> { + let mut forced: Vec = Vec::new(); + if full_refresh { + forced.push(ForcedRun::FullRefresh); + } + match enum_text(&native_field(entry, "reason")?)?.as_str() { + FIRST_RUN => forced.push(ForcedRun::RelationMissing), + FULL_REFRESH => forced.push(ForcedRun::FullRefresh), + reason if UNCHANGED_REASONS.contains(&reason) => {} + reason => forced.push(ForcedRun::PlannerChange(reason.to_owned())), + } + for field in ["start_cursor_override", "end_cursor_override"] { + if entry.hasattr(field)? && !native_field(entry, field)?.is_none() { + forced.push(ForcedRun::CursorInterval); + break; + } + } + Ok(forced) +} + +/// Facts of the selected models and seeds and of every model and seed they read from. +pub(crate) fn build_facts( + (plan, project): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (changed, full_refresh): (bool, bool), +) -> PyResult { + let py: Python<'_> = plan.py(); + let mut compiled: HashMap> = HashMap::new(); + for model in project_reads::models(project)?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + compiled.insert( + node_key(MODEL, &model_reads::model_name(&model)?), + Compiled::Model(model), + ); + } + for seed in project_reads::seeds(project)?.try_iter()? { + let seed: Bound<'_, PyAny> = seed?; + compiled.insert( + node_key(SEED, &seed_reads::name(&seed)?.extract::()?), + Compiled::Seed(seed), + ); + } + let mut function_deps: HashMap<(String, String), Vec<(String, String)>> = HashMap::new(); + let mut function_bodies: HashMap<(String, String), Option> = HashMap::new(); + let functions: Bound<'_, PyAny> = project_reads::functions(project)?; + for function in functions.try_iter()? { + let function: Bound<'_, PyAny> = function?; + let key = graph_key(&function_reads::key(&function)?)?; + function_deps.insert( + (key.kind.clone(), key.name.clone()), + key_rows(&function_reads::deps(&function)?)?, + ); + let body: Option = + if enum_text(&function_reads::language(&function)?)? == SQL_LANGUAGE { + let body: String = function_reads::body_sql(&function)?.extract()?; + Some(if key.kind == UDF { + format!("SELECT ({body})") + } else { + body + }) + } else { + None + }; + function_bodies.insert((key.kind, key.name), body); + } + let mut selected: BTreeMap> = BTreeMap::new(); + let mut runnable: BTreeSet = BTreeSet::new(); + for (entries, kind) in [("model_entries", MODEL), ("seed_entries", SEED)] { + for entry in native_field(plan, entries)?.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + let key: NodeKey = node_key(kind, &native_field(&entry, "name")?.extract::()?); + if enum_text(&native_field(&entry, "action")?)? != SKIP_ACTION { + runnable.insert(key.clone()); + } + let forced: Vec = if kind == MODEL { + entry_forced(&entry, full_refresh)? + } else { + seed_forced(&entry, full_refresh)? + }; + selected.insert(key, forced); + } + } + let sources: HashMap> = source_versions(plan)?; + let existing: BTreeSet = native_field(plan, "existing_model_names")? + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?; + let dialect: Option = native_field(project, "sql_analysis_dialect")?.extract()?; + let function_hashes: Bound<'_, PyDict> = function_local_hashes(&functions, dialect.as_deref())?; + let hooks: Bound<'_, PyAny> = native_field(project, "hook_functions")?; + let hook_hashes: Bound<'_, PyDict> = hook_version_hashes(&hooks)?; + let locations: Option = if changed { + Some(Locations::read(plan, project)?) + } else { + None + }; + let mut function_checks: Option> = + locations.as_ref().map(|locations| FunctionReads { + bodies: &function_bodies, + function_deps: &function_deps, + sources: &sources, + locations, + dialect: dialect.as_deref(), + checked: HashMap::new(), + }); + let mut nodes: Vec = Vec::new(); + let mut pending: VecDeque = selected.keys().cloned().collect(); + let mut visited: BTreeSet = BTreeSet::new(); + while let Some(key) = pending.pop_front() { + if !visited.insert(key.clone()) { + continue; + } + let Some(node) = compiled.get(&key) else { + continue; + }; + let (destination, deps, version_hash) = match node { + Compiled::Model(model) => ( + model_reads::destination(model)?, + model_reads::dep_rows(model)?, + model_local_identity_hash( + &model_query_hash( + py, + &key.name, + ( + &model_reads::query_sql(model)?.extract::()?, + dialect.as_deref(), + ), + )?, + &model_version_identity_metadata_json( + model, + Some(function_hashes.as_any()), + Some(hook_hashes.as_any()), + )?, + ), + ), + Compiled::Seed(seed) => ( + seed_reads::destination(seed)?, + Vec::new(), + seed_identity(seed)?.0, + ), + }; + let node_inputs: Vec = inputs(deps.clone(), &function_deps, &sources); + for input in &node_inputs { + if let InputFact::Node(upstream) = input { + pending.push_back(upstream.clone()); + } + } + let mut forced: Vec = match selected.get(&key) { + Some(forced) => forced.clone(), + None if key.kind == MODEL && !existing.contains(&key.name) => { + vec![ForcedRun::RelationMissing] + } + None => Vec::new(), + }; + let relation: Option = relation(&destination)?; + if let (Compiled::Model(model), Some(locations), Some(function_checks)) = + (node, &locations, &mut function_checks) + && (sql_reads_undeclared( + py, + model_reads::query_sql(model)?.extract()?, + (&node_inputs, relation.as_ref()), + (locations, dialect.as_deref()), + )? || function_checks.undeclared(py, &deps)?) + { + forced.push(ForcedRun::InputsNotEnumerable); + } + let Some(relation) = relation else { + continue; + }; + nodes.push(NodeFacts { + selected: selected.contains_key(&key), + key, + relation, + version_hash, + forced, + inputs: node_inputs, + }); + } + runnable.retain(|key| nodes.iter().any(|node| node.key == *key)); + Ok(BuildFacts { nodes, runnable }) +} + +fn seed_forced(entry: &Bound<'_, PyAny>, full_refresh: bool) -> PyResult> { + let mut forced: Vec = Vec::new(); + if full_refresh { + forced.push(ForcedRun::FullRefresh); + } + match enum_text(&native_field(entry, "reason")?)?.as_str() { + FIRST_RUN => forced.push(ForcedRun::RelationMissing), + reason if UNCHANGED_REASONS.contains(&reason) => {} + reason => forced.push(ForcedRun::PlannerChange(reason.to_owned())), + } + Ok(forced) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/mod.rs new file mode 100644 index 0000000000..1e68918e23 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod facts; +pub(crate) mod recorder; +pub(crate) mod relation_reads; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs new file mode 100644 index 0000000000..89389b1be8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs @@ -0,0 +1,388 @@ +//! Record what each model and seed execution consumed, and decide `--changed` skips, around one build. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_state::consumption::constants::RECENT_EXECUTIONS; +use sqlbuild_state::consumption::main::changed_run_set::changed_run_set; +use sqlbuild_state::consumption::main::consumed_inputs::consumed_inputs; +use sqlbuild_state::consumption::main::consumption_history::consumption_history; +use sqlbuild_state::consumption::main::started_append::started_append; +use sqlbuild_state::consumption::main::succeeded_appends::succeeded_appends; +use sqlbuild_state::consumption::models::{ + ConsumptionAppend, ConsumptionHistory, ConsumptionRow, ForcedRun, InputFact, NodeFacts, + NodeKey, RunReason, +}; +use uuid::Uuid; + +use crate::bindings::_helpers::consumption::facts::{BuildFacts, build_facts}; +use crate::bindings::_helpers::native_host::consumption_state::{ + ConsumptionConnection, Location, NodeCompleteHook, +}; +use crate::bindings::_helpers::records::fields::field as native_field; + +/// Succeeded rows are flushed once this many executions are pending. +const FLUSH_EXECUTIONS: usize = 100; +/// Or once this long has passed since the last flush. +const FLUSH_INTERVAL: Duration = Duration::from_secs(30); +const MODEL_RESULT: &str = "model_name"; +const SEED_RESULT: &str = "seed_name"; +const SUCCESS: &str = "success"; +const ERROR_OUTCOME: &str = "error"; +const SEED_KIND: &str = "seed"; + +/// The mutable side of one build's recording. +struct Recording { + pending: BTreeMap>, + pending_executions: usize, + pending_seeds: Vec, + tables: BTreeSet, + last_flush: Instant, + warnings: Vec, +} + +impl Default for Recording { + fn default() -> Self { + Self { + pending: BTreeMap::new(), + pending_executions: 0, + pending_seeds: Vec::new(), + tables: BTreeSet::new(), + last_flush: Instant::now(), + warnings: Vec::new(), + } + } +} + +/// Appends `started` rows before a build and `succeeded` rows as its models and seeds complete. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct ConsumptionRecorder { + connection: ConsumptionConnection, + run_id: String, + nodes: HashMap, + history: ConsumptionHistory, + executions: HashMap, + failed_start: HashMap, + base_on_node_complete: NodeCompleteHook, + recording: Mutex, +} + +/// What `--changed` decided, for the build's report. +pub(crate) struct ChangedDecision { + pub(crate) run: usize, + pub(crate) skipped: Vec, + pub(crate) reasons: Vec<(NodeKey, RunReason)>, +} + +fn kind_of(result: &Bound<'_, PyAny>) -> PyResult> { + for (field, kind) in [(MODEL_RESULT, "model"), (SEED_RESULT, "seed")] { + if let Some(name) = result.getattr_opt(field)? { + return Ok(Some((kind, name.extract()?))); + } + } + Ok(None) +} + +fn status_text(value: &Bound<'_, PyAny>) -> PyResult { + let value: Bound<'_, PyAny> = value.getattr_opt("value")?.unwrap_or_else(|| value.clone()); + Ok(value.str()?.extract::()?.to_ascii_lowercase()) +} + +impl ConsumptionRecorder { + fn lock(&self) -> PyResult> { + self.recording + .lock() + .map_err(|_| PyRuntimeError::new_err("consumption recorder state is poisoned")) + } + + /// The succeeded rows of one execution that wrote its relation, with their schema. + fn succeeded(&self, key: &NodeKey) -> Option<(Location, Vec)> { + let (Some(node), Some(execution_id)) = (self.nodes.get(key), self.executions.get(key)) + else { + return None; + }; + let inputs = consumed_inputs(node, (&self.nodes, &self.history), &self.executions); + Some(( + Location::of(&node.relation), + succeeded_appends(node, (execution_id, &self.run_id), &inputs), + )) + } + + /// The recording with `key`'s succeeded rows queued. + fn queued(&self, mut recording: Recording, key: &NodeKey) -> Recording { + if let Some((location, rows)) = self.succeeded(key) { + recording.pending.entry(location).or_default().extend(rows); + recording.pending_executions += 1; + } + recording + } + + /// The recording after appending every pending row; a failure is reported, and its executions run again. + fn flushed(&self, py: Python<'_>, mut recording: Recording) -> Recording { + let pending = std::mem::take(&mut recording.pending); + let wrote: bool = !pending.is_empty(); + for (location, rows) in pending { + let created: bool = !recording.tables.insert(location.clone()); + let written: PyResult<()> = (if created { + Ok(()) + } else { + self.connection.ensure_table(py, &location) + }) + .and_then(|()| self.connection.append(py, &location, &rows)); + if let Err(error) = written { + recording.warnings.push(format!( + "could not record consumed inputs in schema {}: {error}; those models and seeds run again under --changed", + location.schema + )); + } + } + if wrote && let Err(error) = self.connection.close(py) { + recording.warnings.push(format!( + "could not close the consumed-input connection: {error}" + )); + } + recording.pending_executions = 0; + recording.last_flush = Instant::now(); + recording + } + + /// Queue the seeds whose error audits passed and flush everything. + pub(crate) fn finish( + &self, + py: Python<'_>, + build_result: &Bound<'_, PyAny>, + ) -> PyResult> { + let mut failed_audits: BTreeSet = BTreeSet::new(); + for audit in native_field(build_result, "source_audit_results")?.try_iter()? { + let audit: Bound<'_, PyAny> = audit?; + let target: Bound<'_, PyAny> = native_field(&audit, "attached_target_name")?; + if !target.is_none() && status_text(&native_field(&audit, "outcome")?)? == ERROR_OUTCOME + { + failed_audits.insert(target.extract()?); + } + } + let mut recording = self.lock()?; + for seed in std::mem::take(&mut recording.pending_seeds) { + if !failed_audits.contains(&seed.name) { + *recording = self.queued(std::mem::take(&mut *recording), &seed); + } + } + *recording = self.flushed(py, std::mem::take(&mut *recording)); + Ok(std::mem::take(&mut recording.warnings)) + } +} + +#[pymethods] +impl ConsumptionRecorder { + /// Forward a finished node, then queue a successful model, or a seed until its audits are known. + fn on_node_complete(&self, py: Python<'_>, node_result: Bound<'_, PyAny>) -> PyResult<()> { + self.base_on_node_complete.forward(&node_result)?; + let Some((kind, name)) = kind_of(&node_result)? else { + return Ok(()); + }; + if status_text(&native_field(&node_result, "status")?)? != SUCCESS { + return Ok(()); + } + let key = NodeKey { + kind: kind.to_owned(), + name, + }; + let mut recording = self.lock()?; + if kind == SEED_KIND { + recording.pending_seeds.push(key); + return Ok(()); + } + *recording = self.queued(std::mem::take(&mut *recording), &key); + if recording.pending_executions >= FLUSH_EXECUTIONS + || recording.last_flush.elapsed() >= FLUSH_INTERVAL + { + *recording = self.flushed(py, std::mem::take(&mut *recording)); + } + Ok(()) + } + + /// Fail a model or seed whose `started` row could not be appended, before it writes. + #[pyo3(signature = (*, entry, connection))] + fn before_node_write( + &self, + entry: Bound<'_, PyAny>, + connection: Bound<'_, PyAny>, + ) -> PyResult<()> { + let _ = connection; + let name: String = native_field(&entry, "name")?.extract()?; + let kind: &str = if entry.hasattr("csv_settings")? { + "seed" + } else { + "model" + }; + let key = NodeKey { + kind: kind.to_owned(), + name, + }; + match self.failed_start.get(&key) { + Some(error) => Err(PyRuntimeError::new_err(format!( + "not written: SQLBuild could not record that this {kind} started ({error}), so a crash could not be detected later" + ))), + None => Ok(()), + } + } +} + +/// Schemas to read records from: all under `--changed`, else those of unselected upstreams. +fn read_locations(facts: &BuildFacts, changed: bool) -> BTreeSet { + let by_key: HashMap<&NodeKey, &NodeFacts> = + facts.nodes.iter().map(|node| (&node.key, node)).collect(); + if changed { + return facts + .nodes + .iter() + .map(|node| Location::of(&node.relation)) + .collect(); + } + facts + .nodes + .iter() + .filter(|node| facts.runnable.contains(&node.key)) + .flat_map(|node| node.inputs.iter()) + .filter_map(|input| match input { + InputFact::Node(upstream) if !facts.runnable.contains(upstream) => by_key.get(upstream), + _ => None, + }) + .map(|node| Location::of(&node.relation)) + .collect() +} + +/// Everything the build needs from the recorder: the recorder itself and what `--changed` skipped. +pub(crate) struct Prepared<'py> { + pub(crate) recorder: Bound<'py, ConsumptionRecorder>, + pub(crate) decision: Option, + pub(crate) warnings: Vec, +} + +/// Read records, decide `--changed` skips, mint execution IDs and append `started` rows. +pub(crate) fn prepare<'py>( + (plan, project): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (adapter, connection_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (changed, full_refresh): (bool, bool), + (run_id, base_on_node_complete): (String, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = plan.py(); + let facts: BuildFacts = build_facts((plan, project), (changed, full_refresh))?; + let connection = ConsumptionConnection::new(adapter, connection_config); + let mut warnings: Vec = Vec::new(); + let mut rows: Vec = Vec::new(); + for location in read_locations(&facts, changed) { + match connection.read_recent(py, &location) { + Ok(read) => rows.extend(read), + Err(error) => warnings.push(format!( + "could not read consumed inputs in schema {}: {error}; its models and seeds count as changed", + location.schema + )), + } + } + let history: ConsumptionHistory = consumption_history(&rows, RECENT_EXECUTIONS); + let mut skipped: BTreeSet = BTreeSet::new(); + let decision: Option = if changed { + let mut reasons: Vec<(NodeKey, RunReason)> = Vec::new(); + for (key, reason) in changed_run_set(&facts.nodes, &history) { + if !facts.runnable.contains(&key) { + continue; + } + match reason { + Some(reason) => reasons.push((key, reason)), + None => { + skipped.insert(key); + } + } + } + Some(ChangedDecision { + run: reasons.len(), + skipped: skipped.iter().cloned().collect(), + reasons, + }) + } else { + None + }; + let executions: HashMap = facts + .runnable + .iter() + .filter(|key| !skipped.contains(*key)) + .map(|key| (key.clone(), Uuid::now_v7().to_string())) + .collect(); + let nodes: HashMap = facts + .nodes + .into_iter() + .map(|node| (node.key.clone(), node)) + .collect(); + let mut started: BTreeMap> = BTreeMap::new(); + for (key, execution_id) in &executions { + if let Some(node) = nodes.get(key) { + started + .entry(Location::of(&node.relation)) + .or_default() + .push(started_append(node, execution_id, &run_id)); + } + } + let mut failed_start: HashMap = HashMap::new(); + let mut tables: BTreeSet = BTreeSet::new(); + for (location, rows) in started { + let written: PyResult<()> = connection + .ensure_table(py, &location) + .and_then(|()| connection.append(py, &location, &rows)); + match written { + Ok(()) => { + tables.insert(location); + } + Err(error) => { + let message: String = error.to_string(); + for row in rows { + failed_start.insert(row.node, message.clone()); + } + } + } + } + connection.close(py)?; + let recorder = ConsumptionRecorder { + connection, + run_id, + nodes, + history, + executions, + failed_start, + base_on_node_complete: NodeCompleteHook::new(&base_on_node_complete), + recording: Mutex::new(Recording { + tables, + ..Recording::default() + }), + }; + Ok(Prepared { + recorder: Bound::new(py, recorder)?, + decision, + warnings, + }) +} + +/// Why `--changed` runs a node, as one line for verbose output. +pub(crate) fn run_reason_text(reason: &RunReason) -> String { + match reason { + RunReason::Forced(ForcedRun::PlannerChange(kind)) => format!("planner change ({kind})"), + RunReason::Forced(ForcedRun::FullRefresh) => "full refresh".to_owned(), + RunReason::Forced(ForcedRun::RelationMissing) => "relation missing".to_owned(), + RunReason::Forced(ForcedRun::CursorInterval) => "cursor interval requested".to_owned(), + RunReason::Forced(ForcedRun::InputsNotEnumerable) => { + "inputs cannot be enumerated".to_owned() + } + RunReason::NoRecord => "no completed record".to_owned(), + RunReason::Unfinished => "last execution did not finish".to_owned(), + RunReason::AmbiguousOrder => "record order is ambiguous".to_owned(), + RunReason::DefinitionChanged => "definition changed".to_owned(), + RunReason::InputsChanged => "inputs changed".to_owned(), + RunReason::InputChanged(input) => format!("input {input} changed"), + RunReason::InputUnknown(input) => format!("input {input} has no known version"), + RunReason::UpstreamRuns(key) => format!("upstream {} {} runs", key.kind, key.name), + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs new file mode 100644 index 0000000000..ea22765f74 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs @@ -0,0 +1,181 @@ +//! Whether a model's compiled SQL reads a relation that none of its declared inputs writes. + +use std::collections::{BTreeSet, HashMap}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_analysis::auditing::main::replace_reference_calls::replace_reference_calls; +use sqlbuild_state::consumption::models::{InputFact, NodeKey, RelationKey}; + +use crate::bindings::_helpers::analysis::relation_names::{ + RelationRow, extract_sql_relation_names, +}; +use crate::bindings::_helpers::compiled_views::{model_reads, project_reads, seed_reads}; +use crate::bindings::_helpers::records::fields::{field as native_field, mapping_items}; + +/// Function calls, `__udf("name")(...)`, parsed as plain calls; their reads are declared inputs. +const FUNCTION_CALLS: [&str; 2] = ["__udf", "__table_fn"]; + +/// The relations each model, seed and source names, by node key or source name. +pub(crate) struct Locations { + nodes: HashMap, + sources: HashMap, Option, String)>, + /// Every project relation, distinct and lower-cased, by lower-cased relation name. + by_name: HashMap>, +} + +fn optional(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.extract()?)) +} + +fn location(destination: &Bound<'_, PyAny>) -> PyResult { + Ok(RelationKey { + database: optional(native_field(destination, "database")?)?, + schema: optional(native_field(destination, "schema")?)?.unwrap_or_default(), + name: native_field(destination, "name")?.extract()?, + }) +} + +impl Locations { + pub(crate) fn read(plan: &Bound<'_, PyAny>, project: &Bound<'_, PyAny>) -> PyResult { + let mut nodes: HashMap = HashMap::new(); + for model in project_reads::models(project)?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + nodes.insert( + NodeKey { + kind: "model".to_owned(), + name: model_reads::model_name(&model)?, + }, + location(&model_reads::destination(&model)?)?, + ); + } + for seed in project_reads::seeds(project)?.try_iter()? { + let seed: Bound<'_, PyAny> = seed?; + nodes.insert( + NodeKey { + kind: "seed".to_owned(), + name: seed_reads::name(&seed)?.extract()?, + }, + location(&seed_reads::destination(&seed)?)?, + ); + } + let mut sources: HashMap, Option, String)> = HashMap::new(); + for map in ["source_map", "source_read_map"] { + for item in mapping_items(&native_field(plan, map)?)?.try_iter()? { + let (name, entry): (String, Bound<'_, PyAny>) = item?.extract()?; + let table: String = + optional(native_field(&entry, "table")?)?.unwrap_or_else(|| name.clone()); + sources.insert( + name, + ( + optional(native_field(&entry, "database")?)?, + optional(native_field(&entry, "schema")?)?, + table, + ), + ); + } + } + let mut by_name: HashMap> = HashMap::new(); + let source_keys = sources + .values() + .map(|(database, schema, table)| RelationKey { + database: database.clone(), + schema: schema.clone().unwrap_or_default(), + name: table.clone(), + }); + for key in nodes.values().cloned().chain(source_keys) { + let key: RelationKey = RelationKey { + database: key.database.map(|database| database.to_lowercase()), + schema: key.schema.to_lowercase(), + name: key.name.to_lowercase(), + }; + by_name.entry(key.name.clone()).or_default().insert(key); + } + Ok(Self { + nodes, + sources, + by_name, + }) + } + + /// Whether the SQL relation, with the qualifiers it spells, can name several project relations. + fn ambiguous(&self, relation: &RelationRow) -> bool { + self.by_name + .get(&relation.0.to_lowercase()) + .is_some_and(|keys| keys.iter().filter(|key| names(relation, key)).count() > 1) + } + + fn allows(&self, input: &InputFact, relation: &RelationRow) -> bool { + match input { + InputFact::Node(key) => self + .nodes + .get(key) + .is_some_and(|location| names(relation, location)), + InputFact::Source { name, .. } => { + self.sources + .get(name) + .is_some_and(|(database, schema, table)| { + names( + relation, + &RelationKey { + database: database.clone(), + schema: schema.clone().unwrap_or_default(), + name: table.clone(), + }, + ) + }) + } + InputFact::Opaque(_) => false, + } + } +} + +/// Whether the SQL relation, with the qualifiers it spells, can name `location`. +fn names((name, schema, database): &RelationRow, location: &RelationKey) -> bool { + let same = |left: &str, right: &str| left.eq_ignore_ascii_case(right); + same(name, &location.name) + && schema + .as_deref() + .is_none_or(|schema| same(schema, &location.schema)) + && database.as_deref().is_none_or(|database| { + location + .database + .as_deref() + .is_some_and(|known| same(database, known)) + }) +} + +/// True when `sql` cannot be parsed or names an ambiguous relation or one no input writes. +pub(crate) fn sql_reads_undeclared( + py: Python<'_>, + sql: String, + (inputs, own): (&[InputFact], Option<&RelationKey>), + (locations, dialect): (&Locations, Option<&str>), +) -> PyResult { + let sql: String = FUNCTION_CALLS.iter().fold(sql, |sql, function| { + replace_reference_calls(&sql, function, |_| { + Ok::<_, std::convert::Infallible>(Some((*function).to_owned())) + }) + .unwrap_or_else(|never| match never {}) + }); + let Some((relations, temporary)) = extract_sql_relation_names(py, &sql, dialect)? else { + return Ok(true); + }; + for relation in &relations { + let created: bool = temporary + .iter() + .any(|created| created.eq_ignore_ascii_case(&relation.0)); + let declared: bool = inputs.iter().any(|input| locations.allows(input, relation)); + if created { + continue; + } + if locations.ambiguous(relation) + || (!own.is_some_and(|own| names(relation, own)) && !declared) + { + return Ok(true); + } + } + Ok(false) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index 511a468e3d..5366b69604 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -18,10 +18,10 @@ use sqlbuild_core::command_output::constants::{ INVOCATION_CONTEXT_ENV, }; use sqlbuild_core::constants::{ - COLUMN_MIGRATION_TABLE_NAME, CTE_AST_KIND, FINGERPRINT_TABLE_NAME, MIGRATION_TABLE_NAME, - OLD_NAME_VIEW_TABLE_NAME, PROJECT_CONFIG_FILENAME, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, - SETTINGS_SECTION, SOURCE_FRESHNESS_TABLE_NAME, STAGED_PROMOTION_MODE, - TABLE_PROMOTION_MODE_SETTING_KEY, ZERO_DAY_DURATION, + COLUMN_MIGRATION_TABLE_NAME, CONSUMPTION_TABLE_NAME, CTE_AST_KIND, FINGERPRINT_TABLE_NAME, + MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, PROJECT_CONFIG_FILENAME, + PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, SETTINGS_SECTION, SOURCE_FRESHNESS_TABLE_NAME, + STAGED_PROMOTION_MODE, TABLE_PROMOTION_MODE_SETTING_KEY, ZERO_DAY_DURATION, }; use sqlbuild_core::diagnostic_routing::constants::{ SQL_DIGEST_FIELD, SQL_PRIVATE_RECORD_FIELD, SQL_TEXT_FIELD, @@ -84,6 +84,7 @@ pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ ("INTEGRATION_RESULT_PATH_ENV", RESULT_PATH_ENV), ("FINGERPRINT_TABLE_NAME", FINGERPRINT_TABLE_NAME), ("SOURCE_FRESHNESS_TABLE_NAME", SOURCE_FRESHNESS_TABLE_NAME), + ("CONSUMPTION_TABLE_NAME", CONSUMPTION_TABLE_NAME), ("MIGRATION_TABLE_NAME", MIGRATION_TABLE_NAME), ("COLUMN_MIGRATION_TABLE_NAME", COLUMN_MIGRATION_TABLE_NAME), ("OLD_NAME_VIEW_TABLE_NAME", OLD_NAME_VIEW_TABLE_NAME), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs index 052592d9e5..fceeda9567 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs @@ -25,7 +25,7 @@ const EXECUTION_LOGGER: &str = "sqlbuild.execution"; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; -use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; +use sqlbuild_core::constants::{CONSUMPTION_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME}; /// The adapter and its open connection. pub(crate) struct Warehouse<'py> { @@ -475,7 +475,7 @@ pub(crate) fn gather_warehouse_facts<'py>( }) } -/// Fingerprint and source freshness history prune candidates for target schemas. +/// Fingerprint, source freshness and consumption history prune candidates for target schemas. pub(crate) fn direct_state_prune_candidates<'py>( warehouse: &Warehouse<'py>, target_schemas: &[SchemaKey], @@ -494,6 +494,7 @@ pub(crate) fn direct_state_prune_candidates<'py>( schema.clone(), SOURCE_FRESHNESS_TABLE_NAME, )); + locations.push((database.clone(), schema.clone(), CONSUMPTION_TABLE_NAME)); } let lookup: HashSet = relation_lookup(warehouse, &locations)?; let mut candidates: Vec> = Vec::new(); @@ -507,6 +508,10 @@ pub(crate) fn direct_state_prune_candidates<'py>( SOURCE_FRESHNESS_TABLE_NAME, "render_prune_source_freshness_history_sql", ), + ( + CONSUMPTION_TABLE_NAME, + "render_prune_consumption_history_sql", + ), ] { if !lookup_exists(&lookup, (database.as_deref(), schema, table_name)) { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 732a57220e..1b33b89fce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -28,6 +28,7 @@ pub(crate) mod compiler_enums; pub(crate) mod compiler_errors; pub(crate) mod compiler_records; pub(crate) mod compute_logs; +pub(crate) mod consumption; pub(crate) mod contract_command; pub(crate) mod contract_records; pub(crate) mod contracts; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs new file mode 100644 index 0000000000..3579c5959f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs @@ -0,0 +1,491 @@ +//! The consumption table through the adapter, and the connection and callbacks a recording build holds. + +use chrono::{DateTime, NaiveDateTime, Utc}; +use std::sync::Mutex; + +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet, PyInt, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::constants::CONSUMPTION_TABLE_NAME; +use sqlbuild_state::consumption::constants::{ + CONSUMPTION_COLUMN_TYPES, INTEGER_STATE_VALUE_TYPE, NULLABLE_COLUMNS, RECENT_EXECUTIONS, +}; +use sqlbuild_state::consumption::main::prune_executions_sql::prune_executions_sql; +use sqlbuild_state::consumption::main::recent_executions_sql::recent_executions_sql; +use sqlbuild_state::consumption::models::{ + ConsumedInput, ConsumptionAppend, ConsumptionRow, NodeKey, RelationKey, +}; +use sqlbuild_state::freshness::constants::STRING_STATE_VALUE_TYPE; +use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; + +use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::contracts::state_literals::render_state_sql_literal; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; + +/// Rows per `INSERT`; SQL Server accepts at most 1,000 rows in one `VALUES` list. +const ROWS_PER_STATEMENT: usize = 500; +/// Warehouse timestamp texts with a UTC offset, as drivers render aware values. +const OFFSET_TIME_FORMATS: [&str; 2] = ["%Y-%m-%dT%H:%M:%S%.f%:z", "%Y-%m-%d %H:%M:%S%.f%:z"]; +/// Warehouse timestamp texts without an offset, which the consumption table stores as UTC. +const NAIVE_TIME_FORMATS: [&str; 2] = ["%Y-%m-%dT%H:%M:%S%.f", "%Y-%m-%d %H:%M:%S%.f"]; + +/// One destination schema of the consumption table. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub(crate) struct Location { + pub(crate) database: Option, + pub(crate) schema: String, +} + +impl Location { + pub(crate) fn of(relation: &RelationKey) -> Self { + Self { + database: relation.database.clone(), + schema: relation.schema.clone(), + } + } + + fn keywords<'py>(&self, py: Python<'py>) -> Vec<(&'static str, Bound<'py, PyAny>)> { + vec![ + ( + "database", + self.database + .as_deref() + .map_or_else(|| py.None().into_bound(py), |database| text(py, database)), + ), + ("schema", text(py, &self.schema)), + ] + } +} + +fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +/// The adapter's qualified name of the consumption table in `(database, schema)`. +fn qualified_name( + render_qualified_name: &Bound<'_, PyAny>, + (database, schema): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult> { + let py: Python<'_> = render_qualified_name.py(); + let name: Bound<'_, PyAny> = call_with( + render_qualified_name, + &[ + ("database", database.clone()), + ("schema", schema.clone()), + ("name", text(py, CONSUMPTION_TABLE_NAME)), + ], + )?; + if name.is_none() { + return Ok(None); + } + Ok(Some(name.extract()?)) +} + +fn table_of(adapter: &Bound<'_, PyAny>, location: &Location) -> PyResult> { + let py: Python<'_> = adapter.py(); + let database: Bound<'_, PyAny> = location + .database + .as_deref() + .map_or_else(|| py.None().into_bound(py), |database| text(py, database)); + qualified_name( + &native_field(adapter, "render_qualified_name")?, + (&database, &text(py, &location.schema)), + ) +} + +/// Create-if-missing DDL for the consumption table, with adapter-rendered column types. +#[pyfunction] +#[pyo3(signature = (*, database, schema, render_qualified_name, render_framework_type))] +fn build_consumption_create_table_sql( + database: &Bound<'_, PyAny>, + schema: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, + render_framework_type: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = database.py(); + let Some(table) = qualified_name(render_qualified_name, (database, schema))? else { + return Ok(String::new()); + }; + let columns: Bound<'_, PyAny> = PyTuple::new( + py, + CONSUMPTION_COLUMN_TYPES.iter().map(|(column, _)| *column), + )? + .into_any(); + let types: Bound<'_, PyDict> = PyDict::new(py); + for (column, declared) in CONSUMPTION_COLUMN_TYPES { + types.set_item(column, declared)?; + } + let required: Vec<&str> = CONSUMPTION_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .filter(|column| !NULLABLE_COLUMNS.contains(column)) + .collect(); + state_table_create_sql( + &table, + ( + &columns, + &types.into_any(), + &PyFrozenSet::new(py, &required)?.into_any(), + ), + render_framework_type, + ) +} + +/// SQL deleting whole consumption executions beyond the newest `retain_versions` per relation. +#[pyfunction] +#[pyo3(signature = (*, database, schema, render_qualified_name, retain_versions))] +fn build_consumption_prune_sql( + database: &Bound<'_, PyAny>, + schema: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, + retain_versions: i64, +) -> PyResult { + Ok(qualified_name(render_qualified_name, (database, schema))? + .map(|table| prune_executions_sql(&table, retain_versions)) + .unwrap_or_default()) +} + +fn execute<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + sql: &str, +) -> PyResult> { + call_with( + &native_field(adapter, "execute")?, + &[ + ("connection", connection.clone()), + ("sql", text(adapter.py(), sql)), + ], + ) +} + +/// Create the schema, table and indexes in `location`; `started` rows precede schema preparation. +fn ensure_table( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + location: &Location, +) -> PyResult<()> { + let py: Python<'_> = adapter.py(); + let keywords = location.keywords(py); + for statement in + call_with(&native_field(adapter, "render_create_schema")?, &keywords)?.try_iter()? + { + execute(adapter, connection, &statement?.extract::()?)?; + } + let create: String = call_with( + &native_field(adapter, "render_create_consumption_table_sql")?, + &keywords, + )? + .extract()?; + if !create.is_empty() { + execute(adapter, connection, &create)?; + } + for index in call_with( + &native_field(adapter, "render_create_consumption_index_sqls")?, + &keywords, + )? + .try_iter()? + { + execute(adapter, connection, &index?.extract::()?)?; + } + Ok(()) +} + +/// Whether the warehouse catalog lists the consumption table in `location`. +fn table_exists( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + location: &Location, +) -> PyResult { + let py: Python<'_> = adapter.py(); + let mut keywords = location.keywords(py); + keywords.push(("connection", connection.clone())); + keywords.push(("name", text(py, CONSUMPTION_TABLE_NAME))); + call_with(&native_field(adapter, "get_columns")?, &keywords)?.is_truthy() +} + +/// Rows of the newest executions per relation in `location`; none when the table is missing. +fn read_recent( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + location: &Location, +) -> PyResult> { + let Some(table) = table_of(adapter, location)? else { + return Ok(Vec::new()); + }; + if !table_exists(adapter, connection, location)? { + return Ok(Vec::new()); + } + let result: Bound<'_, PyAny> = execute( + adapter, + connection, + &recent_executions_sql(&table, RECENT_EXECUTIONS), + )?; + call_with(&native_field(&result, "fetchall")?, &[])? + .try_iter()? + .map(|row| read_row(&row?)) + .collect() +} + +fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.str()?.extract()?)) +} + +fn required_text(value: &Bound<'_, PyAny>) -> PyResult { + Ok(optional_text(value)?.unwrap_or_default()) +} + +/// Microseconds since the epoch of a warehouse timestamp's text, naive values read as UTC. +fn epoch_micros(value: &Bound<'_, PyAny>) -> PyResult { + let text: String = value.str()?.extract()?; + let text: &str = text.trim(); + for format in OFFSET_TIME_FORMATS { + if let Ok(moment) = DateTime::parse_from_str(text, format) { + return Ok(moment.with_timezone(&Utc).timestamp_micros()); + } + } + for format in NAIVE_TIME_FORMATS { + if let Ok(naive) = NaiveDateTime::parse_from_str(text, format) { + return Ok(naive.and_utc().timestamp_micros()); + } + } + Err(PyValueError::new_err(format!( + "unreadable consumption time {text:?}" + ))) +} + +fn read_row(row: &Bound<'_, PyAny>) -> PyResult { + let cell = |index: usize| row.get_item(index); + let input_kind: Option = optional_text(&cell(11)?)?; + let input: Option = match input_kind { + Some(kind) => Some(ConsumedInput { + kind, + name: required_text(&cell(12)?)?, + version: required_text(&cell(13)?)?, + }), + None => None, + }; + let count: Bound<'_, PyAny> = cell(10)?; + let input_count: i64 = if count.is_instance_of::() { + count.extract()? + } else { + count + .str()? + .extract::()? + .trim() + .parse() + .unwrap_or(-1) + }; + Ok(ConsumptionRow { + row: ConsumptionAppend { + event_id: required_text(&cell(0)?)?, + execution_id: required_text(&cell(1)?)?, + phase: required_text(&cell(2)?)?, + run_id: required_text(&cell(3)?)?, + node: NodeKey { + kind: required_text(&cell(4)?)?, + name: required_text(&cell(5)?)?, + }, + relation: RelationKey { + database: optional_text(&cell(6)?)?, + schema: required_text(&cell(7)?)?, + name: required_text(&cell(8)?)?, + }, + version_hash: required_text(&cell(9)?)?, + input_count, + input, + }, + recorded_at: epoch_micros(&cell(14)?)?, + }) +} + +fn literal(py: Python<'_>, value: Option<&str>) -> PyResult { + let value: Bound<'_, PyAny> = + value.map_or_else(|| py.None().into_bound(py), |value| text(py, value)); + render_state_sql_literal(py, &value, STRING_STATE_VALUE_TYPE, None) +} + +fn row_cells(py: Python<'_>, row: &ConsumptionAppend, recorded_at: &str) -> PyResult> { + let input = row.input.as_ref(); + Ok(vec![ + literal(py, Some(&row.event_id))?, + literal(py, Some(&row.execution_id))?, + literal(py, Some(&row.phase))?, + literal(py, Some(&row.run_id))?, + literal(py, Some(&row.node.kind))?, + literal(py, Some(&row.node.name))?, + literal(py, row.relation.database.as_deref())?, + literal(py, Some(&row.relation.schema))?, + literal(py, Some(&row.relation.name))?, + literal(py, Some(&row.version_hash))?, + render_state_sql_literal( + py, + &PyInt::new(py, row.input_count).into_any(), + INTEGER_STATE_VALUE_TYPE, + None, + )?, + literal(py, input.map(|input| input.kind.as_str()))?, + literal(py, input.map(|input| input.name.as_str()))?, + literal(py, input.map(|input| input.version.as_str()))?, + recorded_at.to_owned(), + ]) +} + +/// Append `rows` to `location`'s table in statements of whole executions where they fit. +fn append( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + location: &Location, + rows: &[ConsumptionAppend], +) -> PyResult<()> { + let py: Python<'_> = adapter.py(); + let Some(table) = table_of(adapter, location)? else { + return Ok(()); + }; + let recorded_at: String = + call_with(&native_field(adapter, "render_current_utc_timestamp")?, &[])?.extract()?; + let columns: Vec<&str> = CONSUMPTION_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .collect(); + for chunk in execution_chunks(rows) { + let cells: Vec> = chunk + .iter() + .map(|row| row_cells(py, row, &recorded_at)) + .collect::>()?; + execute( + adapter, + connection, + &values_insert_sql(&table, &columns, &cells), + )?; + } + Ok(()) +} + +/// Contiguous rows in statements of whole executions, splitting only an execution that alone overflows. +fn execution_chunks(rows: &[ConsumptionAppend]) -> Vec<&[ConsumptionAppend]> { + let mut chunks: Vec<&[ConsumptionAppend]> = Vec::new(); + let (mut start, mut index): (usize, usize) = (0, 0); + while index < rows.len() { + let end: usize = index + + rows[index..] + .iter() + .take_while(|row| row.execution_id == rows[index].execution_id) + .count(); + if end - start > ROWS_PER_STATEMENT && index > start { + chunks.push(&rows[start..index]); + start = index; + } + while end - start > ROWS_PER_STATEMENT { + chunks.push(&rows[start..start + ROWS_PER_STATEMENT]); + start += ROWS_PER_STATEMENT; + } + index = end; + } + if start < rows.len() { + chunks.push(&rows[start..]); + } + chunks +} + +/// One build's consumption I/O, connected only per phase so loaders can lock file warehouses. +pub(crate) struct ConsumptionConnection { + adapter: Py, + connection_config: Py, + connection: Mutex>>, +} + +impl ConsumptionConnection { + pub(crate) fn new(adapter: &Bound<'_, PyAny>, connection_config: &Bound<'_, PyAny>) -> Self { + Self { + adapter: adapter.clone().unbind(), + connection_config: connection_config.clone().unbind(), + connection: Mutex::new(None), + } + } + + fn connected<'py>(&self, py: Python<'py>) -> PyResult> { + let mut open = self + .connection + .lock() + .map_err(|_| PyRuntimeError::new_err("consumption connection lock poisoned"))?; + if let Some(connection) = open.as_ref() { + return Ok(connection.bind(py).clone()); + } + let connection: Bound<'py, PyAny> = native_field(self.adapter.bind(py), "connect")? + .call1((self.connection_config.bind(py),))?; + *open = Some(connection.clone().unbind()); + Ok(connection) + } + + pub(crate) fn ensure_table(&self, py: Python<'_>, location: &Location) -> PyResult<()> { + ensure_table(self.adapter.bind(py), &self.connected(py)?, location) + } + + pub(crate) fn read_recent( + &self, + py: Python<'_>, + location: &Location, + ) -> PyResult> { + read_recent(self.adapter.bind(py), &self.connected(py)?, location) + } + + pub(crate) fn append( + &self, + py: Python<'_>, + location: &Location, + rows: &[ConsumptionAppend], + ) -> PyResult<()> { + append(self.adapter.bind(py), &self.connected(py)?, location, rows) + } + + /// Close the connection if one is open; the next operation connects again. + pub(crate) fn close(&self, py: Python<'_>) -> PyResult<()> { + let open: Option> = self + .connection + .lock() + .map_err(|_| PyRuntimeError::new_err("consumption connection lock poisoned"))? + .take(); + if let Some(connection) = open { + native_field(self.adapter.bind(py), "close")?.call1((connection.bind(py),))?; + } + Ok(()) + } +} + +/// The build's own node-completion callback, which the recorder forwards every result to. +pub(crate) struct NodeCompleteHook(Py); + +impl NodeCompleteHook { + pub(crate) fn new(callback: &Bound<'_, PyAny>) -> Self { + Self(callback.clone().unbind()) + } + + pub(crate) fn forward(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { + self.0.bind(result.py()).call1((result,))?; + Ok(()) + } +} + +/// The recorder's `on_node_complete` and `before_node_write` methods, as build callbacks. +pub(crate) fn recorder_callbacks<'py>( + recorder: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + Ok(( + recorder.getattr("on_node_complete")?, + recorder.getattr("before_node_write")?, + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_consumption_create_table_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(build_consumption_prune_sql, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 849f8bf165..3fb073463c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod connections; pub(crate) mod constants; +pub(crate) mod consumption_state; pub(crate) mod header_projection; pub(crate) mod host; pub(crate) mod interrupt; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index a5c265e81e..240edbdff5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -8,12 +8,15 @@ use sqlbuild_analysis::fingerprint_storage::constants::{COLUMN_TIMESTAMP, FINGER use sqlbuild_config::command_connection::constants::{ DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, LOCAL_DATABASE_PATH_KEYS, }; -use sqlbuild_core::constants::{FINGERPRINT_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME}; +use sqlbuild_core::constants::{ + CONSUMPTION_TABLE_NAME, FINGERPRINT_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, +}; use sqlbuild_core::models::SqlValue; use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use sqlbuild_orchestration::warehouse_metadata::models::{ WarehouseHistoryPage, WarehouseHistoryRead, WarehouseHistoryRecord, WarehouseStateLocation, }; +use sqlbuild_state::consumption::constants::{CONSUMPTION_COLUMN_TYPES, RECORDED_AT_COLUMN}; use sqlbuild_state::events::constants::{MIGRATION_COLUMNS, OLD_NAME_VIEW_COLUMNS}; use sqlbuild_state::freshness::constants::{ OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES, SQLITE_ADAPTER, @@ -50,7 +53,8 @@ pub(crate) fn read_history( | WarehouseHistoryRead::JanitorEvents(read) | WarehouseHistoryRead::Migrations(read) | WarehouseHistoryRead::OldNameViews(read) - | WarehouseHistoryRead::Microbatches(read) => &read.location, + | WarehouseHistoryRead::Microbatches(read) + | WarehouseHistoryRead::Consumption(read) => &read.location, }; if location.schema.trim().is_empty() { return Err(PyValueError::new_err( @@ -110,6 +114,7 @@ fn read_page( request: &WarehouseHistoryRead, ) -> PyResult { let freshness_columns = SOURCE_FRESHNESS_COLUMN_TYPES.map(|(name, _)| name); + let consumption_columns = CONSUMPTION_COLUMN_TYPES.map(|(name, _)| name); let event_columns: Vec = match request { WarehouseHistoryRead::AuditResults(_) => py .import("sqlbuild.executor.audit_results.constants")? @@ -196,6 +201,15 @@ fn read_page( event_columns.as_slice(), "created_at", ), + WarehouseHistoryRead::Consumption(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + read.limit, + CONSUMPTION_TABLE_NAME, + consumption_columns.as_slice(), + RECORDED_AT_COLUMN, + ), }; let limit = limit.unwrap_or(MAX_HISTORY_RECORDS); if limit == 0 || limit > MAX_HISTORY_RECORDS { @@ -269,6 +283,7 @@ fn read_page( WarehouseHistoryRead::Migrations(_) | WarehouseHistoryRead::OldNameViews(_) | WarehouseHistoryRead::Microbatches(_) => Some(&["event_id", "created_at"]), + WarehouseHistoryRead::Consumption(_) => Some(&["event_id", RECORDED_AT_COLUMN]), _ => None, }; let mut cursor_indexes: Vec = columns @@ -389,6 +404,7 @@ fn read_page( WarehouseHistoryRecord::Migration(record) => record.created_at, WarehouseHistoryRecord::OldNameView(record) => record.created_at, WarehouseHistoryRecord::Microbatch(record) => record.created_at, + WarehouseHistoryRecord::Consumption(record) => record.recorded_at, }; if since.is_none_or(|since| observed >= since) { records.push(record); @@ -427,7 +443,9 @@ fn decode_record( qualified, )?, ), - WarehouseHistoryRead::AuditResults(read) | WarehouseHistoryRead::Migrations(read) => { + WarehouseHistoryRead::AuditResults(read) + | WarehouseHistoryRead::Migrations(read) + | WarehouseHistoryRead::Consumption(read) => { return decode_event_record(&read.location, row, (columns, &[timestamp])); } WarehouseHistoryRead::JanitorEvents(read) => { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs index cc372a504b..e6041ffcc6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs @@ -51,7 +51,9 @@ fn string_pairs(mapping: &Bound<'_, PyAny>) -> PyResult> { } /// `{name: identity.version_hash}` of the discovered Python hooks. -fn hook_version_hashes<'py>(hook_functions: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn hook_version_hashes<'py>( + hook_functions: &Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = hook_functions.py(); let hashes: Bound<'py, PyDict> = PyDict::new(py); let identities: Bound<'py, PyAny> = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index 3fdd402cc8..c34a7e8402 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -243,6 +243,7 @@ const BUILD_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("no_color", FieldDefault::False), FieldSpec::new("fail_fast", FieldDefault::False), FieldSpec::new("full_refresh", FieldDefault::False), + FieldSpec::new("changed", FieldDefault::False), FieldSpec::new("load_sources", FieldDefault::None), FieldSpec::new("reload_sources", FieldDefault::False), FieldSpec::new("include_python", FieldDefault::True), diff --git a/crates/sqlbuild-state/src/consumption/_helpers/decision.rs b/crates/sqlbuild-state/src/consumption/_helpers/decision.rs new file mode 100644 index 0000000000..211c8a548f --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/_helpers/decision.rs @@ -0,0 +1,166 @@ +//! Decide which selected nodes `--changed` runs, from their facts and their recent executions. + +use std::collections::{BTreeSet, HashMap}; + +use crate::consumption::_helpers::history::Window; +use crate::consumption::_helpers::rules::DecisionRules; +use crate::consumption::constants::{NODE_INPUT, OPAQUE_INPUT, SOURCE_INPUT}; +use crate::consumption::models::{ConsumptionHistory, InputFact, NodeFacts, NodeKey, RunReason}; + +/// The state table key of an input: its kind and name. +pub(crate) fn input_key(input: &InputFact) -> (&'static str, String) { + match input { + InputFact::Source { name, .. } => (SOURCE_INPUT, name.clone()), + InputFact::Node(key) => (NODE_INPUT, node_input_name(key)), + InputFact::Opaque(name) => (OPAQUE_INPUT, name.clone()), + } +} + +/// The recorded name of an upstream node input: `:`. +pub(crate) fn node_input_name(key: &NodeKey) -> String { + format!("{}:{}", key.kind, key.name) +} + +/// Each selected node in input order with the reason it runs, or None when it is skipped. +pub(crate) fn decide( + rules: &R, + nodes: &[NodeFacts], + history: &ConsumptionHistory, +) -> Vec<(NodeKey, Option)> { + let by_key: HashMap<&NodeKey, &NodeFacts> = + nodes.iter().map(|node| (&node.key, node)).collect(); + let mut stale: HashMap<&NodeKey, Option> = HashMap::new(); + for node in topological(nodes, &by_key) { + let reason: Option = match own_reason(rules, node, &by_key, history) { + Some(reason) => Some(reason), + None if rules.follows_upstream() => stale_upstream(node, &stale), + None => None, + }; + stale.insert(&node.key, reason); + } + nodes + .iter() + .filter(|node| node.selected) + .map(|node| (node.key.clone(), stale.remove(&node.key).flatten())) + .collect() +} + +/// The first upstream of `node` that runs or is stale. +fn stale_upstream( + node: &NodeFacts, + stale: &HashMap<&NodeKey, Option>, +) -> Option { + for input in &node.inputs { + if let InputFact::Node(upstream) = input + && matches!(stale.get(upstream), Some(Some(_))) + { + return Some(RunReason::UpstreamRuns(upstream.clone())); + } + } + None +} + +/// Nodes with every upstream that has facts before it. +fn topological<'a>( + nodes: &'a [NodeFacts], + by_key: &HashMap<&NodeKey, &'a NodeFacts>, +) -> Vec<&'a NodeFacts> { + let mut ordered: Vec<&'a NodeFacts> = Vec::with_capacity(nodes.len()); + let mut placed: BTreeSet<&NodeKey> = BTreeSet::new(); + let mut stack: Vec<(&'a NodeFacts, usize)> = Vec::new(); + for root in nodes { + if placed.contains(&root.key) { + continue; + } + let mut entered: BTreeSet<&NodeKey> = BTreeSet::from([&root.key]); + stack.push((root, 0)); + while let Some((node, next)) = stack.pop() { + let upstream: Option<&'a NodeFacts> = + node.inputs.get(next).and_then(|input| match input { + InputFact::Node(key) => by_key.get(key).copied(), + _ => None, + }); + if next < node.inputs.len() { + stack.push((node, next + 1)); + if let Some(upstream) = upstream + && !placed.contains(&upstream.key) + && entered.insert(&upstream.key) + { + stack.push((upstream, 0)); + } + continue; + } + if placed.insert(&node.key) { + ordered.push(node); + } + } + } + ordered +} + +/// Why the node's own facts or records make it run, ignoring its upstreams' staleness. +fn own_reason( + rules: &R, + node: &NodeFacts, + by_key: &HashMap<&NodeKey, &NodeFacts>, + history: &ConsumptionHistory, +) -> Option { + if let Some(forced) = node.forced.iter().find(|forced| rules.honours(forced)) { + return Some(RunReason::Forced(forced.clone())); + } + let Some(window) = history.window(&node.relation) else { + return Some(RunReason::NoRecord); + }; + if window.ambiguous { + return Some(RunReason::AmbiguousOrder); + } + execution_reason(rules, node, (by_key, history), &window) +} + +fn execution_reason( + rules: &R, + node: &NodeFacts, + (by_key, history): (&HashMap<&NodeKey, &NodeFacts>, &ConsumptionHistory), + window: &Window<'_>, +) -> Option { + let expected: BTreeSet<(&str, String)> = node.inputs.iter().map(input_key).collect(); + for execution in window.executions { + if !execution.complete { + return Some(RunReason::Unfinished); + } + if execution.version_hash.as_deref() != Some(node.version_hash.as_str()) { + return Some(RunReason::DefinitionChanged); + } + let recorded: BTreeSet<(&str, String)> = execution + .inputs + .iter() + .map(|input| (input.kind.as_str(), input.name.clone())) + .collect(); + if recorded != expected { + return Some(RunReason::InputsChanged); + } + for input in &node.inputs { + let (kind, name) = input_key(input); + let current: Option<&str> = match input { + InputFact::Source { version, .. } => version.as_deref(), + InputFact::Node(upstream) if rules.compares_node_inputs() => by_key + .get(upstream) + .and_then(|facts| history.latest_complete(&facts.relation)), + InputFact::Node(_) => continue, + InputFact::Opaque(_) => None, + }; + let Some(current) = current else { + return Some(RunReason::InputUnknown(name)); + }; + let read: Option<&str> = execution + .inputs + .iter() + .find(|recorded| recorded.kind == kind && recorded.name == name) + .map(|recorded| recorded.version.as_str()); + if read != Some(current) { + return Some(RunReason::InputChanged(name)); + } + } + } + None +} diff --git a/crates/sqlbuild-state/src/consumption/_helpers/event_id.rs b/crates/sqlbuild-state/src/consumption/_helpers/event_id.rs new file mode 100644 index 0000000000..4eeb317f1a --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/_helpers/event_id.rs @@ -0,0 +1,19 @@ +//! A consumption row's identity: its execution, phase and the input it records. + +use sha2::{Digest, Sha256}; + +use crate::consumption::constants::EVENT_NAMESPACE; +use crate::consumption::models::ConsumedInput; + +/// SHA-256 of `(execution_id, phase, input_kind, input_name, input_version)`, so a retried append repeats it. +pub(crate) fn event_id(execution_id: &str, phase: &str, input: Option<&ConsumedInput>) -> String { + let identity = serde_json::json!([ + EVENT_NAMESPACE, + execution_id, + phase, + input.map(|input| input.kind.as_str()), + input.map(|input| input.name.as_str()), + input.map(|input| input.version.as_str()), + ]); + format!("{:x}", Sha256::digest(identity.to_string().as_bytes())) +} diff --git a/crates/sqlbuild-state/src/consumption/_helpers/history.rs b/crates/sqlbuild-state/src/consumption/_helpers/history.rs new file mode 100644 index 0000000000..39c4855eca --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/_helpers/history.rs @@ -0,0 +1,164 @@ +//! Rebuild executions from deduplicated rows and read the window `--changed` compares. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use crate::consumption::_helpers::rules::DecisionRules; +use crate::consumption::constants::{ORDER_SKEW_MICROS, SUCCEEDED_PHASE}; +use crate::consumption::models::{ + ConsumedInput, ConsumptionHistory, ConsumptionRow, Execution, RelationHistory, RelationKey, +}; + +/// The rows of one execution with the earliest and latest times seen for each event. +#[derive(Default)] +struct ExecutionRows<'a> { + rows: Vec<&'a ConsumptionRow>, + earliest: i64, + latest: i64, +} + +/// Recent executions per relation, from rows read newest first with at most `limit` per relation. +pub(crate) fn build_history( + rules: &R, + rows: &[ConsumptionRow], + limit: usize, +) -> ConsumptionHistory { + let mut events: HashMap<&str, (&ConsumptionRow, i64, i64)> = HashMap::new(); + for row in rows { + let entry = events.entry(row.row.event_id.as_str()).or_insert(( + row, + row.recorded_at, + row.recorded_at, + )); + if row.recorded_at < entry.1 { + *entry = (row, row.recorded_at, entry.2); + } + entry.2 = entry.2.max(row.recorded_at); + } + let mut grouped: BTreeMap<(&RelationKey, &str), ExecutionRows<'_>> = BTreeMap::new(); + for (row, earliest, latest) in events.into_values() { + let key = (&row.row.relation, row.row.execution_id.as_str()); + let execution = grouped.entry(key).or_insert(ExecutionRows { + rows: Vec::new(), + earliest, + latest, + }); + execution.earliest = execution.earliest.min(earliest); + execution.latest = execution.latest.max(latest); + execution.rows.push(row); + } + let mut relations: HashMap = HashMap::new(); + for ((relation, execution_id), execution_rows) in grouped { + relations + .entry(relation.clone()) + .or_default() + .executions + .push(execution(rules, execution_id, &execution_rows)); + } + for history in relations.values_mut() { + history.executions.sort_by(|left, right| { + (right.started_at, &right.execution_id).cmp(&(left.started_at, &left.execution_id)) + }); + history.truncated = history.executions.len() >= limit; + } + ConsumptionHistory { relations } +} + +fn execution( + rules: &R, + execution_id: &str, + rows: &ExecutionRows<'_>, +) -> Execution { + let succeeded: Vec<&ConsumptionRow> = rows + .rows + .iter() + .copied() + .filter(|row| row.row.phase == SUCCEEDED_PHASE) + .collect(); + let versions: BTreeSet<&str> = rows + .rows + .iter() + .map(|row| row.row.version_hash.as_str()) + .collect(); + let counts: BTreeSet = succeeded.iter().map(|row| row.row.input_count).collect(); + let mut inputs: BTreeMap<(&str, &str), BTreeSet<&str>> = BTreeMap::new(); + let mut inputless: bool = false; + for row in &succeeded { + match &row.row.input { + Some(input) => { + inputs + .entry((input.kind.as_str(), input.name.as_str())) + .or_default() + .insert(input.version.as_str()); + } + None => inputless = true, + } + } + let consistent: bool = versions.len() == 1 + && counts.len() == 1 + && inputs.values().all(|versions| versions.len() == 1); + let expected: Option = counts.first().copied(); + let complete: bool = consistent + && match expected { + Some(0) => inputless && inputs.is_empty(), + Some(count) => !inputless && i64::try_from(inputs.len()) == Ok(count), + None => false, + }; + Execution { + execution_id: execution_id.to_owned(), + started_at: rules.execution_time(rows.earliest, rows.latest), + complete, + version_hash: (versions.len() == 1) + .then(|| versions.first().map(|version| (*version).to_owned())) + .flatten(), + inputs: if complete { + inputs.into_iter().filter_map(single_version).collect() + } else { + Vec::new() + }, + } +} + +/// The input with its one recorded version. +fn single_version( + ((kind, name), versions): ((&str, &str), BTreeSet<&str>), +) -> Option { + let version: &str = versions.first()?; + Some(ConsumedInput { + kind: kind.to_owned(), + name: name.to_owned(), + version: version.to_owned(), + }) +} + +/// The executions `--changed` must all check: those within the order skew of the newest. +pub(crate) struct Window<'a> { + pub(crate) executions: &'a [Execution], + /// Older unread executions may also fall inside the skew. + pub(crate) ambiguous: bool, +} + +impl ConsumptionHistory { + /// The newest executions of `relation` within the skew, or None when it has none. + pub(crate) fn window(&self, relation: &RelationKey) -> Option> { + let history: &RelationHistory = self.relations.get(relation)?; + let newest: i64 = history.executions.first()?.started_at; + let length: usize = history + .executions + .iter() + .take_while(|execution| execution.started_at >= newest - ORDER_SKEW_MICROS) + .count(); + Some(Window { + executions: &history.executions[..length], + ambiguous: history.truncated && length == history.executions.len(), + }) + } + + /// The execution ID of `relation`'s newest execution when it alone is newest and complete. + pub fn latest_complete(&self, relation: &RelationKey) -> Option<&str> { + let window: Window<'_> = self.window(relation)?; + match window.executions { + [only] if !window.ambiguous && only.complete => Some(only.execution_id.as_str()), + _ => None, + } + } +} diff --git a/crates/sqlbuild-state/src/consumption/_helpers/mod.rs b/crates/sqlbuild-state/src/consumption/_helpers/mod.rs new file mode 100644 index 0000000000..06e3432e89 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod decision; +pub(crate) mod event_id; +pub(crate) mod history; +pub(crate) mod rules; diff --git a/crates/sqlbuild-state/src/consumption/_helpers/rules.rs b/crates/sqlbuild-state/src/consumption/_helpers/rules.rs new file mode 100644 index 0000000000..24a08a25aa --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/_helpers/rules.rs @@ -0,0 +1,33 @@ +//! The decision rules `--changed` applies; tests swap single rules to prove the simulation notices. + +use crate::consumption::models::ForcedRun; + +/// Each rule of the decision, with the shipped behaviour as its default. +pub(crate) trait DecisionRules { + /// An execution's time: the earliest of its rows, so a late duplicate cannot make it newer. + fn execution_time(&self, earliest: i64, latest: i64) -> i64 { + let _ = latest; + earliest + } + + /// Whether upstream model and seed inputs compare their recorded execution IDs. + fn compares_node_inputs(&self) -> bool { + true + } + + /// Whether a stale upstream, selected or not, makes its downstream run. + fn follows_upstream(&self) -> bool { + true + } + + /// Whether the planner's own reason to run a node is honoured. + fn honours(&self, forced: &ForcedRun) -> bool { + let _ = forced; + true + } +} + +/// The shipped rules. +pub(crate) struct Shipped; + +impl DecisionRules for Shipped {} diff --git a/crates/sqlbuild-state/src/consumption/constants.rs b/crates/sqlbuild-state/src/consumption/constants.rs new file mode 100644 index 0000000000..06a1975b65 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/constants.rs @@ -0,0 +1,51 @@ +//! Columns, phases, input kinds and read bounds of the consumption state table. + +use crate::freshness::constants::{STRING_STATE_VALUE_TYPE, TEXT_TIMESTAMP_STATE_VALUE_TYPE}; + +/// State value type of an integer column. +pub const INTEGER_STATE_VALUE_TYPE: &str = "integer"; +/// The column the warehouse clock fills on insert. +pub const RECORDED_AT_COLUMN: &str = "recorded_at"; +/// Columns in declaration order with their state value types; `recorded_at` is last. +pub const CONSUMPTION_COLUMN_TYPES: [(&str, &str); 15] = [ + ("event_id", STRING_STATE_VALUE_TYPE), + ("execution_id", STRING_STATE_VALUE_TYPE), + ("phase", STRING_STATE_VALUE_TYPE), + ("run_id", STRING_STATE_VALUE_TYPE), + ("node_type", STRING_STATE_VALUE_TYPE), + ("node_name", STRING_STATE_VALUE_TYPE), + ("target_database", STRING_STATE_VALUE_TYPE), + ("target_schema", STRING_STATE_VALUE_TYPE), + ("target_name", STRING_STATE_VALUE_TYPE), + ("version_hash", STRING_STATE_VALUE_TYPE), + ("input_count", INTEGER_STATE_VALUE_TYPE), + ("input_kind", STRING_STATE_VALUE_TYPE), + ("input_name", STRING_STATE_VALUE_TYPE), + ("input_version", STRING_STATE_VALUE_TYPE), + (RECORDED_AT_COLUMN, TEXT_TIMESTAMP_STATE_VALUE_TYPE), +]; +/// Columns that may be NULL: the relation's database and the input of `started` and input-less rows. +pub const NULLABLE_COLUMNS: [&str; 4] = [ + "target_database", + "input_kind", + "input_name", + "input_version", +]; +/// Phase of the row appended before a model or seed writes. +pub const STARTED_PHASE: &str = "started"; +/// Phase of the rows appended after the write succeeded. +pub const SUCCEEDED_PHASE: &str = "succeeded"; +/// Input kind of a source, versioned by its observed `data_version_hash`. +pub const SOURCE_INPUT: &str = "source"; +/// Input kind of an upstream model or seed, versioned by its `execution_id`. +pub const NODE_INPUT: &str = "node"; +/// Input kind of anything whose version cannot be known. +pub const OPAQUE_INPUT: &str = "opaque"; +/// The version recorded for an input whose current version is unknown. +pub const UNKNOWN_VERSION: &str = "unknown"; +/// Executions read per relation, newest first. +pub const RECENT_EXECUTIONS: usize = 8; +/// Executions of one relation appended this close to the newest are all checked, in microseconds. +pub const ORDER_SKEW_MICROS: i64 = 5_000_000; +/// Event ID namespace of consumption rows. +pub(crate) const EVENT_NAMESPACE: &str = "sqlbuild.consumption.event.v1"; diff --git a/crates/sqlbuild-state/src/consumption/main/changed_run_set.rs b/crates/sqlbuild-state/src/consumption/main/changed_run_set.rs new file mode 100644 index 0000000000..1f5764554f --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/changed_run_set.rs @@ -0,0 +1,13 @@ +//! Which selected nodes `--changed` runs. + +use crate::consumption::_helpers::decision::decide; +use crate::consumption::_helpers::rules::Shipped; +use crate::consumption::models::{ConsumptionHistory, NodeFacts, NodeKey, RunReason}; + +/// Each selected node in order with why it runs, or None when nothing it or an upstream read changed. +pub fn changed_run_set( + nodes: &[NodeFacts], + history: &ConsumptionHistory, +) -> Vec<(NodeKey, Option)> { + decide(&Shipped, nodes, history) +} diff --git a/crates/sqlbuild-state/src/consumption/main/consumed_inputs.rs b/crates/sqlbuild-state/src/consumption/main/consumed_inputs.rs new file mode 100644 index 0000000000..6e13ee2c80 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/consumed_inputs.rs @@ -0,0 +1,42 @@ +//! The input versions an execution consumed, recorded with its `succeeded` rows. + +use std::collections::HashMap; + +use crate::consumption::_helpers::decision::input_key; +use crate::consumption::constants::UNKNOWN_VERSION; +use crate::consumption::models::{ + ConsumedInput, ConsumptionHistory, InputFact, NodeFacts, NodeKey, +}; + +/// Each distinct direct input with its source hash, upstream execution ID or `unknown`. +pub fn consumed_inputs( + node: &NodeFacts, + (nodes, history): (&HashMap, &ConsumptionHistory), + this_build: &HashMap, +) -> Vec { + let mut inputs: Vec = node + .inputs + .iter() + .map(|input| { + let (kind, name) = input_key(input); + let version: Option = match input { + InputFact::Source { version, .. } => version.clone(), + InputFact::Node(upstream) => this_build.get(upstream).cloned().or_else(|| { + nodes + .get(upstream) + .and_then(|facts| history.latest_complete(&facts.relation)) + .map(str::to_owned) + }), + InputFact::Opaque(_) => None, + }; + ConsumedInput { + kind: kind.to_owned(), + name, + version: version.unwrap_or_else(|| UNKNOWN_VERSION.to_owned()), + } + }) + .collect(); + inputs.sort(); + inputs.dedup_by(|left, right| left.kind == right.kind && left.name == right.name); + inputs +} diff --git a/crates/sqlbuild-state/src/consumption/main/consumption_history.rs b/crates/sqlbuild-state/src/consumption/main/consumption_history.rs new file mode 100644 index 0000000000..83ab8b9385 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/consumption_history.rs @@ -0,0 +1,10 @@ +//! The recent executions of each relation, rebuilt from rows read back. + +use crate::consumption::_helpers::history::build_history; +use crate::consumption::_helpers::rules::Shipped; +use crate::consumption::models::{ConsumptionHistory, ConsumptionRow}; + +/// Executions per relation from rows read with at most `limit` executions per relation. +pub fn consumption_history(rows: &[ConsumptionRow], limit: usize) -> ConsumptionHistory { + build_history(&Shipped, rows, limit) +} diff --git a/crates/sqlbuild-state/src/consumption/main/mod.rs b/crates/sqlbuild-state/src/consumption/main/mod.rs new file mode 100644 index 0000000000..8f097fabfd --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/mod.rs @@ -0,0 +1,7 @@ +pub mod changed_run_set; +pub mod consumed_inputs; +pub mod consumption_history; +pub mod prune_executions_sql; +pub mod recent_executions_sql; +pub mod started_append; +pub mod succeeded_appends; diff --git a/crates/sqlbuild-state/src/consumption/main/prune_executions_sql.rs b/crates/sqlbuild-state/src/consumption/main/prune_executions_sql.rs new file mode 100644 index 0000000000..cd7e064f21 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/prune_executions_sql.rs @@ -0,0 +1,15 @@ +//! Prune old executions from one consumption table. + +use crate::consumption::constants::RECORDED_AT_COLUMN; + +/// Delete every row of executions ranked beyond the newest `retain` per relation, whole executions at a time. +pub fn prune_executions_sql(qualified_name: &str, retain: i64) -> String { + format!( + "DELETE FROM {table} WHERE execution_id IN (SELECT execution_id FROM (SELECT execution_id, \ +ROW_NUMBER() OVER (PARTITION BY target_database, target_schema, target_name ORDER BY started_at DESC, \ +execution_id DESC) AS execution_rank FROM (SELECT execution_id, target_database, target_schema, \ +target_name, MIN({RECORDED_AT_COLUMN}) AS started_at FROM {table} GROUP BY execution_id, \ +target_database, target_schema, target_name) AS executions) AS ranked WHERE execution_rank > {retain})", + table = qualified_name, + ) +} diff --git a/crates/sqlbuild-state/src/consumption/main/recent_executions_sql.rs b/crates/sqlbuild-state/src/consumption/main/recent_executions_sql.rs new file mode 100644 index 0000000000..9ffdb3b498 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/recent_executions_sql.rs @@ -0,0 +1,21 @@ +//! Read the recent executions of every relation in one consumption table. + +use crate::consumption::constants::{CONSUMPTION_COLUMN_TYPES, RECORDED_AT_COLUMN}; + +/// Every row of the newest `limit` executions per relation, timing each execution by its earliest row. +pub fn recent_executions_sql(qualified_name: &str, limit: usize) -> String { + let columns: Vec<&str> = CONSUMPTION_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .collect(); + format!( + "SELECT {columns} FROM {table} WHERE execution_id IN (SELECT execution_id FROM (\ +SELECT execution_id, ROW_NUMBER() OVER (PARTITION BY target_database, target_schema, target_name \ +ORDER BY started_at DESC, execution_id DESC) AS execution_rank FROM (SELECT execution_id, \ +target_database, target_schema, target_name, MIN({RECORDED_AT_COLUMN}) AS started_at FROM {table} \ +GROUP BY execution_id, target_database, target_schema, target_name) AS executions) AS ranked \ +WHERE execution_rank <= {limit})", + columns = columns.join(", "), + table = qualified_name, + ) +} diff --git a/crates/sqlbuild-state/src/consumption/main/started_append.rs b/crates/sqlbuild-state/src/consumption/main/started_append.rs new file mode 100644 index 0000000000..bcd8554c79 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/started_append.rs @@ -0,0 +1,24 @@ +//! The row appended before a model or seed writes. + +use crate::consumption::_helpers::decision::input_key; +use crate::consumption::_helpers::event_id::event_id; +use crate::consumption::constants::STARTED_PHASE; +use crate::consumption::models::{ConsumptionAppend, NodeFacts}; + +/// The `started` row of one execution of `node`; its input count is the node's distinct direct inputs. +pub fn started_append(node: &NodeFacts, execution_id: &str, run_id: &str) -> ConsumptionAppend { + let mut keys: Vec<(&str, String)> = node.inputs.iter().map(input_key).collect(); + keys.sort(); + keys.dedup(); + ConsumptionAppend { + event_id: event_id(execution_id, STARTED_PHASE, None), + execution_id: execution_id.to_owned(), + phase: STARTED_PHASE.to_owned(), + run_id: run_id.to_owned(), + node: node.key.clone(), + relation: node.relation.clone(), + version_hash: node.version_hash.clone(), + input_count: i64::try_from(keys.len()).unwrap_or(i64::MAX), + input: None, + } +} diff --git a/crates/sqlbuild-state/src/consumption/main/succeeded_appends.rs b/crates/sqlbuild-state/src/consumption/main/succeeded_appends.rs new file mode 100644 index 0000000000..092f796cfe --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/main/succeeded_appends.rs @@ -0,0 +1,28 @@ +//! The rows appended after a model or seed's write succeeded. + +use crate::consumption::_helpers::event_id::event_id; +use crate::consumption::constants::SUCCEEDED_PHASE; +use crate::consumption::models::{ConsumedInput, ConsumptionAppend, NodeFacts}; + +/// One `succeeded` row per consumed input, or one input-less row when there are none. +pub fn succeeded_appends( + node: &NodeFacts, + (execution_id, run_id): (&str, &str), + inputs: &[ConsumedInput], +) -> Vec { + let row = |input: Option<&ConsumedInput>| ConsumptionAppend { + event_id: event_id(execution_id, SUCCEEDED_PHASE, input), + execution_id: execution_id.to_owned(), + phase: SUCCEEDED_PHASE.to_owned(), + run_id: run_id.to_owned(), + node: node.key.clone(), + relation: node.relation.clone(), + version_hash: node.version_hash.clone(), + input_count: i64::try_from(inputs.len()).unwrap_or(i64::MAX), + input: input.cloned(), + }; + if inputs.is_empty() { + return vec![row(None)]; + } + inputs.iter().map(|input| row(Some(input))).collect() +} diff --git a/crates/sqlbuild-state/src/consumption/mod.rs b/crates/sqlbuild-state/src/consumption/mod.rs new file mode 100644 index 0000000000..f01181f853 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-state/src/consumption/models.rs b/crates/sqlbuild-state/src/consumption/models.rs new file mode 100644 index 0000000000..d9f9c96cba --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/models.rs @@ -0,0 +1,122 @@ +//! Consumption rows, the per-node facts `--changed` decides from, and its run reasons. + +/// A node identity: its resource kind, such as `model` or `seed`, and its name. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct NodeKey { + pub kind: String, + pub name: String, +} + +/// The relation a node writes, as the state table keys it. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct RelationKey { + pub database: Option, + pub schema: String, + pub name: String, +} + +/// One input an execution consumed: its kind, name and the version it read. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct ConsumedInput { + pub kind: String, + pub name: String, + pub version: String, +} + +/// One row to append; the warehouse clock fills `recorded_at`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConsumptionAppend { + pub event_id: String, + pub execution_id: String, + pub phase: String, + pub run_id: String, + pub node: NodeKey, + pub relation: RelationKey, + pub version_hash: String, + pub input_count: i64, + pub input: Option, +} + +/// One row read back, with its warehouse time in microseconds since the epoch. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConsumptionRow { + pub row: ConsumptionAppend, + pub recorded_at: i64, +} + +/// A direct input of a node as this build sees it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum InputFact { + /// A source and its observed `data_version_hash`, or None when unknown. + Source { + name: String, + version: Option, + }, + /// An upstream model or seed. + Node(NodeKey), + /// An input whose version cannot be known, such as a Python node. + Opaque(String), +} + +/// Why the planner already runs a node, whatever its records say. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ForcedRun { + /// The planner detected a change of the node itself, named by its change kind. + PlannerChange(String), + FullRefresh, + RelationMissing, + CursorInterval, + /// The compiler cannot enumerate what the node reads. + InputsNotEnumerable, +} + +/// One node of the selection or its ancestry, as `--changed` decides from it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NodeFacts { + pub key: NodeKey, + pub relation: RelationKey, + pub version_hash: String, + pub selected: bool, + pub forced: Vec, + pub inputs: Vec, +} + +/// Why `--changed` runs a node. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RunReason { + Forced(ForcedRun), + NoRecord, + Unfinished, + AmbiguousOrder, + DefinitionChanged, + InputsChanged, + InputChanged(String), + InputUnknown(String), + UpstreamRuns(NodeKey), +} + +/// One execution of a relation rebuilt from its deduplicated rows. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Execution { + pub execution_id: String, + /// The earliest warehouse time of its rows. + pub started_at: i64, + /// Whether its `succeeded` rows are present, consistent and complete. + pub complete: bool, + pub version_hash: Option, + /// Input versions of a complete execution, keyed by `(kind, name)`. + pub inputs: Vec, +} + +/// The recent executions of each relation, newest first, as `--changed` reads them. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ConsumptionHistory { + pub(crate) relations: std::collections::HashMap, +} + +/// One relation's executions, newest first, and whether older ones were left unread. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct RelationHistory { + pub(crate) executions: Vec, + pub(crate) truncated: bool, +} diff --git a/crates/sqlbuild-state/src/consumption/tests.rs b/crates/sqlbuild-state/src/consumption/tests.rs new file mode 100644 index 0000000000..5ba8b6e3a0 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_simulation.rs"] +mod test_simulation; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-state/src/consumption/tests/helpers.rs b/crates/sqlbuild-state/src/consumption/tests/helpers.rs new file mode 100644 index 0000000000..04d74a8125 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/tests/helpers.rs @@ -0,0 +1,577 @@ +//! A seeded simulation of builds against a warehouse, with an oracle that knows what each model read. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use crate::consumption::_helpers::decision::decide; +use crate::consumption::_helpers::history::build_history; +use crate::consumption::_helpers::rules::DecisionRules; +use crate::consumption::constants::RECENT_EXECUTIONS; +use crate::consumption::main::consumed_inputs::consumed_inputs; +use crate::consumption::main::started_append::started_append; +use crate::consumption::main::succeeded_appends::succeeded_appends; +use crate::consumption::models::{ + ConsumptionAppend, ConsumptionHistory, ConsumptionRow, ForcedRun, InputFact, NodeFacts, + NodeKey, RelationKey, RunReason, +}; +use crate::consumption::tests::test_types::{ + Content, Derivation, Input, LatestRowTime, Miss, NoPlannerGuard, NoUpstreamRecursion, Node, + Random, Source, TransitiveOnly, World, Writer, +}; + +impl Random { + pub(crate) fn new(seed: u64) -> Self { + Self(seed) + } + + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z: u64 = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + fn below(&mut self, bound: u64) -> u64 { + self.next() % bound + } + + fn chance(&mut self, percent: u64) -> bool { + self.below(100) < percent + } +} + +const SECOND: i64 = 1_000_000; +const SIMULATED_CASES: u64 = 400; +const SIMULATED_BUILDS: usize = 14; + +impl World { + pub(crate) fn new(random: &mut Random) -> Self { + let source_count: usize = 2 + usize::try_from(random.below(2)).unwrap_or(0); + let node_count: usize = 5 + usize::try_from(random.below(5)).unwrap_or(0); + let sources: Vec = (0..source_count) + .map(|_| Source { + version: 1, + observed: random.chance(85), + monotonic: random.chance(50), + }) + .collect(); + let mut nodes: Vec = Vec::new(); + for index in 0..node_count { + let mut inputs: BTreeSet = BTreeSet::new(); + for source in 0..source_count { + if random.chance(35) { + inputs.insert(Input::Source(source)); + } + } + for upstream in 0..index { + if random.chance(40) { + inputs.insert(Input::Node(upstream)); + } + } + nodes.push(Node { + inputs: inputs.into_iter().collect(), + definition: 1, + schema: usize::try_from(random.below(2)).unwrap_or(0), + nondeterministic: random.chance(25), + enumerable: random.chance(92), + content: Content::Missing, + fingerprint: None, + }); + } + Self { + sources, + nodes, + rows: Vec::new(), + clock: 1_000 * SECOND, + nonce: 0, + flushed: Vec::new(), + trace: Vec::new(), + } + } + + fn key(index: usize) -> NodeKey { + NodeKey { + kind: "model".to_owned(), + name: format!("m{index}"), + } + } + + fn relation(&self, index: usize) -> RelationKey { + RelationKey { + database: None, + schema: format!("s{}", self.nodes[index].schema), + name: format!("m{index}"), + } + } + + fn content_id(&self, input: Input) -> u64 { + match input { + Input::Source(source) => self.sources[source].version, + Input::Node(node) => match &self.nodes[node].content { + Content::Missing => 0, + Content::Partial => 1, + Content::Data(derivation) => 1_000 + derivation.nonce, + }, + } + } + + fn fresh_own(&self, index: usize) -> bool { + let node: &Node = &self.nodes[index]; + match &node.content { + Content::Data(derivation) => { + derivation.definition == node.definition + && derivation + .inputs + .iter() + .all(|(input, read)| self.content_id(*input) == *read) + } + _ => false, + } + } + + /// Fresh and every upstream, transitively, fresh: what a skip requires. + fn fresh_closure(&self, index: usize) -> bool { + self.fresh_own(index) + && self.nodes[index].inputs.iter().all(|input| match input { + Input::Node(upstream) => self.fresh_closure(*upstream), + Input::Source(_) => true, + }) + } + + fn change_sources(&mut self, random: &mut Random) { + for (index, source) in self.sources.iter_mut().enumerate() { + if !random.chance(30) { + continue; + } + let previous: u64 = source.version; + source.version = if source.monotonic || random.chance(50) { + source.version + 1 + } else { + 1 + random.below(source.version.max(1)) + }; + self.trace + .push(format!("source s{index}: {previous} -> {}", source.version)); + } + } + + fn change_definitions(&mut self, random: &mut Random) { + for (index, node) in self.nodes.iter_mut().enumerate() { + if random.chance(12) { + node.definition += 1; + self.trace + .push(format!("definition m{index} -> {}", node.definition)); + } + } + } + + /// The reader's SQL: rows of the newest executions per relation by earliest row time. + fn read_rows(&self) -> Vec { + let mut started: BTreeMap<(RelationKey, String), i64> = BTreeMap::new(); + for row in &self.rows { + let entry = started + .entry((row.row.relation.clone(), row.row.execution_id.clone())) + .or_insert(row.recorded_at); + *entry = (*entry).min(row.recorded_at); + } + let mut by_relation: BTreeMap> = BTreeMap::new(); + for ((relation, execution), time) in started { + by_relation + .entry(relation) + .or_default() + .push((time, execution)); + } + let mut kept: BTreeSet = BTreeSet::new(); + for executions in by_relation.values_mut() { + executions.sort_by(|left, right| right.cmp(left)); + kept.extend( + executions + .iter() + .take(RECENT_EXECUTIONS) + .map(|(_, execution)| execution.clone()), + ); + } + self.rows + .iter() + .filter(|row| kept.contains(&row.row.execution_id)) + .cloned() + .collect() + } + + /// The janitor's SQL: keep the newest `retain` executions per relation. + fn prune(&mut self, retain: usize) { + let mut started: BTreeMap<(RelationKey, String), i64> = BTreeMap::new(); + for row in &self.rows { + let entry = started + .entry((row.row.relation.clone(), row.row.execution_id.clone())) + .or_insert(row.recorded_at); + *entry = (*entry).min(row.recorded_at); + } + let mut by_relation: BTreeMap> = BTreeMap::new(); + for ((relation, execution), time) in started { + by_relation + .entry(relation) + .or_default() + .push((time, execution)); + } + let mut dropped: BTreeSet = BTreeSet::new(); + for executions in by_relation.values_mut() { + executions.sort_by(|left, right| right.cmp(left)); + dropped.extend( + executions + .iter() + .skip(retain) + .map(|(_, execution)| execution.clone()), + ); + } + self.rows + .retain(|row| !dropped.contains(&row.row.execution_id)); + self.trace.push(format!("prune to {retain}")); + } + + fn append(&mut self, rows: &[ConsumptionAppend]) { + for row in rows { + self.clock += SECOND / 10; + self.rows.push(ConsumptionRow { + row: row.clone(), + recorded_at: self.clock, + }); + } + } + + fn facts( + &self, + selected: &BTreeSet, + observed: &[Option], + full_refresh: bool, + ) -> Vec { + (0..self.nodes.len()) + .map(|index| { + let node: &Node = &self.nodes[index]; + let mut forced: Vec = Vec::new(); + if full_refresh { + forced.push(ForcedRun::FullRefresh); + } + if node.content == Content::Missing { + forced.push(ForcedRun::RelationMissing); + } else if node.fingerprint != Some(node.definition) { + forced.push(ForcedRun::PlannerChange("query_changed".to_owned())); + } + if !node.enumerable { + forced.push(ForcedRun::InputsNotEnumerable); + } + NodeFacts { + key: Self::key(index), + relation: self.relation(index), + version_hash: format!("d{}", node.definition), + selected: selected.contains(&index), + forced, + inputs: node + .inputs + .iter() + .map(|input| match input { + Input::Source(source) => InputFact::Source { + name: format!("s{source}"), + version: observed[*source].map(|version| format!("v{version}")), + }, + Input::Node(upstream) => InputFact::Node(Self::key(*upstream)), + }) + .collect(), + } + }) + .collect() + } + + /// One build; under `--changed` the oracle checks every skip before anything executes. + pub(crate) fn build( + &mut self, + (rules, writer): (&R, Writer), + random: &mut Random, + step: usize, + ) -> Result<(), Miss> { + self.clock += i64::try_from(random.below(20)).unwrap_or(0) * SECOND; + self.late_duplicate(random); + self.change_sources(random); + self.change_definitions(random); + let changed: bool = random.chance(60); + let selected: BTreeSet = if random.chance(50) { + (0..self.nodes.len()).collect() + } else { + (0..self.nodes.len()) + .filter(|_| random.chance(50)) + .collect() + }; + let full_refresh: bool = random.chance(5); + let observed: Vec> = self + .sources + .iter() + .map(|source| source.observed.then_some(source.version)) + .collect(); + let facts: Vec = self.facts(&selected, &observed, full_refresh); + let history: ConsumptionHistory = + build_history(rules, &self.read_rows(), RECENT_EXECUTIONS); + let decided: Vec<(NodeKey, Option)> = decide(rules, &facts, &history); + let run: Vec = (0..self.nodes.len()) + .filter(|index| selected.contains(index)) + .filter(|index| { + !changed + || decided + .iter() + .any(|(key, reason)| *key == Self::key(*index) && reason.is_some()) + }) + .collect(); + self.trace.push(format!( + "step {step}: {} select {:?} run {:?}{}", + if changed { "--changed" } else { "plain" }, + selected, + run, + if full_refresh { " full refresh" } else { "" } + )); + if changed { + for index in &selected { + if run.contains(index) { + continue; + } + if !self.fresh_closure(*index) { + return Err(Miss { + step, + node: *index, + why: "skipped a model whose data or an upstream's is stale", + }); + } + if self.nodes[*index].fingerprint != Some(self.nodes[*index].definition) { + return Err(Miss { + step, + node: *index, + why: "skipped a model with a pending replay", + }); + } + } + } + self.execute((&facts, &history), &run, (writer, random)); + if random.chance(15) { + let retain: usize = 1 + usize::try_from(random.below(3)).unwrap_or(0); + self.prune(retain); + } + Ok(()) + } + + fn execute( + &mut self, + (facts, history): (&[NodeFacts], &ConsumptionHistory), + run: &[usize], + (writer, random): (Writer, &mut Random), + ) { + let by_key: HashMap = facts + .iter() + .map(|facts| (facts.key.clone(), facts.clone())) + .collect(); + let executions: HashMap = run + .iter() + .map(|index| { + self.nonce += 1; + (Self::key(*index), format!("e{}", self.nonce)) + }) + .collect(); + let mut failed_start: BTreeSet = BTreeSet::new(); + for schema in 0..2 { + let batch: Vec = run + .iter() + .filter(|index| self.nodes[**index].schema == schema) + .map(|index| started_append(&facts[*index], &executions[&Self::key(*index)], "run")) + .collect(); + if batch.is_empty() || writer == Writer::NoStartedRow { + continue; + } + if random.chance(4) { + self.trace + .push(format!("started append fails in s{schema}")); + failed_start.extend( + run.iter() + .filter(|index| self.nodes[**index].schema == schema), + ); + continue; + } + self.append(&batch); + } + let mut done: BTreeSet = BTreeSet::new(); + let mut pending: Vec> = vec![Vec::new(), Vec::new()]; + for index in run.iter().copied() { + let blocked: bool = self.nodes[index].inputs.iter().any(|input| match input { + Input::Node(upstream) => run.contains(upstream) && !done.contains(upstream), + Input::Source(_) => false, + }); + if blocked || failed_start.contains(&index) { + continue; + } + if random.chance(5) { + self.trace.push(format!("crash while writing m{index}")); + self.nodes[index].content = Content::Partial; + return; + } + if random.chance(6) { + self.trace.push(format!("m{index} fails")); + if random.chance(50) { + self.nodes[index].content = Content::Partial; + } + continue; + } + self.late_load(random); + self.nonce += 1; + let derivation = Derivation { + definition: self.nodes[index].definition, + inputs: self.nodes[index] + .inputs + .iter() + .map(|input| (*input, self.content_id(*input))) + .collect(), + nonce: if self.nodes[index].nondeterministic { + self.nonce + } else { + self.deterministic_nonce(index) + }, + }; + self.nodes[index].content = Content::Data(derivation); + if random.chance(85) { + self.nodes[index].fingerprint = Some(self.nodes[index].definition); + } else { + self.trace + .push(format!("fingerprint write fails for m{index}")); + } + if random.chance(4) { + self.trace.push(format!("crash after writing m{index}")); + return; + } + done.insert(index); + let key: NodeKey = Self::key(index); + let inputs = consumed_inputs(&facts[index], (&by_key, history), &executions); + pending[self.nodes[index].schema].extend(succeeded_appends( + &facts[index], + (&executions[&key], "run"), + &inputs, + )); + if random.chance(40) && !self.flush(&mut pending, random) { + return; + } + } + self.flush(&mut pending, random); + } + + /// A retried append of a recent batch lands again, later, with the same event IDs. + fn late_duplicate(&mut self, random: &mut Random) { + if self.flushed.is_empty() || !random.chance(15) { + return; + } + let recent: u64 = self.flushed.len().min(3) as u64; + let batch: usize = + self.flushed.len() - 1 - usize::try_from(random.below(recent)).unwrap_or(0); + let rows: Vec = self.flushed[batch].clone(); + let retained: bool = rows.iter().all(|duplicate| { + self.rows + .iter() + .any(|row| row.row.execution_id == duplicate.execution_id) + }); + if retained { + self.trace.push(format!("late duplicate of batch {batch}")); + self.append(&rows); + } + } + + /// A deterministic model's content depends only on what it read. + fn deterministic_nonce(&self, index: usize) -> u64 { + let node: &Node = &self.nodes[index]; + let mut value: u64 = u64::from(node.definition) * 1_000_003 + index as u64; + for input in &node.inputs { + value = value.wrapping_mul(31).wrapping_add(self.content_id(*input)); + } + value % 1_000_000_000 + } + + /// A monotonic source may load new data between observation and a model's read. + fn late_load(&mut self, random: &mut Random) { + for (index, source) in self.sources.iter_mut().enumerate() { + if source.monotonic && random.chance(3) { + source.version += 1; + self.trace + .push(format!("late load of s{index} -> {}", source.version)); + } + } + } + + /// Append each schema's pending rows as one statement; a crash loses that statement whole. + fn flush(&mut self, pending: &mut [Vec], random: &mut Random) -> bool { + for batch in pending.iter_mut() { + if batch.is_empty() { + continue; + } + if random.chance(3) { + self.trace + .push("crash during the succeeded append".to_owned()); + return false; + } + let rows: Vec = std::mem::take(batch); + self.append(&rows); + self.flushed.push(rows); + } + true + } +} + +/// Run `cases` seeded worlds of `steps` builds and return the first miss, with its seed and trace. +pub(crate) fn first_miss( + (rules, writer): (&R, Writer), + (cases, steps): (u64, usize), +) -> Option<(u64, Miss, Vec)> { + (0..cases).find_map(|seed| world_miss((rules, writer), seed, steps)) +} + +/// The first miss of one seeded world. +fn world_miss( + (rules, writer): (&R, Writer), + seed: u64, + steps: usize, +) -> Option<(u64, Miss, Vec)> { + let mut random = Random::new(seed); + let mut world = World::new(&mut random); + let miss: Miss = + (0..steps).find_map(|step| world.build((rules, writer), &mut random, step).err())?; + Some((seed, miss, world.trace)) +} + +impl DecisionRules for TransitiveOnly { + fn compares_node_inputs(&self) -> bool { + false + } +} + +impl DecisionRules for LatestRowTime { + fn execution_time(&self, earliest: i64, latest: i64) -> i64 { + let _ = earliest; + latest + } +} + +impl DecisionRules for NoUpstreamRecursion { + fn follows_upstream(&self) -> bool { + false + } +} + +impl DecisionRules for NoPlannerGuard { + fn honours(&self, forced: &ForcedRun) -> bool { + !matches!(forced, ForcedRun::PlannerChange(_)) + } +} + +/// The first miss of `rules` over the standard cases, rendered for an assertion message. +pub(crate) fn found( + rules: &R, + writer: Writer, +) -> Option<(u64, String, Vec)> { + first_miss((rules, writer), (SIMULATED_CASES, SIMULATED_BUILDS)).map(|(seed, miss, trace)| { + ( + seed, + format!("step {} m{}: {}", miss.step, miss.node, miss.why), + trace, + ) + }) +} diff --git a/crates/sqlbuild-state/src/consumption/tests/test_simulation.rs b/crates/sqlbuild-state/src/consumption/tests/test_simulation.rs new file mode 100644 index 0000000000..c515a1f587 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/tests/test_simulation.rs @@ -0,0 +1,50 @@ +use crate::consumption::_helpers::rules::Shipped; +use crate::consumption::tests::helpers::found; +use crate::consumption::tests::test_types::{ + LatestRowTime, NoPlannerGuard, NoUpstreamRecursion, SimulationTestCase, TransitiveOnly, Writer, +}; + +#[test] +fn given_random_builds_crashes_and_appends_when_deciding_changed_then_only_broken_rules_miss() { + let test_cases = [ + SimulationTestCase { + description: "the shipped rules never skip a stale model or a pending replay", + miss: found(&Shipped, Writer::Shipped), + expected_miss: false, + }, + SimulationTestCase { + description: "without the started row a crash mid-write is skipped", + miss: found(&Shipped, Writer::NoStartedRow), + expected_miss: true, + }, + SimulationTestCase { + description: "comparing sources only misses a rebuilt upstream", + miss: found(&TransitiveOnly, Writer::Shipped), + expected_miss: true, + }, + SimulationTestCase { + description: "timing executions by their latest row lets a late duplicate hide a crash", + miss: found(&LatestRowTime, Writer::Shipped), + expected_miss: true, + }, + SimulationTestCase { + description: "without upstream recursion a stale unselected upstream is ignored", + miss: found(&NoUpstreamRecursion, Writer::Shipped), + expected_miss: true, + }, + SimulationTestCase { + description: "without the planner guard a pending replay is skipped", + miss: found(&NoPlannerGuard, Writer::Shipped), + expected_miss: true, + }, + ]; + for test_case in test_cases { + assert_eq!( + test_case.miss.is_some(), + test_case.expected_miss, + "{}: {:?}", + test_case.description, + test_case.miss + ); + } +} diff --git a/crates/sqlbuild-state/src/consumption/tests/test_types.rs b/crates/sqlbuild-state/src/consumption/tests/test_types.rs new file mode 100644 index 0000000000..5d74e04f01 --- /dev/null +++ b/crates/sqlbuild-state/src/consumption/tests/test_types.rs @@ -0,0 +1,92 @@ +//! Simulated worlds, the oracle's view of their data, and broken decision rules. + +use crate::consumption::models::ConsumptionAppend; +use crate::consumption::models::ConsumptionRow; + +/// SplitMix64, so every case replays from its seed. +pub(crate) struct Random(pub(crate) u64); + +/// The broken writer variant: whether `started` rows are appended at all. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Writer { + Shipped, + NoStartedRow, +} + +/// What a relation holds: nothing, a partial write, or data derived from known inputs. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Content { + Missing, + Partial, + Data(Derivation), +} + +/// Exactly what one successful write read: its definition, each input's content and a nonce. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Derivation { + pub(crate) definition: u32, + pub(crate) inputs: Vec<(Input, u64)>, + pub(crate) nonce: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum Input { + Source(usize), + Node(usize), +} + +pub(crate) struct Source { + pub(crate) version: u64, + /// Whether its freshness signal exists; without one the version is unknown. + pub(crate) observed: bool, + /// Monotonic versions only increase; others may revert to any earlier value. + pub(crate) monotonic: bool, +} + +pub(crate) struct Node { + pub(crate) inputs: Vec, + pub(crate) definition: u32, + pub(crate) schema: usize, + pub(crate) nondeterministic: bool, + pub(crate) enumerable: bool, + pub(crate) content: Content, + /// The planner's recorded definition: written best effort after a successful write. + pub(crate) fingerprint: Option, +} + +/// One simulated project and warehouse. +pub(crate) struct World { + pub(crate) sources: Vec, + pub(crate) nodes: Vec, + pub(crate) rows: Vec, + pub(crate) clock: i64, + pub(crate) nonce: u64, + pub(crate) flushed: Vec>, + pub(crate) trace: Vec, +} + +/// A skip the oracle rejects. +#[derive(Debug)] +pub(crate) struct Miss { + pub(crate) step: usize, + pub(crate) node: usize, + pub(crate) why: &'static str, +} + +/// Compares only source inputs, as if transitive sources were all a model recorded. +pub(crate) struct TransitiveOnly; + +/// Times an execution by its latest row, so a late duplicate looks newest. +pub(crate) struct LatestRowTime; + +/// Ignores stale upstreams, selected or not. +pub(crate) struct NoUpstreamRecursion; + +/// Ignores the planner's own change detection. +pub(crate) struct NoPlannerGuard; + +pub(crate) struct SimulationTestCase { + pub(crate) description: &'static str, + pub(crate) miss: Option<(u64, String, Vec)>, + pub(crate) expected_miss: bool, +} diff --git a/crates/sqlbuild-state/src/lib.rs b/crates/sqlbuild-state/src/lib.rs index 571e27e44c..d0fcba2598 100644 --- a/crates/sqlbuild-state/src/lib.rs +++ b/crates/sqlbuild-state/src/lib.rs @@ -2,6 +2,7 @@ #![forbid(unsafe_code)] +pub mod consumption; pub mod events; pub mod execution_history; pub mod freshness; diff --git a/fensu.toml b/fensu.toml index ecc2f19c8d..d70b715043 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1673,7 +1673,7 @@ paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/fingerpr reason = "Typed observation field projections intentionally mirror provenance checks while preserving different required identity fields and timestamp names. Bounds, rejection rules and reconciliation completion share the ingestion owner." [[dupes.allowlist]] -paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_janitor_events.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_microbatches.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_migrations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_old_name_views.rs"] +paths = ["crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_audit_results.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_consumption.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_janitor_events.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_microbatches.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_migrations.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_old_name_views.rs"] reason = "Typed event projections retain distinct required identities and indexed payload columns. Event persistence, fencing, atomic retries and reconciliation share the events ingestion owner; no per-table lifecycle is copied." [[targets.python.rule_exceptions]] @@ -2623,6 +2623,7 @@ module_layers = [ "compute_logs=output", "contract_command=commands", "contract_records=infra", + "consumption=plan", "contracts=project", "contracts/effective_config=front", "contracts/resource_identities=infra", @@ -2674,6 +2675,7 @@ module_layers = [ "microbatches=plan", "model_config=front", "native_host=infra", + "native_host/consumption_state=front", "native_host/model_lookup=project", "native_host/orchestration_sensors=commands", "native_host/orchestration_triggers=project", diff --git a/scripts/planner_differential/constants.py b/scripts/planner_differential/constants.py index 58d1c11d62..498c0b447b 100644 --- a/scripts/planner_differential/constants.py +++ b/scripts/planner_differential/constants.py @@ -121,7 +121,7 @@ RUN_RELATIVE_PAST_DAYS: int = 7 RUN_RELATIVE_FUTURE_DAYS: int = 400 MAPPING_MARKER: str = "__mapping__" -VOLATILE_KEY_SUFFIXES: tuple[str, ...] = ("event_id",) +VOLATILE_KEY_SUFFIXES: tuple[str, ...] = ("event_id", "execution_id") VOLATILE_KEYS: frozenset[str] = frozenset( { "invocation_time", @@ -140,6 +140,7 @@ "created_at", "updated_at", "recorded_at", + "input_version", } ) BASE64_COLUMN_SUFFIX: str = "_b64" diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index 7d2eee3bf4..5bd84c5745 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -12,6 +12,7 @@ Online: https://sqlbuild.com/docs/cli/build/ - Global options - Flags - Fast iteration +- Building only what changed - Execution order - Output - Deferred builds @@ -55,6 +56,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | | `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | | `--full-refresh` | Drop and rebuild selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model even without this flag | +| `--changed` | Build only the selected models and seeds whose definition or inputs changed since they last succeeded; see [Building only what changed](#building-only-what-changed) | | `--defer-to` | Resolve unselected model references against another target | | `--defer-sources-to` | Read managed source data from another target | | `--fail-fast` | Stop on first failure and skip remaining nodes | @@ -87,6 +89,57 @@ sqb build --no-tests --no-audits This replaces the former `sqb run` command. The full lifecycle (tests + audits) is always the default; skip flags opt out of specific phases when you need speed. +## Building only what changed + +`--changed` skips each selected model or seed that would read exactly what it read the last time +it built successfully: + +```bash +sqb build --changed +``` + +``` +Changed: 2 of 3 selected models and seeds run; 1 unchanged are skipped. +``` + +Add `--verbose` to print why each one runs, such as `input raw_orders changed`. + +A selected model or seed runs when any of these is true: + +| Reason | Example | +|--------|---------| +| It has no successful build on record | A new model | +| Its last build did not finish | The build was killed or failed after the model started writing | +| Its relation is missing, or `--full-refresh` or a cursor override applies | `sqb build --changed --full-refresh` | +| Its definition changed | Edited SQL, config, hooks, contract or a function it calls | +| A source it reads has a different [freshness](../concepts/sources.md#source-freshness) value | New rows raise `MAX(loaded_at)` | +| An upstream model or seed was rebuilt since, or runs in this build | `stg_orders` ran, so `customer_orders` runs | +| An upstream model or seed needs to run, even when it is not selected | `sqb build customer_orders --changed` runs `customer_orders` on every build until a changed `stg_orders` is built | +| What it reads can't be listed | SQL that reads relations not declared as sources or models, or a Python task or asset upstream | + +A model that runs under `--changed` plans exactly as in a plain build: its +[`replay_on_change`](../concepts/planning/replay-decisions.md) policy still decides how much +history an incremental model reprocesses. A model that runs only because an upstream was rebuilt +takes its normal incremental step. + +In a scheduled job, set `settings={'changed': True}`. Skipped models count as successful nodes of +the run. + +Every build, with or without `--changed`, records what each model and seed read in +`_sqlbuild_consumption` in its target schema, so the build needs permission to create and insert +into that table. [`sqb janitor`](janitor.md) keeps the newest +`direct_state_history_versions` builds of each relation. + +Limitations: + +- A source is compared only through its freshness value. A change that doesn't move it, such as an + update that keeps the old `loaded_at`, isn't seen. A source without freshness configuration counts + as changed on every build. +- SQL that depends on the time, such as `current_date`, or on random values never counts as changed. + Build those models without `--changed`. +- Two builds of one model less than 5 seconds apart can't be ordered, so its downstream models run + on the next `--changed` build. + ## Execution order 1. Managed sources are loaded (unless `--no-load`, which audits their existing data instead) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md index 0f416c4451..998306878c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md @@ -160,8 +160,9 @@ like a review schema is left alone. ### State history pruning The janitor prunes old rows from `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness`, keeping -the latest `direct_state_history_versions` rows per identity (default `20`). This is separate from -relation retention. +the latest `direct_state_history_versions` rows per identity (default `20`), and from +`_sqlbuild_consumption`, keeping the latest `direct_state_history_versions` builds of each model and +seed. This is separate from relation retention. ## Safety diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 9a67ec2de1..9fa72d06df 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -809,7 +809,7 @@ keep_in_use = false | `archive_retention_days` | `14` | How many days an archive is kept before it is deleted. `0` deletes it in the same run. | | `delete_tracked_only` | `true` | Only clean relations that appear in fingerprint tracking | | `exclude_patterns` | `[]` | Glob patterns for relations to skip | -| `direct_state_history_versions` | `20` | How many state-history rows to keep per identity in `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness` | +| `direct_state_history_versions` | `20` | How many state-history rows to keep per identity in `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness`, and how many builds of each model and seed to keep in `_sqlbuild_consumption` | | `in_use_window` | `"7d"` | How far back the janitor looks for reads of each relation it would archive. A fixed duration such as `"7d"` or `"36h"`. See [Relations in use](../cli/janitor.md#relations-in-use) | | `keep_in_use` | `false` | Keep relations read within `in_use_window` instead of archiving them. `--keep-in-use` turns it on for one run | diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8a67d7d699..9bba402174 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -184,6 +184,7 @@ COMMAND_OUTPUT_LOSS_RECORD_TYPE: str INTEGRATION_RESULT_PATH_ENV: str FINGERPRINT_TABLE_NAME: str SOURCE_FRESHNESS_TABLE_NAME: str +CONSUMPTION_TABLE_NAME: str MIGRATION_TABLE_NAME: str COLUMN_MIGRATION_TABLE_NAME: str OLD_NAME_VIEW_TABLE_NAME: str @@ -5455,6 +5456,20 @@ def evaluate_source_freshness_age_policy( def build_source_freshness_read_latest_sql( *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] ) -> str: ... +def build_consumption_create_table_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], +) -> str: ... +def build_consumption_prune_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + retain_versions: int, +) -> str: ... def build_source_freshness_create_table_sql( *, database: str | None, @@ -6452,6 +6467,7 @@ class BuildCommandRequest(NativeRecord): no_color: bool fail_fast: bool full_refresh: bool + changed: bool load_sources: bool | None reload_sources: bool include_python: bool @@ -6490,6 +6506,7 @@ class BuildCommandRequest(NativeRecord): no_color: bool = False, fail_fast: bool = False, full_refresh: bool = False, + changed: bool = False, load_sources: bool | None = None, reload_sources: bool = False, include_python: bool = True, diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 683da84321..dfc315a9e2 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -2285,6 +2285,34 @@ def render_create_source_freshness_index_sqls( del database, schema return () + def render_create_consumption_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the consumption table when it is missing.""" + + from sqlbuild._native import build_consumption_create_table_sql + + return build_consumption_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + + def render_create_consumption_index_sqls( + self, + *, + database: str | None, + schema: str, + ) -> tuple[str, ...]: + """Render optional consumption table index DDL statements.""" + + del database, schema + return () + + def render_current_utc_timestamp(self) -> str: + """Render the warehouse clock as a UTC timestamp expression.""" + + return "CURRENT_TIMESTAMP" + def render_insert_source_freshness_records_sql( self, *, @@ -2432,6 +2460,18 @@ def render_prune_source_freshness_history_sql( del database, schema, retain_versions raise AdapterUserError(message="direct-state history pruning is adapter-specific") + def render_prune_consumption_history_sql( + self, + *, + database: str | None, + schema: str, + retain_versions: int, + ) -> str: + """Render SQL that prunes old consumption executions.""" + + del database, schema, retain_versions + raise AdapterUserError(message="direct-state history pruning is adapter-specific") + def sql_analysis_dialect(self) -> str | None: """Return the configured SQL analysis dialect name, if any.""" diff --git a/src/sqlbuild/adapter/contract/classes/consumption.py b/src/sqlbuild/adapter/contract/classes/consumption.py new file mode 100644 index 0000000000..4d405b1bc7 --- /dev/null +++ b/src/sqlbuild/adapter/contract/classes/consumption.py @@ -0,0 +1,45 @@ +"""Default adapter implementations for the consumption-record state table.""" + +from __future__ import annotations + +from typing import Any, cast + + +class ConsumptionMixin: + """Provide portable consumption-table DDL and pruning for first-class warehouse adapters.""" + + def render_create_consumption_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the consumption table when it is missing.""" + + from sqlbuild._native import build_consumption_create_table_sql + + adapter: Any = cast(Any, self) + return build_consumption_create_table_sql( + database=database, + schema=schema, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + ) + + def render_create_consumption_index_sqls( + self, *, database: str | None, schema: str + ) -> tuple[str, ...]: + """Return no index DDL unless the warehouse adapter explicitly opts in.""" + + del database, schema + return () + + def render_prune_consumption_history_sql( + self, *, database: str | None, schema: str, retain_versions: int + ) -> str: + """Render SQL deleting whole executions beyond the newest `retain_versions` per relation.""" + + from sqlbuild._native import build_consumption_prune_sql + + adapter: Any = cast(Any, self) + return build_consumption_prune_sql( + database=database, + schema=schema, + render_qualified_name=adapter.render_qualified_name, + retain_versions=retain_versions, + ) diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 119592a31a..1dde873e0c 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -341,6 +341,9 @@ def render_replace_table_from_relation( def render_current_timestamp(self) -> str: return "CURRENT_TIMESTAMP" + def render_current_utc_timestamp(self) -> str: + return "CAST(get_current_timestamp() AT TIME ZONE 'UTC' AS TIMESTAMP)" + def drop_view( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index ae849af077..3504c8f4c4 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -1034,6 +1034,26 @@ def render_create_source_freshness_index_sqls( """Render optional source freshness table index DDL statements.""" ... + @abstractmethod + def render_create_consumption_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the consumption table when it is missing.""" + ... + + @abstractmethod + def render_create_consumption_index_sqls( + self, + *, + database: str | None, + schema: str, + ) -> tuple[str, ...]: + """Render optional consumption table index DDL statements.""" + ... + + @abstractmethod + def render_current_utc_timestamp(self) -> str: + """Render the warehouse clock as a UTC timestamp expression.""" + ... + @abstractmethod def render_insert_source_freshness_records_sql( self, @@ -1116,6 +1136,17 @@ def render_prune_source_freshness_history_sql( """Render SQL that prunes old source freshness history rows.""" ... + @abstractmethod + def render_prune_consumption_history_sql( + self, + *, + database: str | None, + schema: str, + retain_versions: int, + ) -> str: + """Render SQL that prunes old consumption executions.""" + ... + @abstractmethod def sql_analysis_dialect(self) -> str | None: """Return the SQL analysis dialect name for this adapter, if any.""" diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 7c6500412d..ac7ed14d08 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -27,6 +27,7 @@ _typed_collection_items, _typed_scalar_payload, ) +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.historical_check_snapshot_sql import ( HistoricalCheckSnapshotSql, ) @@ -148,7 +149,7 @@ from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -class BigQueryAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): +class BigQueryAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): """BigQuery adapter backed by google-cloud-bigquery.""" def get_columns_for_relations( @@ -1065,6 +1066,9 @@ def render_merge( def render_current_timestamp(self) -> str: return "CURRENT_TIMESTAMP" + def render_current_utc_timestamp(self) -> str: + return "CURRENT_TIMESTAMP()" + def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: match type_name: case LoaderLogicalType.BOOLEAN: diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 7a4bf15518..4155286972 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -29,6 +29,7 @@ _typed_scalar_payload, _validate_rectangular_typed_array, ) +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.historical_snapshot_sql import render_is_distinct_from from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql @@ -153,7 +154,7 @@ ) -class ClickHouseAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): +class ClickHouseAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): """ClickHouse adapter that builds tables aside and rewrites whole partitions.""" adapter_name: ClassVar[str] = BuiltinAdapter.CLICKHOUSE.value @@ -674,6 +675,9 @@ def render_max_cursor_at_or_before( def render_current_timestamp(self) -> str: return "now64(6, 'UTC')" + def render_current_utc_timestamp(self) -> str: + return "now64(6, 'UTC')" + def normalize_row_diff_numeric_type(self, column_type: str) -> str | None: return normalize_numeric_family(type_sql=canonical_type(column_type), dialect=None) @@ -1917,6 +1921,11 @@ def render_create_source_freshness_table_sql(self, *, database: str | None, sche super().render_create_source_freshness_table_sql(database=database, schema=schema) ) + def render_create_consumption_table_sql(self, *, database: str | None, schema: str) -> str: + return _state_table_ddl( + super().render_create_consumption_table_sql(database=database, schema=schema) + ) + def render_create_node_result_table_sql(self, *, database: str | None, schema: str) -> str: return _state_table_ddl( super().render_create_node_result_table_sql(database=database, schema=schema) diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index a7eae3813d..59c4438f80 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -26,6 +26,7 @@ _render_ansi_typed_scalar, _render_typed_value_list, ) +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.historical_check_snapshot_sql import ( HistoricalCheckSnapshotSql, ) @@ -140,7 +141,7 @@ from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -class DatabricksAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): +class DatabricksAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): """Databricks adapter backed by databricks-sql-connector.""" def get_columns_for_relations( @@ -851,6 +852,9 @@ def create_function( def render_current_timestamp(self) -> str: return "CURRENT_TIMESTAMP" + def render_current_utc_timestamp(self) -> str: + return "current_timestamp()" + def sql_analysis_dialect(self) -> str | None: """Return the SQL analysis dialect name for this adapter, if any.""" diff --git a/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py b/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py index 7294fd39c3..faf2a9effb 100644 --- a/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py +++ b/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py @@ -7,13 +7,14 @@ from uuid import uuid4 from sqlbuild.adapter.contract.classes.base_adapter import _quote_sql_string +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.microbatches.constants import MICROBATCH_GENERATION_COMMENT_PREFIX -class DuckDbAdapter(MicrobatchMixin, DuckDbBackedAdapter): +class DuckDbAdapter(MicrobatchMixin, ConsumptionMixin, DuckDbBackedAdapter): """DuckDB adapter allowing implicit main for targetless and named local targets.""" adapter_name: ClassVar[str] = BuiltinAdapter.DUCKDB.value diff --git a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py index 434c33a209..c3ef1b029d 100644 --- a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py +++ b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py @@ -8,6 +8,7 @@ from typing import Any, ClassVar from urllib.parse import urlencode +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.exceptions import AdapterUserError @@ -21,7 +22,7 @@ ) -class MotherDuckAdapter(MicrobatchMixin, DuckDbBackedAdapter): +class MotherDuckAdapter(MicrobatchMixin, ConsumptionMixin, DuckDbBackedAdapter): """MotherDuck adapter backed by DuckDB's MotherDuck connection support.""" adapter_name: ClassVar[str] = BuiltinAdapter.MOTHERDUCK.value diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 05bb55cd76..918ea98ffb 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -28,6 +28,7 @@ _render_typed_value_list, _validate_rectangular_typed_array, ) +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.historical_check_snapshot_sql import ( HistoricalCheckSnapshotSql, ) @@ -121,7 +122,7 @@ from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -class PostgresAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): +class PostgresAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): """PostgreSQL adapter backed by psycopg.""" def get_columns_for_relations( @@ -977,6 +978,9 @@ def render_merge( def render_current_timestamp(self) -> str: return "CURRENT_TIMESTAMP" + def render_current_utc_timestamp(self) -> str: + return "(now() AT TIME ZONE 'UTC')" + def create_table_as( self, *, @@ -1706,6 +1710,21 @@ def render_create_source_freshness_index_sqls( "observed_at DESC, run_id DESC)", ) + def render_create_consumption_index_sqls( + self, *, database: str | None, schema: str + ) -> tuple[str, ...]: + from sqlbuild._native import CONSUMPTION_TABLE_NAME + + table_name: str | None = self.render_qualified_name( + database=database, schema=schema, name=CONSUMPTION_TABLE_NAME + ) + if table_name is None: + return () + return ( + "CREATE INDEX IF NOT EXISTS _sqlbuild_consumption_relation_idx " + f"ON {table_name} (target_schema, target_name, recorded_at)", + ) + def render_insert_source_freshness_records_sql( self, *, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index c66c388a16..5a5990ea81 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -29,6 +29,7 @@ _render_ansi_typed_scalar, _render_typed_value_list, ) +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.historical_check_snapshot_sql import ( HistoricalCheckSnapshotSql, ) @@ -195,7 +196,7 @@ _BULK_COLUMN_RELATION_CHUNK_SIZE: int = 2_000 -class SnowflakeAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): +class SnowflakeAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): """Snowflake adapter backed by snowflake-connector-python.""" adapter_name: ClassVar[str] = BuiltinAdapter.SNOWFLAKE.value @@ -649,6 +650,15 @@ def render_create_source_freshness_index_sqls( del database, schema return () + def render_create_consumption_table_sql(self, *, database: str | None, schema: str) -> str: + return self._with_state_table_retention( + create_sql=super().render_create_consumption_table_sql( + database=database, schema=schema + ), + database=database, + schema=schema, + ) + def render_create_microbatch_state_table_sql(self, *, database: str | None, schema: str) -> str: return self._with_state_table_retention( create_sql=super().render_create_microbatch_state_table_sql( @@ -1247,6 +1257,9 @@ def render_merge( def render_current_timestamp(self) -> str: return "CURRENT_TIMESTAMP" + def render_current_utc_timestamp(self) -> str: + return "SYSDATE()" + def create_table_as( self, *, diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py index a04a722fec..f52330e32f 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py @@ -33,6 +33,7 @@ _render_explicit_source_relation_cast_subquery, _render_typed_value_list, ) +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.historical_check_snapshot_sql import ( HistoricalCheckSnapshotSql, ) @@ -164,7 +165,7 @@ _FALSE_SEED_VALUES: frozenset[str] = frozenset({"false", "f", "no", "n", "0"}) -class SqliteAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): +class SqliteAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): """SQLite adapter: one database file, SQLBuild schemas stored as `schema__name` tables.""" adapter_name: ClassVar[str] = BuiltinAdapter.SQLITE.value @@ -988,6 +989,9 @@ def render_current_timestamp(self) -> str: return "strftime('%Y-%m-%d %H:%M:%f', 'now')" + def render_current_utc_timestamp(self) -> str: + return "strftime('%Y-%m-%dT%H:%M:%f', 'now')" + def create_table_as( self, *, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 7bafe69563..2a6dab69f1 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -28,6 +28,7 @@ _render_typed_value_list, _typed_scalar_payload, ) +from sqlbuild.adapter.contract.classes.consumption import ConsumptionMixin from sqlbuild.adapter.contract.classes.historical_check_snapshot_sql import ( HistoricalCheckSnapshotSql, ) @@ -130,7 +131,7 @@ def _render_tsql_distinct_condition(*, left: str, right: str) -> str: ) -class SqlServerAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): +class SqlServerAdapter(MicrobatchMixin, ConsumptionMixin, UnkeyedDiffMixin, BaseAdapter): """Microsoft SQL Server adapter backed by pymssql.""" def get_columns_for_relations( @@ -528,6 +529,45 @@ def render_create_source_freshness_index_sqls( "observed_at DESC, run_id DESC)", ) + def render_create_consumption_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild._native import CONSUMPTION_TABLE_NAME + + create_sql: str = ( + super() + .render_create_consumption_table_sql(database=database, schema=schema) + .replace("CREATE TABLE IF NOT EXISTS ", "CREATE TABLE ", 1) + ) + for column in ("target_schema", "target_name"): + create_sql = create_sql.replace(f"{column} NVARCHAR(MAX)", f"{column} NVARCHAR(450)") + return self._create_table_if_missing_sql( + create_sql=create_sql, + database=database, + schema=schema, + table_name=CONSUMPTION_TABLE_NAME, + ) + + def render_create_consumption_index_sqls( + self, *, database: str | None, schema: str + ) -> tuple[str, ...]: + from sqlbuild._native import CONSUMPTION_TABLE_NAME + + table_name: str | None = self.render_qualified_name( + database=database, schema=schema, name=CONSUMPTION_TABLE_NAME + ) + if table_name is None: + return () + index_name: str = "_sqlbuild_consumption_relation_idx" + escaped_table_name: str = table_name.replace("'", "''") + return ( + "IF NOT EXISTS (" + "SELECT 1 FROM sys.indexes " + f"WHERE name = '{index_name}' " + f"AND object_id = OBJECT_ID(N'{escaped_table_name}')" + ") " + f"CREATE INDEX {index_name} ON {table_name} " + "(target_schema, target_name, recorded_at)", + ) + def render_insert_source_freshness_records_sql( self, *, @@ -2246,6 +2286,9 @@ def render_create_initial_snapshot_destination( def render_current_timestamp(self) -> str: return "CURRENT_TIMESTAMP" + def render_current_utc_timestamp(self) -> str: + return "SYSUTCDATETIME()" + def render_delete_insert( self, *, diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 06203a8fcd..0231a20d37 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -89,7 +89,7 @@ SourceAuditRunResult, SourceLoadPlanEntry, ) -from sqlbuild.executor.build.types import BeforeModelMaterializeCallback +from sqlbuild.executor.build.types import BeforeNodeWriteCallback from sqlbuild.executor.custom.models import MaterializationResult from sqlbuild.executor.functions.constants import FUNCTION_ENTRY_MISSING_CODE from sqlbuild.executor.functions.main._execute import execute_function @@ -209,9 +209,7 @@ def __init__( self._on_node_start: NodeStartCallback | None = callbacks.on_node_start self._on_node_complete: Callable[[object], None] | None = callbacks.on_node_complete self._on_progress: Callable[[str], None] | None = callbacks.on_progress - self._before_model_materialize: BeforeModelMaterializeCallback | None = ( - callbacks.before_model_materialize - ) + self._before_node_write: BeforeNodeWriteCallback | None = callbacks.before_node_write self._custom_materializations: Mapping[str, Callable[..., MaterializationResult]] = ( customizations.custom_materializations or {} ) @@ -986,6 +984,17 @@ def _execute_seed_node(self, *, key: CompiledObjectKey, connection: Any) -> Seed sqlbuild_phase="load", sqlbuild_kind="seed", ): + try: + if self._before_node_write is not None: + self._before_node_write(entry=seed_entry, connection=connection) + except Exception as error: + return SeedExecutionResult( + seed_name=seed_entry.name, + status=ExecutionStatus.FAILED, + error_code=BUILD_WORKER_FAILED_CODE, + error_message=str(error), + duration_ms=int((time.monotonic() - start) * 1000), + ) result: SeedExecutionResult = execute_seed( seed_entry=seed_entry, adapter=self._adapter, @@ -1148,6 +1157,8 @@ def _execute_model_node( ) as deadline, ): try: + if self._before_node_write is not None: + self._before_node_write(entry=model_entry, connection=connection) table_type_entry: TableTypePlanEntry | None = self._table_type_entries.get( model_entry.name ) @@ -1157,8 +1168,6 @@ def _execute_model_node( apply_table_type_conversion( entry=table_type_entry, adapter=self._adapter, connection=connection ) - if self._before_model_materialize is not None: - self._before_model_materialize(entry=model_entry, connection=connection) if ( model_entry.incremental_mode == IncrementalMode.MICROBATCH and model_entry.batch_concurrency > 1 diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index 1a0cedf690..ece162eb01 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -23,7 +23,7 @@ from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.cost.models import StatementExecutionTelemetry from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.executor.build.types import BeforeModelMaterializeCallback, BuildStatus +from sqlbuild.executor.build.types import BeforeNodeWriteCallback, BuildStatus from sqlbuild.executor.custom.models import MaterializationResult from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder @@ -107,7 +107,7 @@ class BuildCallbacks: on_batch_complete: Callable[[MicrobatchBatchProgress], None] | None = None on_node_start: NodeStartCallback | None = None on_node_complete: Callable[[object], None] | None = None - before_model_materialize: BeforeModelMaterializeCallback | None = None + before_node_write: BeforeNodeWriteCallback | None = None on_connection_start: Callable[[int], None] | None = None on_connection_complete: ConnectionElapsedCallback | None = None on_connection_error: ConnectionElapsedCallback | None = None diff --git a/src/sqlbuild/executor/build/types.py b/src/sqlbuild/executor/build/types.py index b8075be76a..109032ccc1 100644 --- a/src/sqlbuild/executor/build/types.py +++ b/src/sqlbuild/executor/build/types.py @@ -6,14 +6,16 @@ from typing import TYPE_CHECKING, Any, Protocol if TYPE_CHECKING: - from sqlbuild._native import ModelPlanEntry + from sqlbuild._native import ModelPlanEntry, SeedPlanEntry from sqlbuild.executor.scheduling.types import ExecutionStatus as ExecutionStatus from sqlbuild.runtime.contracts.types import ExecutionResourceKind as ExecutionResourceKind -class BeforeModelMaterializeCallback(Protocol): - def __call__(self, *, entry: ModelPlanEntry, connection: Any) -> None: ... +class BeforeNodeWriteCallback(Protocol): + """Called before a model or seed changes its relation; raising fails that node.""" + + def __call__(self, *, entry: ModelPlanEntry | SeedPlanEntry, connection: Any) -> None: ... class BuildStatus(StrEnum): diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index aef671c86b..a6152e378e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -5,6 +5,8 @@ import subprocess from collections.abc import Callable from dataclasses import dataclass, field +from pathlib import Path +from typing import NamedTuple @dataclass(frozen=True) @@ -1524,3 +1526,41 @@ class QuotedConfiguredColumnsE2ETestCase: cursor: str expected_history_rows: list[tuple[object, ...]] expected_activity_rows: list[tuple[object, ...]] + + +class ChangedWarehouse(NamedTuple): + """One adapter's changed-builds project: configuration, raw tables and how to query it.""" + + name: str + project_toml: str + warehouse_file: str + raw_tables_sql: tuple[str, ...] + consumption_table: str + orders_table: str + customers_table: str + slow_scan_cte: str + execute: Callable[[Path, str], list[tuple[object, ...]]] + + +class ChangedBuildStep(NamedTuple): + """One mutation of the warehouse or project followed by one build.""" + + description: str + warehouse_sql: tuple[str, ...] + project_files: dict[str, str] + arguments: tuple[str, ...] + + +@dataclass(frozen=True) +class ChangedBuildE2ETestCase: + description: str + warehouse: ChangedWarehouse + expected_ran: tuple[tuple[str, frozenset[str]], ...] + + +@dataclass(frozen=True) +class ChangedCrashE2ETestCase: + description: str + warehouse: ChangedWarehouse + expected_ran: frozenset[str] + expected_killed_rows: list[tuple[object, ...]] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index a71e643d6e..29dad07049 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -1,15 +1,26 @@ from __future__ import annotations import json +import re +import signal +import sqlite3 import subprocess +import time +from contextlib import closing +from itertools import takewhile from pathlib import Path from textwrap import dedent from typing import Any +import duckdb + from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + ChangedBuildStep, + ChangedWarehouse, DeferCloneBuildE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + REPO_ROOT, execute_duckdb, prepare_inline_project, query_duckdb, @@ -1234,3 +1245,199 @@ def target_concurrency_microbatch_files() -> dict[str, str]: "WHERE event_time >= __cursor_start() AND event_time < __cursor_end()\n" ), } + + +CHANGED_BUILDS_FIXTURE: Path = REPO_ROOT / "benchmarks/duckshop/edge_cases/changed_builds" +CHANGED_BUILDS_UDF_FIXTURE: Path = REPO_ROOT / "benchmarks/duckshop/edge_cases/changed_builds_udf" +CHANGED_BUILDS_UNQUALIFIED_FIXTURE: Path = ( + REPO_ROOT / "benchmarks/duckshop/edge_cases/changed_builds_unqualified" +) +CHANGED_ORDER_SKEW_SECONDS: float = 5.2 +_CHANGED_PROGRESS_PATTERN: re.Pattern[str] = re.compile( + r"^\s+\d+/\d+\s+\S+\s+(\S+)\s+OK\b", re.MULTILINE +) +_CHANGED_SLOW_MODEL: str = ( + 'MODEL (description "Order totals over a long synthetic scan.", materialized table);\n' + "WITH {scan}\n" + "SELECT o.customer_id, count(*) AS weight\n" + 'FROM __ref("stg_orders") AS o\n' + "CROSS JOIN scan\n" + "GROUP BY o.customer_id\n" +) + + +def _duckdb_rows(path: Path, sql: str) -> list[tuple[object, ...]]: + with duckdb.connect(str(path)) as connection: + return connection.execute(sql).fetchall() + + +def _sqlite_rows(path: Path, sql: str) -> list[tuple[object, ...]]: + with closing(sqlite3.connect(path)) as connection, connection: + return connection.execute(sql).fetchall() + + +DUCKDB_CHANGED: ChangedWarehouse = ChangedWarehouse( + name="duckdb", + project_toml=( + 'name = "changed_builds"\nadapter = "duckdb"\n\n[connection]\ndatabase = "shop.duckdb"\n' + ), + warehouse_file="shop.duckdb", + raw_tables_sql=( + "CREATE SCHEMA raw", + "CREATE TABLE raw.orders AS SELECT 1 AS order_id, 1 AS customer_id, 1 AS load_id", + "CREATE TABLE raw.customers AS SELECT 1 AS customer_id, 1 AS load_id", + ), + consumption_table="main._sqlbuild_consumption", + orders_table="raw.orders", + customers_table="raw.customers", + slow_scan_cte="scan AS (SELECT r.i FROM range(4000000000) AS r(i))", + execute=_duckdb_rows, +) +SQLITE_CHANGED: ChangedWarehouse = ChangedWarehouse( + name="sqlite", + project_toml=( + 'name = "changed_builds"\nadapter = "sqlite"\n\n' + '[connection]\npath = "shop.sqlite"\n\n[targets.dev]\nschema = "main"\n' + ), + warehouse_file="shop.sqlite", + raw_tables_sql=( + "CREATE TABLE raw__orders (order_id INTEGER, customer_id INTEGER, load_id INTEGER)", + "INSERT INTO raw__orders VALUES (1, 1, 1)", + "CREATE TABLE raw__customers (customer_id INTEGER, load_id INTEGER)", + "INSERT INTO raw__customers VALUES (1, 1)", + ), + consumption_table="_sqlbuild_consumption", + orders_table="raw__orders", + customers_table="raw__customers", + slow_scan_cte=( + "RECURSIVE scan(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM scan WHERE i < 30000000)" + ), + execute=_sqlite_rows, +) + + +def prepare_changed_project(*, tmp_path: Path, warehouse: ChangedWarehouse) -> Path: + """The DuckShop changed-builds edge case on `warehouse`, with its raw tables loaded.""" + + files: dict[str, str] = { + str(path.relative_to(CHANGED_BUILDS_FIXTURE)): path.read_text(encoding="utf-8") + for path in sorted(CHANGED_BUILDS_FIXTURE.glob("*/*")) + } + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="changed_builds", + repo_files=files | {"sqlbuild_project.toml": warehouse.project_toml}, + ) + for sql in warehouse.raw_tables_sql: + warehouse.execute(project_dir / warehouse.warehouse_file, sql) + return project_dir + + +def add_changed_overlay(project_dir: Path, fixture: Path) -> None: + """Overlay one of the DuckShop changed-builds cases on the base case.""" + + for path in fixture.rglob("*.sql"): + destination: Path = project_dir / path.relative_to(fixture) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(path.read_text(encoding="utf-8"), encoding="utf-8") + + +def run_changed_build(project_dir: Path, arguments: tuple[str, ...]) -> frozenset[str]: + """Wait out the order skew, build successfully, and return the models and seeds written.""" + + time.sleep(CHANGED_ORDER_SKEW_SECONDS) + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", *arguments), project_dir=project_dir + ) + assert result.returncode == 0, result.stdout + result.stderr + return frozenset(_CHANGED_PROGRESS_PATTERN.findall(result.stdout)) + + +def apply_changed_step( + project_dir: Path, warehouse: ChangedWarehouse, step: ChangedBuildStep +) -> None: + """Apply one step's warehouse statements and project edits.""" + + for sql in step.warehouse_sql: + warehouse.execute(project_dir / warehouse.warehouse_file, sql) + for relative_path, contents in step.project_files.items(): + (project_dir / relative_path).write_text(contents, encoding="utf-8") + + +def kill_changed_build_mid_write( + project_dir: Path, warehouse: ChangedWarehouse +) -> tuple[frozenset[str], list[tuple[object, ...]]]: + """Kill a build while a slow model writes; return what it finished and the slow model's rows.""" + + slow_model: Path = project_dir / "models/order_weights.sql" + slow_model.write_text( + _CHANGED_SLOW_MODEL.format(scan=warehouse.slow_scan_cte), encoding="utf-8" + ) + time.sleep(CHANGED_ORDER_SKEW_SECONDS) + build: subprocess.Popen[str] = subprocess.Popen( + ( + str(REPO_ROOT / ".venv/bin/sqb"), + "--no-color", + "--project-dir", + str(project_dir), + "build", + "--concurrency", + "1", + ), + cwd=REPO_ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + assert build.stdout is not None + output: str = "".join(takewhile(lambda line: "order_weights START" not in line, build.stdout)) + time.sleep(0.5) + build.send_signal(signal.SIGKILL) + build.wait(timeout=30) + build.stdout.close() + slow_model.unlink() + rows: list[tuple[object, ...]] = warehouse.execute( + project_dir / warehouse.warehouse_file, + f"SELECT node_name, phase FROM {warehouse.consumption_table} " + "WHERE node_name = 'order_weights'", + ) + return frozenset(_CHANGED_PROGRESS_PATTERN.findall(output)), rows + + +def changed_build_steps(warehouse: ChangedWarehouse) -> tuple[ChangedBuildStep, ...]: + """Mutations and builds whose `--changed` run sets follow from consumed inputs alone.""" + + return ( + ChangedBuildStep("first --changed build", (), {}, ("--changed",)), + ChangedBuildStep("nothing changed", (), {}, ("--changed",)), + ChangedBuildStep( + "a new order", + (f"INSERT INTO {warehouse.orders_table} VALUES (2, 1, 2)",), + {}, + ("--changed",), + ), + ChangedBuildStep( + "a definition change", + (), + { + "models/stg_customers.sql": ( + 'MODEL (description "Staged customers.", materialized table);\n' + 'SELECT customer_id FROM __source("raw_customers") WHERE customer_id > 0\n' + ) + }, + ("--changed",), + ), + ChangedBuildStep( + "a stale unselected upstream", + (f"INSERT INTO {warehouse.customers_table} VALUES (2, 2)",), + {}, + ("--changed", "--select", "customer_orders"), + ), + ChangedBuildStep( + "the upstream is still stale", (), {}, ("--changed", "--select", "customer_orders") + ), + ChangedBuildStep("building the whole selection", (), {}, ("--changed",)), + ChangedBuildStep("a plain build", (), {}, ()), + ChangedBuildStep("after the plain build", (), {}, ("--changed",)), + ChangedBuildStep("full refresh", (), {}, ("--changed", "--full-refresh")), + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py new file mode 100644 index 0000000000..0578a7d699 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py @@ -0,0 +1,193 @@ +"""`sqb build --changed` runs exactly what consumed changed inputs, across crashes, on DuckDB and SQLite.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + ChangedBuildE2ETestCase, + ChangedCrashE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( + CHANGED_BUILDS_UDF_FIXTURE, + CHANGED_BUILDS_UNQUALIFIED_FIXTURE, + DUCKDB_CHANGED, + SQLITE_CHANGED, + add_changed_overlay, + apply_changed_step, + changed_build_steps, + kill_changed_build_mid_write, + prepare_changed_project, + run_changed_build, +) + +EVERY_MODEL: frozenset[str] = frozenset({"stg_orders", "stg_customers", "customer_orders"}) +# Functions deploy on every build; `--changed` decides models and seeds only. +FUNCTIONS: frozenset[str] = frozenset( + { + "udf__order_key", + "table_fn__customer_order_keys", + "udf__raw_customer_count", + "table_fn__raw_customers_from", + } +) +# Their functions read raw.customers without declaring it, so they run on every build. +UNDECLARED_READERS: frozenset[str] = frozenset({"customers_via_table_fn", "customer_count_via_udf"}) +EXPECTED_CHANGED_RUNS: tuple[tuple[str, frozenset[str]], ...] = ( + ("first --changed build", EVERY_MODEL), + ("nothing changed", frozenset()), + ("a new order", frozenset({"stg_orders", "customer_orders"})), + ("a definition change", frozenset({"stg_customers", "customer_orders"})), + ("a stale unselected upstream", frozenset({"customer_orders"})), + ("the upstream is still stale", frozenset({"customer_orders"})), + ("building the whole selection", frozenset({"stg_customers", "customer_orders"})), + ("a plain build", EVERY_MODEL), + ("after the plain build", frozenset()), + ("full refresh", EVERY_MODEL), +) + + +@pytest.mark.parametrize( + "test_case", + [ + ChangedBuildE2ETestCase( + description=warehouse.name, warehouse=warehouse, expected_ran=EXPECTED_CHANGED_RUNS + ) + for warehouse in (DUCKDB_CHANGED, SQLITE_CHANGED) + ], + ids=lambda case: case.description, +) +def test_given_input_changes_when_building_with_changed_then_exactly_their_consumers_run( + test_case: ChangedBuildE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_changed_project(tmp_path=tmp_path, warehouse=test_case.warehouse) + observed: list[tuple[str, frozenset[str]]] = [] + for step in changed_build_steps(test_case.warehouse): + apply_changed_step(project_dir, test_case.warehouse, step) + observed.append((step.description, run_changed_build(project_dir, step.arguments))) + + assert tuple(observed) == test_case.expected_ran + + +@pytest.mark.parametrize( + "test_case", + [ + ChangedBuildE2ETestCase( + description="duckdb", + warehouse=DUCKDB_CHANGED, + expected_ran=( + ( + "first --changed build", + EVERY_MODEL | UNDECLARED_READERS | {"first_customer_order_keys"}, + ), + ("nothing changed", UNDECLARED_READERS), + ( + "a new order", + UNDECLARED_READERS + | {"stg_orders", "customer_orders", "first_customer_order_keys"}, + ), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_models_calling_functions_when_building_with_changed_then_only_changed_inputs_run( + test_case: ChangedBuildE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_changed_project(tmp_path=tmp_path, warehouse=test_case.warehouse) + add_changed_overlay(project_dir, CHANGED_BUILDS_UDF_FIXTURE) + observed: list[tuple[str, frozenset[str]]] = [] + for step in changed_build_steps(test_case.warehouse)[: len(test_case.expected_ran)]: + apply_changed_step(project_dir, test_case.warehouse, step) + ran: frozenset[str] = run_changed_build(project_dir, step.arguments) + observed.append((step.description, ran - FUNCTIONS)) + warehouse_file: Path = project_dir / test_case.warehouse.warehouse_file + test_case.warehouse.execute( + warehouse_file, f"INSERT INTO {test_case.warehouse.customers_table} VALUES (2, 3)" + ) + run_changed_build(project_dir, ("--changed",)) + function_reads: list[tuple[object, ...]] = test_case.warehouse.execute( + warehouse_file, + "SELECT (SELECT count(*) FROM main.customers_via_table_fn), " + "(SELECT customer_count FROM main.customer_count_via_udf)", + ) + + assert tuple(observed) == test_case.expected_ran + assert function_reads == [(2, 2)] + + +@pytest.mark.parametrize( + "test_case", + [ + ChangedBuildE2ETestCase( + description="duckdb", + warehouse=DUCKDB_CHANGED, + expected_ran=( + ("nothing changed", frozenset({"unqualified_customer_reader"})), + ("customers redefined", frozenset({"customers", "unqualified_customer_reader"})), + ("customers settled", frozenset({"unqualified_customer_reader"})), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_an_unqualified_name_matching_two_relations_when_building_with_changed_then_it_runs( + test_case: ChangedBuildE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_changed_project(tmp_path=tmp_path, warehouse=test_case.warehouse) + add_changed_overlay(project_dir, CHANGED_BUILDS_UNQUALIFIED_FIXTURE) + customers_model: Path = project_dir / "models/customers.sql" + run_changed_build(project_dir, ("--select", "+customers")) + run_changed_build(project_dir, ("--changed",)) + nothing_changed: frozenset[str] = run_changed_build(project_dir, ("--changed",)) + customers_model.write_text( + customers_model.read_text(encoding="utf-8").replace( + '__ref("stg_customers")', '__ref("stg_customers") WHERE customer_id > 100' + ), + encoding="utf-8", + ) + redefined: frozenset[str] = run_changed_build(project_dir, ("--changed",)) + # Nothing orders the reader after the undeclared `customers`, so it may read it before the + # rebuild; it runs on every `--changed` build, so the next one reads the settled model. + settled: frozenset[str] = run_changed_build(project_dir, ("--changed",)) + modelled: list[tuple[object, ...]] = test_case.warehouse.execute( + project_dir / test_case.warehouse.warehouse_file, + "SELECT modelled FROM main.unqualified_customer_reader", + ) + + assert ( + ("nothing changed", nothing_changed), + ("customers redefined", redefined), + ("customers settled", settled), + ) == test_case.expected_ran + assert modelled == [(0,)] + + +@pytest.mark.parametrize( + "test_case", + [ + ChangedCrashE2ETestCase( + description=warehouse.name, + warehouse=warehouse, + expected_ran=EVERY_MODEL, + expected_killed_rows=[("order_weights", "started")], + ) + for warehouse in (DUCKDB_CHANGED, SQLITE_CHANGED) + ], + ids=lambda case: case.description, +) +def test_given_a_build_killed_mid_write_when_building_with_changed_then_every_interrupted_model_runs( + test_case: ChangedCrashE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_changed_project(tmp_path=tmp_path, warehouse=test_case.warehouse) + first: frozenset[str] = run_changed_build(project_dir, ("--changed",)) + settled: frozenset[str] = run_changed_build(project_dir, ("--changed",)) + finished, killed_rows = kill_changed_build_mid_write(project_dir, test_case.warehouse) + + assert (first, settled) == (test_case.expected_ran, frozenset()) + assert "stg_orders" in finished + assert killed_rows == test_case.expected_killed_rows + assert run_changed_build(project_dir, ("--changed",)) == test_case.expected_ran + assert run_changed_build(project_dir, ("--changed",)) == frozenset() diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index f4896cd156..a7c21d7385 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -77,12 +77,13 @@ def test_given_default_config_when_running_janitor_then_it_reports_disabled( expected_stdout_fragments=( "relations to archive 1", "archives to delete 0", - "objects skipped 4", + "objects skipped 5", "main.janitor_tracked_extra -> main._sqb_archive__", "main.janitor_untracked_extra relation is not tracked by SQLBuild", "main.partition_state relation matches exclude pattern 'partition_*'", "main._sqlbuild_fingerprints relation matches exclude pattern", "main._sqlbuild_microbatches relation matches exclude pattern", + "main._sqlbuild_consumption relation matches exclude pattern", "Archived 1 relation.", ), expected_existing_tables=( @@ -92,6 +93,7 @@ def test_given_default_config_when_running_janitor_then_it_reports_disabled( "_sqlbuild_fingerprints", "_sqlbuild_microbatches", "_sqlbuild_janitor_events", + "_sqlbuild_consumption", ), expected_missing_tables=("janitor_tracked_extra",), expected_archived_original_names=("janitor_tracked_extra",), @@ -315,7 +317,7 @@ def test_given_source_in_managed_schema_when_running_direct_janitor_then_blocks_ expected_exit_code=0, expected_stdout_fragments=( "relations to archive 2", - "objects skipped 2", + "objects skipped 3", "main.__sqb_a13f09c2e7b8__model__daily_revenue -> main._sqb_archive__", "main.__sqb_a13f09c2e7b8__source__raw_orders -> main._sqb_archive__", "main.__sqb_a13f09c2e7b__model__daily_revenue relation is not tracked by SQLBuild", @@ -391,11 +393,12 @@ def test_given_scenario_artifacts_when_running_direct_janitor_then_it_archives_t plan_command=("--no-color", "plan"), expected_exit_code=0, expected_stdout_fragments=( - "direct state pruned 2", + "direct state pruned 3", "Eligible direct state pruning", "main._sqlbuild_fingerprints keep latest 2", "main._sqlbuild_source_freshness keep latest 2", - "Pruned 2 direct state tables.", + "main._sqlbuild_consumption keep latest 2", + "Pruned 3 direct state tables.", ), expected_fingerprint_count_before=5, expected_fingerprint_count_after=3, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py index 22d535943f..f7f1c0d7f6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_progress.py @@ -62,10 +62,11 @@ "Deleting 1 expired archive...", " 1/1 deleted main._sqb_archive__20200101t000000z__old_products OK", "Deleted 1 expired archive. (", - "Pruning 1 direct state table...", - " 1/1 pruned main._sqlbuild_fingerprints OK", - "Pruned 1 direct state table. (", - "\u2713 Janitor completed archived 12, deleted 1, pruned 1 (", + "Pruning 2 direct state tables...", + " 1/2 pruned main._sqlbuild_fingerprints OK", + " 2/2 pruned main._sqlbuild_consumption OK", + "Pruned 2 direct state tables. (", + "\u2713 Janitor completed archived 12, deleted 1, pruned 2 (", ), expected_archive_events=12, unexpected_fragments=( @@ -80,8 +81,9 @@ expected_exit_code=0, expected_ordered_lines=( " main.old_orders_011 -> main._sqb_archive__", - "Skipped objects\n main._sqlbuild_fingerprints relation matches exclude pattern", - "\u2713 Janitor completed archived 12, deleted 1, pruned 1 (", + "Skipped objects\n main._sqlbuild_consumption relation matches exclude pattern", + " main._sqlbuild_fingerprints relation matches exclude pattern", + "\u2713 Janitor completed archived 12, deleted 1, pruned 2 (", ), expected_archive_events=12, unexpected_fragments=("... and", "Skipped objects ("), @@ -146,12 +148,12 @@ def test_given_stale_relations_when_running_janitor_then_progress_streams_in_ord stale_tables=3, expected_status="completed", expected_archived=3, - expected_item_statuses=("ok",) * 5, + expected_item_statuses=("ok",) * 6, expected_stderr_fragments=( "Janitor preview default", "Archiving 3 relations...", " 1/3 archived main.old_orders_000 -> main._sqb_archive__", - "\u2713 Janitor completed archived 3, deleted 1, pruned 1", + "\u2713 Janitor completed archived 3, deleted 1, pruned 2", ), ), ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py index 75a481aa83..ba85d4120e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_targets.py @@ -162,7 +162,7 @@ def test_given_prod_old_name_view_when_previewing_early_drop_as_prod_then_view_i expected_stdout_fragments=( "Janitor preview prod", "prod.customers -> prod._sqb_archive__", - "\u2713 Janitor completed archived 1, deleted 1, pruned 2", + "\u2713 Janitor completed archived 1, deleted 1, pruned 3", ), expected_deleted_names=(EXPIRED_ARCHIVE_NAME, "customers"), expected_archived_original_names=("customers",), @@ -176,7 +176,7 @@ def test_given_prod_old_name_view_when_previewing_early_drop_as_prod_then_view_i expected_stdout_fragments=( "Janitor preview dev", "dev.customers -> dev._sqb_archive__", - "\u2713 Janitor completed archived 1, deleted 1, pruned 2", + "\u2713 Janitor completed archived 1, deleted 1, pruned 3", ), expected_deleted_names=(EXPIRED_ARCHIVE_NAME, "customers"), expected_archived_original_names=("customers",), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py index a2935276c8..2d8d724049 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py @@ -50,7 +50,7 @@ " relations to archive 3\n", "\n usage not checked: unsupported adapter\n", "Relations to archive", - "\u2713 Janitor completed archived 3, deleted 1, pruned 1", + "\u2713 Janitor completed archived 3, deleted 1, pruned 2", ), expected_archived=3, ), @@ -61,7 +61,7 @@ " usage not checked: unsupported adapter\n" " nothing kept in use: no usage could be checked, so relations in use are " "archived too\n", - "\u2713 Janitor completed archived 3, deleted 1, pruned 1", + "\u2713 Janitor completed archived 3, deleted 1, pruned 2", ), expected_archived=3, ), @@ -151,7 +151,7 @@ def test_given_duckdb_when_running_janitor_json_then_usage_check_is_not_checked( "(40 queries in 7 days); frozen since 2026-09-01, still in use\n", " main.old_orders_view queried 2026-10-21 by svc_reporting " "(14 queries in 7 days)\n", - "\u2713 Janitor completed archived 4, deleted 1, pruned 1", + "\u2713 Janitor completed archived 4, deleted 1, pruned 2", ), unexpected_fragments=( "Kept (in use)", @@ -174,7 +174,7 @@ def test_given_duckdb_when_running_janitor_json_then_usage_check_is_not_checked( "Checked relation usage: 2 of 4 in use. (", " relations to archive 2\n kept (in use) 2\n", "\nKept (in use)\n main.old_orders_000 queried 2026-10-21", - "\u2713 Janitor completed archived 2, deleted 1, pruned 1", + "\u2713 Janitor completed archived 2, deleted 1, pruned 2", ), unexpected_fragments=( "\nIn use\n", @@ -210,7 +210,7 @@ def test_given_duckdb_when_running_janitor_json_then_usage_check_is_not_checked( " kept (in use) 1\n", " main.old_orders_001 queried 2026-10-21 by alice (1 query in 36 hours); " "frozen since 2026-09-01, still in use\n", - "\u2713 Janitor completed archived 3, deleted 1, pruned 1", + "\u2713 Janitor completed archived 3, deleted 1, pruned 2", ), unexpected_fragments=("main.old_orders_001 ->",), expected_remaining=("old_orders_001",), @@ -332,7 +332,7 @@ def test_given_usage_source_when_janitor_prints_json_then_candidates_carry_in_us reads={"old_orders_000": (("alice",), 3)}, unavailable=None, expected_fragments=( - "Janitor will archive 3 and delete 2 items from default.\r\n" + "Janitor will archive 3 and delete 3 items from default.\r\n" "3 relations will be archived; 1 of them is in use.\r\n", "Janitor cancelled.", ), diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index c9866865b7..03c0519a35 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -325,3 +325,15 @@ class SchedulerCrashTestCase: fixture: str expected_anomalies: dict[str, int] expected_minimum_kills: int + + +@dataclass(frozen=True, kw_only=True) +class ChangedJobTestCase(WorkerExecutionTestCase): + description: str + project_toml: str + warehouse_file: str + consumption_table: str + raw_tables_sql: tuple[str, ...] + new_order_sql: str + execute: Callable[[Path, str], list[tuple[Any, ...]]] + expected_runs: tuple[tuple[str, frozenset[str], dict[str, int]], ...] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 28fb6f4797..6a3de302be 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -217,6 +217,32 @@ def append_microbatch_history(adapter: Any, connection: Any, fixture: dict[str, assert repeated.inserted == 0 +def append_consumption_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: + """Real consumption DDL with duplicated physical appends, as an ambiguous retry leaves.""" + _ = adapter.execute( + connection=connection, + sql=adapter.render_create_consumption_table_sql(database=None, schema="main"), + ) + columns: tuple[str, ...] = tuple(fixture["record"])[2:] + rows: list[dict[str, Any]] = [ + fixture["record"] + | {"event_id": f"{fixture['id_prefix']}-{index:03}"} + | fixture["record_overrides"].get(f"{fixture['id_prefix']}-{index:03}", {}) + for index in range(fixture["source_rows"]) + ] + [fixture["record"]] * fixture["duplicate_rows"] + literals: dict[type, Callable[[Any], str]] = { + type(None): lambda _value: "NULL", + int: str, + str: lambda value: "'" + value.removesuffix("Z").replace("'", "''") + "'", + } + for row in rows: + _ = adapter.execute( + connection=connection, + sql=f"INSERT INTO _sqlbuild_consumption ({', '.join(columns)}) VALUES " + f"({', '.join(literals[type(row[column])](row[column]) for column in columns)})", + ) + + def append_migration_history(adapter: Any, connection: Any, fixture: dict[str, Any]) -> None: """Real model-move facts, including idempotent duplicate attempts.""" data: dict[str, Any] = migration_history_data(fixture, ("origin", "destination")) @@ -1221,3 +1247,59 @@ def scheduler_crash_history(directory: Path, fixture: dict[str, Any]) -> dict[st "ticks_before_survivor": interrupted, "ticks": len(ticks), } + + +CHANGED_FIXTURE: Path = Path("benchmarks/duckshop/edge_cases/changed_builds") +CHANGED_JOB: str = ( + "from sqlbuild.orchestration import job\n" + "changed_orders = job('changed_orders', description='Build what changed.', " + "select='stg_orders stg_customers customer_orders', settings={'changed': True})\n" +) + + +def duckdb_rows(path: Path, sql: str) -> list[tuple[Any, ...]]: + """Run `sql` on a DuckDB warehouse file and return its rows.""" + with duckdb.connect(str(path)) as connection: + return connection.execute(sql).fetchall() + + +def sqlite_rows(path: Path, sql: str) -> list[tuple[Any, ...]]: + """Run and commit `sql` on a SQLite warehouse file and return its rows.""" + with closing(sqlite3.connect(path)) as connection, connection: + return connection.execute(sql).fetchall() + + +def changed_job_files(project_toml: str) -> dict[str, str]: + """The DuckShop changed-builds edge case with one scheduled `changed` job.""" + return { + str(path.relative_to(CHANGED_FIXTURE)): path.read_text() + for path in sorted(CHANGED_FIXTURE.glob("*/*")) + } | {"orchestration/orders.py": CHANGED_JOB, "sqlbuild_project.toml": project_toml} + + +def request_job_run(worker: ScheduledWorker, job: str) -> dict[str, Any]: + """Queue `job` through privileged administration and wait for its terminal run.""" + run_id: str = str(uuid4()) + requested: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "request-run", + worker.api.location_id, + job, + "--target", + "development", + "--run-id", + run_id, + "--reason", + "Build what changed", + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert requested.returncode == 0, requested.stderr + return wait_for_run(worker.api, worker.process, worker.log, run_id=run_id) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py new file mode 100644 index 0000000000..92f74bf4fe --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py @@ -0,0 +1,159 @@ +"""Scheduled jobs with `changed` run only models whose consumed inputs changed, on DuckDB and SQLite.""" + +import os +import sqlite3 +import sys +import time +from contextlib import closing +from dataclasses import replace +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ChangedJobTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + changed_job_files, + duckdb_rows, + location_process, + request_job_run, + scheduled_worker, + sqlite_rows, +) + +ORDER_SKEW_SECONDS: float = 5.2 +EXPECTED_CHANGED_RUNS: tuple[tuple[str, frozenset[str], dict[str, int]], ...] = ( + ("first run", frozenset({"stg_orders", "stg_customers", "customer_orders"}), {"success": 3}), + ("nothing changed", frozenset(), {"success": 3}), + ("a new order", frozenset({"stg_orders", "customer_orders"}), {"success": 3}), +) + + +@pytest.mark.parametrize( + "test_case", + [ + ChangedJobTestCase( + description="DuckDB job with changed runs only consumers of new orders", + model_sql="SELECT 1 AS amount", + warehouse_query="", + expected_status="success", + expected_rows=[], + project_toml=( + 'name = "orders"\nadapter = "duckdb"\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n' + '[targets.development]\nschema = "main"\n' + ), + warehouse_file="warehouse.duckdb", + consumption_table="main._sqlbuild_consumption", + raw_tables_sql=( + "CREATE SCHEMA raw", + "CREATE TABLE raw.orders AS SELECT 1 AS order_id, 1 AS customer_id, 1 AS load_id", + "CREATE TABLE raw.customers AS SELECT 1 AS customer_id, 1 AS load_id", + ), + new_order_sql="INSERT INTO raw.orders VALUES (2, 1, 2)", + execute=duckdb_rows, + expected_runs=EXPECTED_CHANGED_RUNS, + ), + ChangedJobTestCase( + description="SQLite job with changed runs only consumers of new orders", + model_sql="SELECT 1 AS amount", + warehouse_query="", + expected_status="success", + expected_rows=[], + project_toml=( + 'name = "orders"\nadapter = "sqlite"\n' + '[connections.local]\npath = "warehouse.sqlite"\n' + '[targets.development]\nschema = "main"\n' + ), + warehouse_file="warehouse.sqlite", + consumption_table="_sqlbuild_consumption", + raw_tables_sql=( + "CREATE TABLE raw__orders (order_id INTEGER, customer_id INTEGER, load_id INTEGER)", + "INSERT INTO raw__orders VALUES (1, 1, 1)", + "CREATE TABLE raw__customers (customer_id INTEGER, load_id INTEGER)", + "INSERT INTO raw__customers VALUES (1, 1)", + ), + new_order_sql="INSERT INTO raw__orders VALUES (2, 1, 2)", + execute=sqlite_rows, + expected_runs=EXPECTED_CHANGED_RUNS, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_changed_job_when_inputs_change_between_runs_then_only_their_consumers_run( + test_case: ChangedJobTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + test_case = replace(test_case, files=changed_job_files(test_case.project_toml)) + observed: list[tuple[str, frozenset[str], dict[str, int]]] = [] + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + warehouse: Path = worker.project / test_case.warehouse_file + for sql in test_case.raw_tables_sql: + test_case.execute(warehouse, sql) + recorded: set[tuple[Any, ...]] = set() + for description, mutation in [ + ("first run", ()), + ("nothing changed", ()), + ("a new order", (test_case.new_order_sql,)), + ]: + time.sleep(ORDER_SKEW_SECONDS) + for sql in mutation: + test_case.execute(warehouse, sql) + run: dict[str, Any] = request_job_run(worker, "changed_orders") + assert run["status"] == "success", worker.log.read_text() + started: set[tuple[Any, ...]] = set( + test_case.execute( + warehouse, + f"SELECT event_id, node_name FROM {test_case.consumption_table} " + "WHERE phase = 'started'", + ) + ) + observed.append( + ( + description, + frozenset(row[1] for row in started - recorded), + run["node_counts_json"], + ) + ) + recorded |= started + warehouse_events: set[str] = { + row[0] + for row in test_case.execute( + warehouse, f"SELECT event_id FROM {test_case.consumption_table}" + ) + } + log: Path = tmp_path / "location.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(worker.project), + "location", + "serve", + "--server", + worker.api.url, + "--location-id", + worker.api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": worker.api.bearer}, + ) as location: + deadline: float = time.monotonic() + 45 + mirrored: set[str] = set() + while mirrored != warehouse_events: + assert location.poll() is None, log.read_text() + assert time.monotonic() < deadline, (warehouse_events - mirrored, log.read_text()) + with closing(sqlite3.connect(worker.control_store)) as control: + mirrored = { + row[0] + for row in control.execute( + "SELECT source_event_id FROM sqb_warehouse_consumption" + ) + } + time.sleep(0.2) + + assert tuple(observed) == test_case.expected_runs diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index 8e5ce46011..90310c9bab 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -44,6 +44,7 @@ ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( append_audit_history, + append_consumption_history, append_contended_fingerprints, append_janitor_history, append_legacy_history, @@ -103,6 +104,22 @@ ("sqlite", "path", "janitor_event_history", append_janitor_history, "cleanup-run", ()), # SQLite has no supported concurrent-microbatch coordination writer. ("duckdb", "database", "microbatch_history", append_microbatch_history, "", ()), + ( + "duckdb", + "database", + "consumption_history", + append_consumption_history, + "external-build", + (), + ), + ( + "sqlite", + "path", + "consumption_history", + append_consumption_history, + "external-build", + (), + ), ( "duckdb", "database", diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py index a573ae34f9..764ad8b30d 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py @@ -20,6 +20,7 @@ PlanOutput, PlanReason, PythonHookEntry, + SeedPlanEntry, SourceEntry, SourceLoadPlanEntry, SourceWriteStrategy, @@ -218,7 +219,7 @@ def test_given_model_materialize_hook_when_build_runs_then_it_prepares_or_fails_ ), ) - def before_model_materialize(entry: ModelPlanEntry, *, connection: object) -> None: + def before_node_write(entry: ModelPlanEntry | SeedPlanEntry, *, connection: object) -> None: events.append(f"hook:{entry.name}") hook_actions: dict[bool, Callable[[], object]] = { True: lambda: (_ for _ in ()).throw(RuntimeError("hook failed")), @@ -243,7 +244,7 @@ def before_model_materialize(entry: ModelPlanEntry, *, connection: object) -> No ), callbacks=BuildCallbacks( on_node_start=lambda name, *, resource_kind: events.append(f"start:{name}"), - before_model_materialize=before_model_materialize, + before_node_write=before_node_write, ), ) loaded_rows: tuple[tuple[object, ...], ...] = fetch_rows_or_empty( diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index 8640a900ea..a9a901c12b 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -41,6 +41,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | | `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | | `--full-refresh` | Drop and rebuild selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model even without this flag | +| `--changed` | Build only the selected models and seeds whose definition or inputs changed since they last succeeded; see [Building only what changed](#building-only-what-changed) | | `--defer-to` | Resolve unselected model references against another target | | `--defer-sources-to` | Read managed source data from another target | | `--fail-fast` | Stop on first failure and skip remaining nodes | @@ -73,6 +74,57 @@ sqb build --no-tests --no-audits This replaces the former `sqb run` command. The full lifecycle (tests + audits) is always the default; skip flags opt out of specific phases when you need speed. +## Building only what changed + +`--changed` skips each selected model or seed that would read exactly what it read the last time +it built successfully: + +```bash +sqb build --changed +``` + +``` +Changed: 2 of 3 selected models and seeds run; 1 unchanged are skipped. +``` + +Add `--verbose` to print why each one runs, such as `input raw_orders changed`. + +A selected model or seed runs when any of these is true: + +| Reason | Example | +|--------|---------| +| It has no successful build on record | A new model | +| Its last build did not finish | The build was killed or failed after the model started writing | +| Its relation is missing, or `--full-refresh` or a cursor override applies | `sqb build --changed --full-refresh` | +| Its definition changed | Edited SQL, config, hooks, contract or a function it calls | +| A source it reads has a different [freshness](/docs/concepts/sources#source-freshness) value | New rows raise `MAX(loaded_at)` | +| An upstream model or seed was rebuilt since, or runs in this build | `stg_orders` ran, so `customer_orders` runs | +| An upstream model or seed needs to run, even when it is not selected | `sqb build customer_orders --changed` runs `customer_orders` on every build until a changed `stg_orders` is built | +| What it reads can't be listed | SQL that reads relations not declared as sources or models, or a Python task or asset upstream | + +A model that runs under `--changed` plans exactly as in a plain build: its +[`replay_on_change`](/docs/concepts/planning/replay-decisions) policy still decides how much +history an incremental model reprocesses. A model that runs only because an upstream was rebuilt +takes its normal incremental step. + +In a scheduled job, set `settings={'changed': True}`. Skipped models count as successful nodes of +the run. + +Every build, with or without `--changed`, records what each model and seed read in +`_sqlbuild_consumption` in its target schema, so the build needs permission to create and insert +into that table. [`sqb janitor`](/docs/cli/janitor) keeps the newest +`direct_state_history_versions` builds of each relation. + +Limitations: + +- A source is compared only through its freshness value. A change that doesn't move it, such as an + update that keeps the old `loaded_at`, isn't seen. A source without freshness configuration counts + as changed on every build. +- SQL that depends on the time, such as `current_date`, or on random values never counts as changed. + Build those models without `--changed`. +- Two builds of one model less than 5 seconds apart can't be ordered, so its downstream models run + on the next `--changed` build. + ## Execution order 1. Managed sources are loaded (unless `--no-load`, which audits their existing data instead) diff --git a/website/src/content/docs/docs/cli/janitor.mdx b/website/src/content/docs/docs/cli/janitor.mdx index 9314df64f4..5c5e080ab6 100644 --- a/website/src/content/docs/docs/cli/janitor.mdx +++ b/website/src/content/docs/docs/cli/janitor.mdx @@ -147,8 +147,9 @@ like a review schema is left alone. ### State history pruning The janitor prunes old rows from `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness`, keeping -the latest `direct_state_history_versions` rows per identity (default `20`). This is separate from -relation retention. +the latest `direct_state_history_versions` rows per identity (default `20`), and from +`_sqlbuild_consumption`, keeping the latest `direct_state_history_versions` builds of each model and +seed. This is separate from relation retention. ## Safety diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 4a521e186d..e987399e32 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -784,7 +784,7 @@ keep_in_use = false | `archive_retention_days` | `14` | How many days an archive is kept before it is deleted. `0` deletes it in the same run. | | `delete_tracked_only` | `true` | Only clean relations that appear in fingerprint tracking | | `exclude_patterns` | `[]` | Glob patterns for relations to skip | -| `direct_state_history_versions` | `20` | How many state-history rows to keep per identity in `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness` | +| `direct_state_history_versions` | `20` | How many state-history rows to keep per identity in `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness`, and how many builds of each model and seed to keep in `_sqlbuild_consumption` | | `in_use_window` | `"7d"` | How far back the janitor looks for reads of each relation it would archive. A fixed duration such as `"7d"` or `"36h"`. See [Relations in use](/docs/cli/janitor#relations-in-use) | | `keep_in_use` | `false` | Keep relations read within `in_use_window` instead of archiving them. `--keep-in-use` turns it on for one run |