From c8c819ec5952f1883658120362ad12358f899604 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sat, 29 Aug 2026 16:44:18 -0600 Subject: [PATCH 1/3] feat(planner): select lifecycle-aware summary plans --- crates/asap-aware-mapping/src/cost_model.rs | 381 ++++++- crates/asap-aware-mapping/src/lib.rs | 35 +- crates/asap-aware-mapping/src/replacement.rs | 764 ++++++++++++-- ...le.rs => summary_maintenance_lifecycle.rs} | 957 ++++++++++++++---- crates/types/src/post_asap/mod.rs | 7 +- ...le.rs => summary_maintenance_lifecycle.rs} | 20 +- 6 files changed, 1829 insertions(+), 335 deletions(-) rename crates/asap-aware-mapping/src/{lifecycle.rs => summary_maintenance_lifecycle.rs} (50%) rename crates/types/src/post_asap/{lifecycle.rs => summary_maintenance_lifecycle.rs} (50%) diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index fae0243f..8fbbb464 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -35,11 +35,14 @@ //! ## CSE sharing (issue #237, #223 stage 4) //! //! [`CseCandidate`]/[`ShareDecision`]/[`CostModel::cse_share_decision`] below -//! decide whether a CSE-detected shared subtree +//! provide the context-free fallback for whether a CSE-detected shared subtree //! ([`asap_types::pre_asap::cse::share_common_subtrees`], issue #223 stages //! 1-2, PR #235) is actually worth sharing, via a real Volcano/Cascades-style -//! cost comparison rather than a fixed rule. See -//! `docs/design_docs/cse-cost-model-decision.md` for the full design discussion (why +//! cost comparison rather than a fixed rule. Workload-aware selection uses +//! [`CostModel::cse_share_decision_with_recurrence`]; the target design also +//! expands each share candidate with its legal summary-maintenance lifecycles +//! before whole-plan ranking. See +//! `docs/design_docs/cost-model.md` for the full design discussion (why //! cost-based, why not a full plan-search engine, the layering constraint //! that forces detection to stay cost-agnostic). //! [`PlanSpace::cost_sorted`](crate::replacement::PlanSpace::cost_sorted) @@ -56,14 +59,221 @@ use asap_types::pre_asap::agg_intent::AggIntent; use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::query_expr::QueryExpr; -use crate::lifecycle::{LifecycleCostInputs, SummaryLifecycleCapabilities}; +use crate::exact_composition::{CompositionPlacement, ExactComposition}; use crate::recurrence::{ - self, Horizon, RecurrenceCostExplanation, RecurrenceError, RecurrenceProfile, + self, CostRate, EvaluationRate, Horizon, RecurrenceCostExplanation, RecurrenceError, + RecurrenceProfile, }; use crate::replacement::{ realize_child, Implementation, Replacement, ReplacementProvenance, ReplacementSubDAG, TargetSubDAG, }; +use crate::summary_maintenance_lifecycle::{ + SummaryMaintenanceCapabilities, SummaryMaintenanceLifecycleCostInputs, +}; + +// ── Recurring-cost vocabulary for mixed exact/summary plans (issue #171) ── + +/// The unit a recurring cost is expressed in. One variant today; an enum so +/// a JSON/DAG export names the unit explicitly instead of a consumer +/// assuming it, and so a future per-resource unit can be added without +/// changing every hook's signature. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum CostUnit { + /// Abstract cost units per wall-clock second — the common currency + /// every recurring alternative (maintain-and-read vs. recompute-per-eval) + /// is compared in. + CostUnitsPerSecond, +} + +impl CostUnit { + /// Stable name for export (`"cost_units_per_second"`). + pub fn as_str(self) -> &'static str { + match self { + Self::CostUnitsPerSecond => "cost_units_per_second", + } + } +} + +/// Who produced a set of [`ExactCompositionCostInputs`], and under which +/// model version — carried into every composed decision's explanation and +/// DAG export so a reviewer can tell a deployment's measured numbers from +/// a placeholder. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CostProvenance { + /// The cost model's own name (e.g. `"DefaultCostModel"`). + pub model: String, + /// The model's own version string, whatever scheme it uses. + pub version: String, +} + +/// Which mixed-execution shapes the downstream runtime can actually +/// execute (issue #171). [`crate::exact_composition::ExactCompositionStrategy`] +/// proposes an `ExactPostProcess` candidate only when +/// `exact_post_process` is set, and an `ExactTransform` candidate only +/// when `exact_update_transform` is — a runtime that cannot run an exact +/// operator on the update path must never be handed one. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct MixedExecutionCapabilities { + /// The runtime can apply an exact operator to summary readouts at + /// query evaluation time. + pub exact_post_process: bool, + /// The runtime can apply an exact row transform on the update path, + /// feeding its output into maintained summary state. + pub exact_update_transform: bool, +} + +impl MixedExecutionCapabilities { + /// Neither shape supported. + pub const NONE: Self = Self { + exact_post_process: false, + exact_update_transform: false, + }; + /// Both shapes supported. + pub const ALL: Self = Self { + exact_post_process: true, + exact_update_transform: true, + }; + + pub fn supports(self, placement: CompositionPlacement) -> bool { + match phase { + CompositionPlacement::PostProcess => self.exact_post_process, + CompositionPlacement::Transform => self.exact_update_transform, + } + } +} + +/// What [`CostModel::exact_composition_cost_inputs`] is asked about: one +/// composed alternative at one site, paired with the concrete summary it +/// composes with. +#[derive(Debug, Clone, Copy)] +pub struct ExactCompositionCostRequest<'a> { + /// The pre-ASAP target the composed candidate replaces. + pub target: &'a QueryExpr, + /// The composition itself — phase, operator, child target. + pub composition: &'a ExactComposition, + /// For [`CompositionPlacement::PostProcess`]: the child target's *selected* + /// summary readout candidate the exact operator consumes. For + /// [`CompositionPlacement::Transform`]: the maintained summary *above* the + /// transform that consumes its output (the `SummaryAgg` this transform + /// feeds). Either way, the summary whose maintenance/read cost the + /// formula charges. + pub summary: &'a SummaryNode, + /// How many times this site actually runs once ancestors' own choices + /// are accounted for (see `PlanSpace::global_selection`). + pub effective_consumer_count: usize, +} + +/// Every input the issue #171 cost formulas need, each individually +/// optional: **an unknown stays `None` — never a zero** — so a formula +/// with a missing input yields no rate at all rather than a spuriously +/// cheap one, and global selection then keeps the conservative +/// `KeepPreAsap` behavior. A deployment model that wants defaults supplies +/// them explicitly by overriding [`CostModel::exact_composition_cost_inputs`]. +#[derive(Debug, Clone, PartialEq)] +pub struct ExactCompositionCostInputs { + /// Exact operator cost per row it processes — per readout row for a + /// post-process, per input row for an update-path transform. + pub exact_cost_per_row: Option, + /// Rows the exact operator consumes per evaluation (post-process) or + /// per update (transform). + pub expected_input_rows: Option, + /// Rows the exact operator emits per evaluation/update. + pub expected_output_rows: Option, + /// Cost of one update to the composed-with summary's maintained state. + pub summary_maintenance_cost_per_update: Option, + /// Cost of one readout of that summary at evaluation time. + pub summary_read_cost: Option, + /// Update (ingest) events per second reaching this site. + pub update_rate: Option, + /// Evaluations per second across every consumer of this site. + pub evaluation_rate: Option, + /// Cost of one full raw recompute of the target from pre-ASAP data — + /// the `KeepPreAsap` baseline's per-evaluation cost. + pub raw_recompute_cost: Option, + pub unit: CostUnit, + pub provenance: CostProvenance, +} + +impl ExactCompositionCostInputs { + /// Every input unknown, attributed to `provenance` — what a model that + /// has no statistics for a site returns. + pub fn unknown(provenance: CostProvenance) -> Self { + Self { + exact_cost_per_row: None, + expected_input_rows: None, + expected_output_rows: None, + summary_maintenance_cost_per_update: None, + summary_read_cost: None, + update_rate: None, + evaluation_rate: None, + raw_recompute_cost: None, + unit: CostUnit::CostUnitsPerSecond, + provenance, + } + } + + /// The rate for whichever phase `phase` names — + /// [`postprocess_plan_cost_rate`] or [`pretransform_plan_cost_rate`]. + pub fn composed_plan_cost_rate(&self, placement: CompositionPlacement) -> Option { + match phase { + CompositionPlacement::PostProcess => postprocess_plan_cost_rate(self), + CompositionPlacement::Transform => pretransform_plan_cost_rate(self), + } + } +} + +/// Outer exact post-process over a maintained summary: +/// +/// ```text +/// postprocess_plan_cost_rate = +/// update_rate * summary_maintenance_cost_per_update +/// + evaluation_rate * (summary_read_cost +/// + output_rows_per_eval * exact_postprocess_cost_per_row) +/// ``` +/// +/// `None` if any input is unknown — see [`ExactCompositionCostInputs`]. +pub fn postprocess_plan_cost_rate(inputs: &ExactCompositionCostInputs) -> Option { + let maintenance = inputs.update_rate? * inputs.summary_maintenance_cost_per_update?; + let per_eval = + inputs.summary_read_cost? + inputs.expected_output_rows? * inputs.exact_cost_per_row?; + let evaluation = inputs.evaluation_rate?.0 * per_eval; + finite_rate(maintenance + evaluation) +} + +/// Outer maintained summary over an exact update-time transform: +/// +/// ```text +/// pretransform_plan_cost_rate = +/// update_rate * (exact_transform_cost_per_input_row +/// + summary_maintenance_cost_per_update) +/// + evaluation_rate * summary_read_cost +/// ``` +/// +/// `None` if any input is unknown — see [`ExactCompositionCostInputs`]. +pub fn pretransform_plan_cost_rate(inputs: &ExactCompositionCostInputs) -> Option { + let per_update = inputs.exact_cost_per_row? + inputs.summary_maintenance_cost_per_update?; + let maintenance = inputs.update_rate? * per_update; + let evaluation = inputs.evaluation_rate?.0 * inputs.summary_read_cost?; + finite_rate(maintenance + evaluation) +} + +/// The raw/pre-ASAP fallback baseline: +/// +/// ```text +/// raw_recompute_cost_rate = evaluation_rate * raw_recompute_cost +/// ``` +/// +/// `None` if either input is unknown — see [`ExactCompositionCostInputs`]. +pub fn raw_recompute_cost_rate(inputs: &ExactCompositionCostInputs) -> Option { + finite_rate(inputs.evaluation_rate?.0 * inputs.raw_recompute_cost?) +} + +fn finite_rate(units_per_second: f64) -> Option { + units_per_second + .is_finite() + .then_some(CostRate(units_per_second)) +} /// A CSE-detected, legality-gated shared subtree with two or more consumers /// — the unit [`CostModel::cse_share_decision`] decides over. Built by @@ -72,7 +282,7 @@ use crate::replacement::{ /// needs a representative bound node for a subtree that /// [`asap_types::pre_asap::cse::share_common_subtrees`] already collapsed /// onto one `Rc` for two or more workload roots. See -/// `docs/design_docs/cse-cost-model-decision.md`. +/// `docs/design_docs/cost-model.md`. pub struct CseCandidate<'a> { /// The shared pre-ASAP subtree itself. pub subtree: &'a QueryExpr, @@ -153,12 +363,12 @@ pub fn default_cse_recompute_cost(subtree: &QueryExpr) -> Cost { Cost(asap_types::pre_asap::cse::dag_node_count(subtree) as f64) } -/// Default [`CostModel::cse_shared_maintenance_cost`]: a small +/// Default context-free [`CostModel::cse_shared_maintenance_cost`]: a small /// per-[`SummaryFamilyType`] weight, scaled to the same order of magnitude /// as [`default_cse_recompute_cost`]'s typical output (a small node /// count, not a byte length), reflecting that families differ in how -/// expensive they are to keep *continuously updated* for the life of a -/// workload — an exact accumulator is the cheapest (an O(1) merge), +/// expensive they are to maintain as shared state — an exact accumulator is +/// the cheapest (an O(1) merge), /// sketches/samples cost more (a whole data structure to update per new /// row), wavelets/fitted models cost the most (coefficient/parameter /// maintenance). These weights are illustrative, not measured — a @@ -306,19 +516,22 @@ pub trait CostModel { /// Estimate the one-time cost of recomputing `candidate.subtree` /// independently at a single use site. Default: /// [`default_cse_recompute_cost`] (a structural-size proxy). See - /// `docs/design_docs/cse-cost-model-decision.md`. + /// `docs/design_docs/cost-model.md`. fn cse_recompute_cost(&self, candidate: &CseCandidate) -> Cost { default_cse_recompute_cost(candidate.subtree) } - /// Estimate the cost of maintaining `candidate.bound_summary` as one - /// continuously-updated shared summary for the life of the workload. + /// Estimate a context-free proxy for maintaining `candidate.bound_summary` + /// as shared state. This fallback has no query recurrence, data arrival, + /// or horizon; workload-aware selection uses + /// [`Self::cse_share_decision_with_recurrence`], and full physical + /// selection additionally uses [`Self::summary_maintenance_lifecycle_cost_inputs`]. /// Default: [`default_cse_shared_maintenance_cost`] (a per-family /// weight table), applied to whichever field of /// `candidate.bound_summary`'s output schema actually carries summary /// state (falls back to the cheapest, `Plain`, weight if none does — /// e.g. `bound_summary` is a passthrough `KeepPreAsap` node with nothing - /// summary-shaped to maintain). See `docs/design_docs/cse-cost-model-decision.md`. + /// summary-shaped to maintain). See `docs/design_docs/cost-model.md`. fn cse_shared_maintenance_cost(&self, candidate: &CseCandidate) -> Cost { let family = candidate .bound_summary @@ -337,7 +550,7 @@ pub trait CostModel { /// Decide whether to reuse one shared `SummaryNode` across every /// consumer of `candidate`, or bind each occurrence independently — a /// Volcano/Cascades-style cost comparison (issue #237, #223 stage 4; see - /// `docs/design_docs/cse-cost-model-decision.md`): share iff the estimated cost of + /// `docs/design_docs/cost-model.md`): share iff the estimated cost of /// maintaining one shared summary is no greater than the estimated total /// cost of recomputing it independently everywhere it's used. /// @@ -506,26 +719,73 @@ pub trait CostModel { f64::NAN } + /// Which mixed exact/summary execution shapes the downstream runtime + /// advertises (issue #171). Gates candidate *generation* in + /// [`crate::exact_composition::ExactCompositionStrategy`]: a shape the + /// runtime can't execute is never proposed, so it can't be selected + /// either. + /// + /// Default: [`MixedExecutionCapabilities::ALL`]. The built-in model + /// describes no particular runtime, and leaving both shapes *visible* + /// in `PlanSpace` (for explanations and the DAG viewer) is the more + /// informative default; selection is still gated separately by + /// [`Self::exact_composition_cost_inputs`], whose default supplies no + /// statistics, so nothing is ever *committed* to under the built-in + /// model. A deployment whose runtime lacks a shape narrows this. + fn mixed_execution_capabilities(&self) -> MixedExecutionCapabilities { + MixedExecutionCapabilities::ALL + } + + /// The statistics the issue #171 recurring-cost formulas need for one + /// composed alternative — see [`ExactCompositionCostInputs`] for each + /// input and [`postprocess_plan_cost_rate`]/ + /// [`pretransform_plan_cost_rate`]/[`raw_recompute_cost_rate`] for how + /// they combine. One structured hook rather than eight scalar ones, so + /// a deployment answers them all from one place (and can attach its own + /// [`CostProvenance`]). + /// + /// Default: every input unknown ([`ExactCompositionCostInputs::unknown`]) + /// — unknown is never zero, and with no rate derivable + /// `PlanSpace::global_selection` keeps the conservative `KeepPreAsap` + /// behavior for the site. A deployment that wants defaults must supply + /// them here explicitly. + fn exact_composition_cost_inputs( + &self, + request: &ExactCompositionCostRequest<'_>, + ) -> ExactCompositionCostInputs { + let _ = request; + ExactCompositionCostInputs::unknown(CostProvenance { + model: "CostModel::exact_composition_cost_inputs (default)".into(), + version: "unknown".into(), + }) + } + /// Primitive build, update, read, retention, and retirement costs used to - /// compare physical summary-state lifecycles. Unknown values stay - /// unknown, preventing long-lived deployments from winning through - /// optimistic zeroes. - fn summary_lifecycle_cost_inputs(&self, _summary: &SummaryNode) -> LifecycleCostInputs { - LifecycleCostInputs::default() + /// compare physical summary-state lifecycles. This is part of the same + /// cost model as candidate ranking and recurrence; summary maintenance + /// lifecycle planning does not introduce a second optimizer. + /// + /// The default leaves every value unknown, which prevents a long-lived + /// deployment from winning through optimistic zeroes. + fn summary_maintenance_lifecycle_cost_inputs( + &self, + _summary: &SummaryNode, + ) -> SummaryMaintenanceLifecycleCostInputs { + SummaryMaintenanceLifecycleCostInputs::default() } /// Physical update/merge/delete support for one concrete summary. The /// conservative default advertises no long-lived maintenance capability. - fn summary_lifecycle_capabilities( + fn summary_maintenance_capabilities( &self, _summary: &SummaryNode, - ) -> SummaryLifecycleCapabilities { - SummaryLifecycleCapabilities::default() + ) -> SummaryMaintenanceCapabilities { + SummaryMaintenanceCapabilities::default() } /// Cost of evaluating `target` directly from its logical/raw inputs once. - /// When known, lifecycle-aware materialization compares this fallback with - /// the aggregate cost of the selected summary deployments. + /// When known, summary-maintenance-aware materialization compares this + /// fallback with the aggregate cost of the selected summary deployments. fn raw_query_recompute_cost(&self, _target: &QueryExpr) -> Option { None } @@ -686,6 +946,12 @@ impl CostModel for DefaultCostModel { (self.cse_recompute_cost(&cse) * consumer_count).0 } } + // A composed candidate is costed in cost-units-per-second by + // `PlanSpace::global_selection` against the child decision it + // is committed with — a different unit from this structural + // estimate, and unknowable here without that child. `NaN` + // keeps it from ever out-ranking a real estimate by accident. + Replacement::ExactComposition(_) => f64::NAN, } } } @@ -830,6 +1096,73 @@ mod tests { ); } + // ── Recurring-cost formulas (issue #171) ───────────────────────────── + + fn known_inputs() -> ExactCompositionCostInputs { + ExactCompositionCostInputs { + exact_cost_per_row: Some(0.1), + expected_input_rows: Some(50.0), + expected_output_rows: Some(10.0), + summary_maintenance_cost_per_update: Some(0.01), + summary_read_cost: Some(1.0), + update_rate: Some(100.0), + evaluation_rate: Some(EvaluationRate(2.0)), + raw_recompute_cost: Some(100.0), + unit: CostUnit::CostUnitsPerSecond, + provenance: CostProvenance { + model: "test".into(), + version: "1".into(), + }, + } + } + + #[test] + fn composition_formulas_match_the_issue_definitions() { + let inputs = known_inputs(); + // 100 * 0.01 + 2 * (1 + 10 * 0.1) = 1 + 4 = 5 + assert_eq!(postprocess_plan_cost_rate(&inputs).unwrap().0, 5.0); + // 100 * (0.1 + 0.01) + 2 * 1 = 11 + 2 = 13 + assert!((pretransform_plan_cost_rate(&inputs).unwrap().0 - 13.0).abs() < 1e-9); + // 2 * 100 + assert_eq!(raw_recompute_cost_rate(&inputs).unwrap().0, 200.0); + assert_eq!( + crate::recurrence::total_cost(CostRate(5.0), Horizon(10.0), Cost(3.0)), + Cost(53.0) + ); + } + + #[test] + fn a_missing_input_yields_no_rate_not_zero() { + let mut inputs = known_inputs(); + inputs.summary_maintenance_cost_per_update = None; + assert_eq!(postprocess_plan_cost_rate(&inputs), None); + assert_eq!(pretransform_plan_cost_rate(&inputs), None); + // The baseline doesn't need maintenance and is still known. + assert!(raw_recompute_cost_rate(&inputs).is_some()); + let unknown = ExactCompositionCostInputs::unknown(known_inputs().provenance); + assert_eq!(raw_recompute_cost_rate(&unknown), None); + } + + #[test] + fn default_model_advertises_capabilities_but_no_statistics() { + assert_eq!( + DefaultCostModel.mixed_execution_capabilities(), + MixedExecutionCapabilities::ALL + ); + assert!(MixedExecutionCapabilities::NONE + .supports(CompositionPlacement::PostProcess) + .not()); + } + + trait Not { + fn not(self) -> bool; + } + impl Not for bool { + fn not(self) -> bool { + !self + } + } + // ── CSE sharing (issue #237, #223 stage 4) ────────────────────────── use asap_types::post_asap::{ diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index be0f0122..04dafdde 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -184,13 +184,14 @@ pub mod accuracy; pub mod accuracy_reconciliation; pub mod cost_model; +pub mod exact_composition; pub mod explanation; pub mod grouping; -pub mod lifecycle; pub mod recurrence; pub mod replacement; pub mod rewrite; pub mod rollup; +pub mod summary_maintenance_lifecycle; pub mod topk_reuse; pub use accuracy::{ @@ -199,16 +200,16 @@ pub use accuracy::{ PropagationStats, WorkloadAccuracyEvidence, }; pub use accuracy_reconciliation::AccuracyReconciliationStrategy; -pub use cost_model::{CostModel, DefaultCostModel}; +pub use cost_model::{ + postprocess_plan_cost_rate, pretransform_plan_cost_rate, raw_recompute_cost_rate, CostModel, + CostProvenance, CostUnit, DefaultCostModel, ExactCompositionCostInputs, + ExactCompositionCostRequest, MixedExecutionCapabilities, +}; +pub use exact_composition::{CompositionPlacement, ExactComposition, ExactCompositionStrategy}; pub use explanation::{ explain_replacements, explain_replacements_with, ExplanationKind, ReplacementExplanation, }; pub use grouping::{has_subpopulations, HydraGroupingStrategy}; -pub use lifecycle::{ - plan_summary_lifecycles, LifecycleAlternative, LifecycleCapabilities, LifecycleCostInputs, - LifecyclePlan, LifecyclePlanError, LifecycleRejection, StateDeployment, - SummaryLifecycleCapabilities, WorkloadDemand, -}; pub use recurrence::{ evaluation_rate_of, total_cost, update_rate_from_data_workload, CostRate, EvaluationRate, Horizon, RecurrenceCostExplanation, RecurrenceError, RecurrenceProfile, RootRecurrence, @@ -216,11 +217,21 @@ pub use recurrence::{ }; pub use replacement::{ default_strategies, default_strategies_with, search_workload, search_workload_with, - search_workload_with_targets, summary_candidates, GlobalSelection, ImplementError, - Implementation, Matcher, MemoGroup, PlanSpace, Proposals, RankedGroup, RecurrenceProfileMap, - RejectedCandidate, Replacement, ReplacementProvenance, ReplacementStrategy, ReplacementSubDAG, - SelectedGroup, SharedSubtreeStrategy, SketchAlgorithmStrategy, TargetSubDAG, - MAX_SEARCH_ITERATIONS, + search_workload_with_targets, summary_candidates, CompositionDecision, GlobalSelection, + ImplementError, Implementation, Matcher, MemoGroup, PlanSpace, Proposals, RankedGroup, + RecurrenceProfileMap, RejectedCandidate, Replacement, ReplacementProvenance, + ReplacementStrategy, ReplacementSubDAG, SelectedGroup, SharedSubtreeStrategy, + SketchAlgorithmStrategy, TargetSubDAG, MAX_SEARCH_ITERATIONS, }; pub use rewrite::AvgToSumOverCountStrategy; +pub use summary_maintenance_lifecycle::{ + global_selection_with_summary_maintenance_lifecycles, + materialize_with_summary_maintenance_lifecycles, plan_summary_maintenance_lifecycles, + MaterializeSummaryMaintenanceLifecycleError, SummaryMaintenanceCapabilities, + SummaryMaintenanceDeployment, SummaryMaintenanceLifecycleAlternative, + SummaryMaintenanceLifecycleCapabilities, SummaryMaintenanceLifecycleCostInputs, + SummaryMaintenanceLifecyclePlan, SummaryMaintenanceLifecyclePlanError, + SummaryMaintenanceLifecycleRejection, SummaryMaintenanceLifecycleSelectionError, + WorkloadDemand, +}; pub use topk_reuse::TopKLimitReuseStrategy; diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index fdbfdf7a..62951124 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -241,7 +241,7 @@ //! //! [`PlanSpace::cost_sorted`] is the `sorted_by(cost_model)` step, and it //! reuses this crate's existing [`CostModel`] trait rather than inventing a -//! second cost interface (`docs/design_docs/cse-cost-model-decision.md`, +//! second cost interface (`docs/design_docs/cost-model.md`, //! issue #237, explicitly reasoned about *why* a narrow, direct cost //! comparison was enough for the CSE share/recompute decision alone, and //! flagged that a real search engine — this module — is where that stops @@ -345,15 +345,17 @@ //! multi-group joint optimization beyond this per-site recurrence is left //! for whenever that changes. +use std::cell::RefCell; use std::collections::{HashMap, HashSet, VecDeque}; -use asap_types::post_asap::{AccuracyError, CompositionOperator, GuaranteeSource, ResultGuarantee}; use asap_types::post_asap::{ - ExactKind, ExactParams, GroupingStrategy, SamplingKind, SamplingParams, SketchAlgorithm, - SketchKind, SketchParams, SketchQuery as PostAsapSketchQuery, StatModelKind, StatModelParams, - SummaryExpr, SummaryFamilyType, SummaryField, SummaryNode, SummarySchema, WaveletKind, - WaveletParams, + validate_execution_data_states_at, ExactKind, ExactOperatorSchemaError, ExactParams, + ExecutionDataState, ExecutionDataStateError, GroupingStrategy, SamplingKind, SamplingParams, + SketchAlgorithm, SketchKind, SketchParams, SketchQuery as PostAsapSketchQuery, StatModelKind, + StatModelParams, SummaryExpr, SummaryFamilyType, SummaryField, SummaryNode, SummarySchema, + WaveletKind, WaveletParams, }; +use asap_types::post_asap::{AccuracyError, CompositionOperator, GuaranteeSource, ResultGuarantee}; use asap_types::pre_asap::agg_intent::{agg_is_mergeable, AggIntent}; use asap_types::pre_asap::cse::{share_common_subtrees, structural_hash, HashCache}; use asap_types::pre_asap::expr_ir::ColumnRef; @@ -372,10 +374,15 @@ use crate::accuracy::{ KLL_RANK_ERROR_EXPONENT_99, }; use crate::accuracy_reconciliation::AccuracyReconciliationStrategy; -use crate::cost_model::{CostModel, CseCandidate, DefaultCostModel, ShareDecision}; +use crate::cost_model::{ + raw_recompute_cost_rate, Cost, CostModel, CseCandidate, DefaultCostModel, + ExactCompositionCostInputs, ExactCompositionCostRequest, ShareDecision, +}; +use crate::exact_composition::{CompositionPlacement, ExactComposition, ExactCompositionStrategy}; use crate::grouping::HydraGroupingStrategy; use crate::recurrence::{ - evaluation_rate_of, Horizon, RecurrenceError, RecurrenceProfile, RootRecurrence, UpdateRate, + evaluation_rate_of, CostRate, Horizon, RecurrenceError, RecurrenceProfile, RootRecurrence, + UpdateRate, }; use crate::rollup::RollupStrategy; use crate::topk_reuse::TopKLimitReuseStrategy; @@ -398,6 +405,15 @@ pub enum ImplementError { /// records it as a [`RejectedCandidate`] instead of a candidate. #[error("accuracy-illegal candidate: {0}")] Accuracy(#[from] AccuracyError), + /// A constructed plan violates the update/readout phase contract + /// (issue #171) — e.g. a summary readout placed beneath a maintained + /// `SummaryAgg`. Detected at construction, never at runtime. + #[error("execution-data_state violation in post-ASAP plan: {0}")] + ExecutionDataState(#[from] ExecutionDataStateError), + /// An `ExactOperator`'s output schema could not be derived over its + /// child — the child carries summary state the operator can't read. + #[error("exact operator schema derivation failed: {0}")] + ExactOperatorSchema(#[from] ExactOperatorSchemaError), } /// A pre-ASAP sub-DAG a [`ReplacementStrategy`] knows how to replace. @@ -457,6 +473,15 @@ pub enum Replacement { /// different from the target's own `root` (e.g. sharing vs. not sharing /// a subtree) but semantically equivalent to it. Rewrite(Rc), + /// An exact operator composed over another target's *own* selected + /// decision across an explicit update/readout boundary (issue #171): + /// `ExactPostProcess` over a child's summary readout, or + /// `ExactTransform` feeding a maintained summary above. Carries only a + /// reference to the child target — [`PlanSpace::global_selection`] + /// commits the compatible parent/child pair and + /// [`GlobalSelection::materialize`] links it into one validated + /// `SummaryNode`. See [`crate::exact_composition`]. + ExactComposition(ExactComposition), } /// One candidate replacement for a [`TargetSubDAG`], plus a human-readable @@ -498,6 +523,12 @@ pub enum ReplacementProvenance { /// regardless, so pricing it like a full independent rebuild would be /// the wrong shape of cost, not just the wrong number. AccuracyReconciliation, + /// [`Replacement::ExactComposition`] with + /// [`CompositionPlacement::PostProcess`] (issue #171). + ExactPostProcess, + /// [`Replacement::ExactComposition`] with + /// [`CompositionPlacement::Transform`] (issue #171). + ExactTransform, } /// A candidate a strategy considered for a target but refused to propose on @@ -523,6 +554,7 @@ pub struct RejectedCandidate { pub struct Proposals { pub candidates: Vec, pub rejected: Vec, + domain_error: Option, } /// A replacement strategy: given a [`TargetSubDAG`], does this strategy have @@ -568,6 +600,7 @@ pub trait ReplacementStrategy { Proposals { candidates: self.replacements(target), rejected: Vec::new(), + domain_error: None, } } } @@ -1371,6 +1404,22 @@ impl<'a> SketchAlgorithmStrategy<'a> { ); } } + if proposals.candidates.is_empty() { + if let Some(error) = &proposals.domain_error { + if let Ok(node) = keep_pre_asap(root) { + proposals.candidates.push(ReplacementSubDAG { + strategy: "SketchAlgorithmStrategy", + replacement: Replacement::Summary(node), + provenance: ReplacementProvenance::SummaryImplementation, + rationale: format!( + "{} stays pre-ASAP because summary construction crosses an illegal \ + execution-data_state boundary ({error})", + describe_intent(intent) + ), + }); + } + } + } proposals } } @@ -1392,7 +1441,10 @@ impl Proposals { description: rationale, error, }), - Err(ImplementError::Schema(_)) => {} + Err(ImplementError::ExecutionDataState(error)) => { + self.domain_error.get_or_insert(error); + } + Err(ImplementError::Schema(_) | ImplementError::ExactOperatorSchema(_)) => {} } } } @@ -1540,10 +1592,10 @@ pub(crate) fn realize_child_with( .. }) => Ok(node), Some(ReplacementSubDAG { - replacement: Replacement::Rewrite(_), + replacement: Replacement::Rewrite(_) | Replacement::ExactComposition(_), .. }) => { - unreachable!("SketchAlgorithmStrategy never returns a Rewrite candidate") + unreachable!("SketchAlgorithmStrategy never returns a Rewrite/composition candidate") } // No candidate at all: `root` isn't `bindable_intent` shape (or its // intent has no realization `implementations_for_with` can't @@ -1764,6 +1816,10 @@ fn construct_summary_agg( // finalized value does. An exact accumulator's state is its value. guarantee: if estimate { None } else { guarantee.clone() }, }); + // Phase contract (issue #171): a maintained summary consumes update-path + // values or exact accumulator state — never a query-time readout. A + // typed error here, at construction; the caller decides the fallback. + validate_execution_data_states_at(&agg, ExecutionDataState::MAINTENANCE_SUMMARY)?; match query { // The readout: downstream of the estimate the schema is the plain // pre-ASAP row shape again (the summary-state type does not @@ -2081,10 +2137,13 @@ impl MemoGroup { (Replacement::Summary(existing_node), Replacement::Summary(node)) => { is_duplicate_summary(existing_node, node) } - // A `Rewrite` and a `Summary` are never the same candidate — - // they're different `Replacement` variants entirely. - (Replacement::Rewrite(_), Replacement::Summary(_)) - | (Replacement::Summary(_), Replacement::Rewrite(_)) => false, + ( + Replacement::ExactComposition(existing), + Replacement::ExactComposition(candidate), + ) => existing.same_as(candidate), + // Different `Replacement` variants are never the same + // candidate. + _ => false, } }); if is_duplicate { @@ -2181,6 +2240,34 @@ pub struct PlanSpace { order: Vec<*const QueryExpr>, } +/// Lifecycle-aware whole-subplan costs keyed by target and candidate pointer. +/// Built by `lifecycle` before final selection; kept internal so pointer keys +/// never become part of the public planner API. +#[derive(Default)] +pub(crate) struct CandidateCostOverrides { + costs: HashMap<(*const QueryExpr, *const ReplacementSubDAG), Cost>, +} + +impl CandidateCostOverrides { + pub(crate) fn insert( + &mut self, + target: &Rc, + candidate: &ReplacementSubDAG, + cost: Cost, + ) { + self.costs.insert( + (Rc::as_ptr(target), candidate as *const ReplacementSubDAG), + cost, + ); + } + + fn get(&self, target: &Rc, candidate: &ReplacementSubDAG) -> Option { + self.costs + .get(&(Rc::as_ptr(target), candidate as *const ReplacementSubDAG)) + .copied() + } +} + impl PlanSpace { /// Every discovered group, in discovery order. pub fn groups(&self) -> impl Iterator { @@ -2599,6 +2686,56 @@ impl PlanSpace { .map(|rate| UpdateRate(rate.0)); self.recurrence_profiles(&recurrences, update_rate) } + + /// Map every discovered target to the normalized workload entries whose + /// roots can reach it. Each entry appears at most once per target even + /// when a root has several paths to that target; path multiplicity is a + /// separate recurrence/effective-use concern. + pub(crate) fn workload_entries_by_target( + &self, + workload: &QueryWorkload, + root_workload_entries: &[usize], + ) -> Result>, RecurrenceError> { + let entry_count = workload.entries().count(); + if root_workload_entries.len() != self.roots.len() { + return Err(RecurrenceError::RootCountMismatch { + expected: self.roots.len(), + got: root_workload_entries.len(), + }); + } + let mut bindings: HashMap<*const QueryExpr, HashSet> = HashMap::new(); + for ((_, root), &entry_index) in self.roots.iter().zip(root_workload_entries) { + if entry_index >= entry_count { + return Err(RecurrenceError::InvalidWorkloadEntry { + index: entry_index, + entry_count, + }); + } + let mut seen = HashSet::new(); + let mut queue = VecDeque::from([Rc::as_ptr(root)]); + while let Some(ptr) = queue.pop_front() { + if !seen.insert(ptr) { + continue; + } + bindings.entry(ptr).or_default().insert(entry_index); + if let Some(group) = self.groups.get(&ptr) { + queue.extend( + direct_child_counts(&group.target) + .into_iter() + .map(|(child, _)| child), + ); + } + } + } + Ok(bindings + .into_iter() + .map(|(ptr, entries)| { + let mut entries: Vec<_> = entries.into_iter().collect(); + entries.sort_unstable(); + (ptr, entries) + }) + .collect()) + } } /// Record `times` occurrences of `recurrence` against `ptr` — `times > 1` @@ -2730,7 +2867,7 @@ fn rank_group<'a>(group: &'a MemoGroup, cost_model: &dyn CostModel) -> Vec<&'a R .iter() .map(|c| match &c.replacement { Replacement::Summary(node) => sketch_kind_of(node), - Replacement::Rewrite(_) => None, + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => None, }) .collect(); if let Some(kinds) = kinds { @@ -2738,7 +2875,7 @@ fn rank_group<'a>(group: &'a MemoGroup, cost_model: &dyn CostModel) -> Vec<&'a R ranked.sort_by_key(|c| { let kind = match &c.replacement { Replacement::Summary(node) => sketch_kind_of(node), - Replacement::Rewrite(_) => None, + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => None, }; kind.and_then(|k| order.iter().position(|o| *o == k)) .unwrap_or(usize::MAX) @@ -2761,6 +2898,59 @@ fn rank_group<'a>(group: &'a MemoGroup, cost_model: &dyn CostModel) -> Vec<&'a R ranked } +/// Apply lifecycle-aware costs to summary siblings after the ordinary +/// strategy-specific ordering. Known lifecycle totals sort before unknown +/// totals; non-summary alternatives keep their existing relative order and +/// continue through their dedicated CSE/composition selection paths. +fn rank_group_with_candidate_costs<'a>( + group: &'a MemoGroup, + cost_model: &dyn CostModel, + overrides: Option<&CandidateCostOverrides>, +) -> Vec<&'a ReplacementSubDAG> { + let mut ranked = rank_group(group, cost_model); + let Some(overrides) = overrides else { + return ranked; + }; + let positions: Vec = ranked + .iter() + .enumerate() + .filter_map(|(index, candidate)| { + matches!(candidate.replacement, Replacement::Summary(_)).then_some(index) + }) + .collect(); + let mut summaries: Vec<_> = positions.iter().map(|&index| ranked[index]).collect(); + summaries.sort_by(|a, b| { + match ( + overrides.get(&group.target, a), + overrides.get(&group.target, b), + ) { + (Some(a), Some(b)) => a.0.total_cmp(&b.0), + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => std::cmp::Ordering::Equal, + } + }); + for (index, candidate) in positions.into_iter().zip(summaries) { + ranked[index] = candidate; + } + ranked +} + +fn estimated_candidate_cost( + group: &MemoGroup, + candidate: &ReplacementSubDAG, + target: &TargetSubDAG<'_>, + cost_model: &dyn CostModel, + overrides: Option<&CandidateCostOverrides>, +) -> f64 { + overrides + .and_then(|costs| costs.get(&group.target, candidate)) + .map_or_else( + || cost_model.estimate_cost(candidate, target), + |cost| cost.0, + ) +} + /// For a group whose candidates are all [`Replacement::Rewrite`] (the /// [`SharedSubtreeStrategy`] shape): does [`CostModel::cse_share_decision`] /// prefer the candidate that shares `group.target`'s own `Rc` (`true`), or @@ -2866,6 +3056,32 @@ pub struct SelectedGroup<'a> { /// registered strategy proposed anything for (mirrors /// [`MemoGroup::candidates`] being possibly empty). pub chosen: Option<&'a ReplacementSubDAG>, + /// When `chosen` is a [`Replacement::ExactComposition`]: the child + /// decision it was committed together with, and the cost comparison + /// that justified it — the explicit target-to-decision provenance + /// chain (issue #171). + pub composition: Option>, +} + +/// Why [`PlanSpace::global_selection`] committed an exact composition at a +/// site: which child candidate it composes with, and the +/// cost-units-per-second comparison against the raw fallback that it won. +#[derive(Debug)] +pub struct CompositionDecision<'a> { + /// The child target the composed operator consumes. + pub child_target: &'a Rc, + /// For a post-process: the child's own candidate committed alongside + /// (the summary readout the operator folds). `None` for an update-path + /// transform, whose input is raw update data — its cost is charged to + /// the maintained summary *above* it instead. + pub child_candidate: Option<&'a ReplacementSubDAG>, + /// The composed plan's recurring rate — `postprocess_plan_cost_rate` + /// or `pretransform_plan_cost_rate`. + pub cost_rate: CostRate, + /// `raw_recompute_cost_rate` — the `KeepPreAsap` baseline it beat. + pub baseline_rate: CostRate, + /// The statistics (and their provenance) both rates were computed from. + pub inputs: ExactCompositionCostInputs, } /// [`PlanSpace::global_selection`]'s result: one [`SelectedGroup`] per @@ -2875,6 +3091,10 @@ pub struct SelectedGroup<'a> { pub struct GlobalSelection<'a> { order: Vec<*const QueryExpr>, groups: HashMap<*const QueryExpr, SelectedGroup<'a>>, + /// [`Self::materialize`]'s memo — one bound node per target for the + /// life of this selection, so two parents composing over one shared + /// child get the *same* `Rc`. + materialized: RefCell>>, } impl<'a> GlobalSelection<'a> { @@ -2889,6 +3109,272 @@ impl<'a> GlobalSelection<'a> { pub fn for_target(&self, target: &Rc) -> Option<&SelectedGroup<'a>> { self.groups.get(&Rc::as_ptr(target)) } + + /// Link this selection's per-site decisions into one data_state-validated + /// post-ASAP DAG rooted at `target` — the one place a committed + /// composition's child *reference* becomes an actual `Rc` + /// edge (issue #171). `None` if `target` is not a discovered site. + /// + /// Per site: a [`Replacement::ExactComposition`] composes over its + /// child target's own materialization; a [`Replacement::Summary`] is + /// re-linked so its `SummaryAgg` child is the child target's own + /// materialization whenever that is phase-legal beneath maintenance + /// (so a child that chose an `ExactTransform` actually ends up under + /// the summary); a [`Replacement::Rewrite`] or an unmatched site stays + /// the conservative `KeepPreAsap`. Memoized by target identity, so a + /// shared inner summary is one `Rc` no matter how many roots reach it. + pub fn materialize( + &self, + target: &Rc, + ) -> Result>, ImplementError> { + if !self.groups.contains_key(&Rc::as_ptr(target)) { + return Ok(None); + } + self.materialize_inner(target).map(Some) + } + + fn materialize_inner(&self, target: &Rc) -> Result, ImplementError> { + let ptr = Rc::as_ptr(target); + if let Some(node) = self.materialized.borrow().get(&ptr) { + return Ok(Rc::clone(node)); + } + let node = match self + .groups + .get(&ptr) + .and_then(|sel| sel.chosen) + .map(|c| &c.replacement) + { + None => keep_pre_asap(target)?, + Some(Replacement::Rewrite(rewritten)) => keep_pre_asap(rewritten)?, + Some(Replacement::Summary(node)) => self.relink_summary(node, target)?, + Some(Replacement::ExactComposition(composition)) => { + let child = self.materialize_inner(&composition.child_target)?; + let child = if composition.accepts_child(&child) { + child + } else { + keep_pre_asap(&composition.child_target)? + }; + composition.compose(child)? + } + }; + self.materialized.borrow_mut().insert(ptr, Rc::clone(&node)); + Ok(node) + } + + /// Re-link a bound `Summary` candidate's `SummaryAgg` child to the + /// child target's own materialization when that is legal beneath + /// maintenance; otherwise keep the candidate exactly as constructed. + fn relink_summary( + &self, + node: &Rc, + target: &Rc, + ) -> Result, ImplementError> { + let QueryExpr::Aggregate { + child: pre_child, .. + } = target.as_ref() + else { + return Ok(Rc::clone(node)); + }; + if !self.groups.contains_key(&Rc::as_ptr(pre_child)) { + return Ok(Rc::clone(node)); + } + let new_child = self.materialize_inner(pre_child)?; + Ok(relink_agg_child(node, &new_child)) + } +} + +/// Rebuild `node` (a `SummaryAgg`, possibly under a `SummaryEstimate`) with +/// `new_child` as the `SummaryAgg`'s child, if the result still validates +/// as maintained state; otherwise return `node` unchanged. +fn relink_agg_child(node: &Rc, new_child: &Rc) -> Rc { + match &node.expr { + SummaryExpr::SummaryEstimate { + summary_input, + query, + } => { + let inner = relink_agg_child(summary_input, new_child); + if Rc::ptr_eq(&inner, summary_input) { + return Rc::clone(node); + } + Rc::new(SummaryNode { + expr: SummaryExpr::SummaryEstimate { + summary_input: inner, + query: query.clone(), + }, + schema: node.schema.clone(), + guarantee: node.guarantee.clone(), + }) + } + SummaryExpr::SummaryAgg { + child, + family, + col, + reduction, + grouping, + } => { + if Rc::ptr_eq(child, new_child) { + return Rc::clone(node); + } + let rebuilt = Rc::new(SummaryNode { + expr: SummaryExpr::SummaryAgg { + child: Rc::clone(new_child), + family: family.clone(), + col: col.clone(), + reduction: reduction.clone(), + grouping: grouping.clone(), + }, + schema: node.schema.clone(), + guarantee: node.guarantee.clone(), + }); + match validate_execution_data_states_at( + &rebuilt, + ExecutionDataState::MAINTENANCE_SUMMARY, + ) { + Ok(_) => rebuilt, + Err(_) => Rc::clone(node), + } + } + _ => Rc::clone(node), + } +} + +/// The maintained `SummaryAgg` a bound `Summary` candidate builds (under +/// its `SummaryEstimate` readout, if any) — the summary an `ExactTransform` +/// beneath it feeds, for `pretransform_plan_cost_rate`. +fn maintained_summary(node: &Rc) -> Option<&Rc> { + match &node.expr { + SummaryExpr::SummaryEstimate { summary_input, .. } => maintained_summary(summary_input), + SummaryExpr::SummaryAgg { .. } => Some(node), + _ => None, + } +} + +fn is_composition_candidate(candidate: &ReplacementSubDAG) -> bool { + matches!(candidate.replacement, Replacement::ExactComposition(_)) +} + +/// Everything [`PlanSpace::global_selection`] threads between sites for +/// exact compositions (issue #171): child candidates already committed by +/// an earlier parent, and the maintained summary above each site. +#[derive(Default)] +struct CompositionContext { + /// child target ptr → the child's candidate an ancestor's composition + /// already committed to (a later parent must compose with the *same* + /// one, and the child's own selection is forced to it). + committed_child: HashMap<*const QueryExpr, *const ReplacementSubDAG>, + /// site ptr → the maintained `SummaryAgg` directly above it, when its + /// parent chose a bound `Summary` — what an `ExactTransform` here feeds. + maintaining_parent: HashMap<*const QueryExpr, Rc>, +} + +/// One eligible composed alternative at a site, before the cheapest wins. +struct CompositionOption<'a> { + candidate: &'a ReplacementSubDAG, + decision: CompositionDecision<'a>, +} + +/// Every [`Replacement::ExactComposition`] candidate of `group` whose +/// composed-plan rate is *known* and beats the raw-recompute baseline — +/// costed against each compatible child candidate already in `PlanSpace` +/// (or the one an earlier parent committed). Unknown statistics yield no +/// option at all: the conservative `KeepPreAsap` path stays. +fn composition_options<'a>( + group: &'a MemoGroup, + groups: &'a HashMap<*const QueryExpr, MemoGroup>, + effective: usize, + cost_model: &dyn CostModel, + context: &CompositionContext, +) -> Vec> { + let mut options = Vec::new(); + for candidate in &group.candidates { + let Replacement::ExactComposition(composition) = &candidate.replacement else { + continue; + }; + let child_ptr = Rc::as_ptr(&composition.child_target); + let Some(child_group) = groups.get(&child_ptr) else { + continue; + }; + let already_committed = context.committed_child.get(&child_ptr).copied(); + let cost = |summary: &SummaryNode, shared: bool| { + let request = ExactCompositionCostRequest { + target: &group.target, + composition, + summary, + effective_consumer_count: effective, + }; + let mut inputs = cost_model.exact_composition_cost_inputs(&request); + if shared { + // Shared state is counted once: an earlier parent already + // pays this child's maintenance, so the marginal cost here + // is zero — a *known* zero, unlike an unknown input. + if let Some(maintenance) = inputs.summary_maintenance_cost_per_update.as_mut() { + *maintenance = 0.0; + } + } + let rate = inputs.composed_plan_cost_rate(composition.placement)?; + let baseline = raw_recompute_cost_rate(&inputs)?; + (rate < baseline).then_some((rate, baseline, inputs)) + }; + match composition.placement { + CompositionPlacement::PostProcess => { + let child_candidates: Vec<&'a ReplacementSubDAG> = match already_committed { + // SAFETY-free: the pointer was taken from `groups`'s own + // candidate storage, which outlives this borrow. + Some(ptr) => child_group + .candidates + .iter() + .filter(|c| std::ptr::eq(*c, ptr)) + .collect(), + None => child_group.candidates.iter().collect(), + }; + for child_candidate in child_candidates { + let Replacement::Summary(summary) = &child_candidate.replacement else { + continue; + }; + if !composition.accepts_child(summary) { + continue; + } + let Some((rate, baseline, inputs)) = cost(summary, already_committed.is_some()) + else { + continue; + }; + options.push(CompositionOption { + candidate, + decision: CompositionDecision { + child_target: &composition.child_target, + child_candidate: Some(child_candidate), + cost_rate: rate, + baseline_rate: baseline, + inputs, + }, + }); + } + } + CompositionPlacement::Transform => { + // An update-path transform only pays off beneath a + // maintained summary; with nothing above it, its output is + // never read and the raw fallback is the same computation. + let Some(parent) = context.maintaining_parent.get(&Rc::as_ptr(&group.target)) + else { + continue; + }; + let Some((rate, baseline, inputs)) = cost(parent, false) else { + continue; + }; + options.push(CompositionOption { + candidate, + decision: CompositionDecision { + child_target: &composition.child_target, + child_candidate: None, + cost_rate: rate, + baseline_rate: baseline, + inputs, + }, + }); + } + } + } + options } impl PlanSpace { @@ -2900,7 +3386,7 @@ impl PlanSpace { /// [`Self::cost_sorted`], whose per-group ranking only ever sees a /// group's own raw [`MemoGroup::consumer_count`]. pub fn global_selection(&self, cost_model: &dyn CostModel) -> GlobalSelection<'_> { - self.global_selection_impl(cost_model, None, None) + self.global_selection_impl(cost_model, None, None, None) .expect("structural global selection cannot produce a recurrence error") } @@ -2914,7 +3400,20 @@ impl PlanSpace { profiles: &RecurrenceProfileMap, horizon: Option, ) -> Result, RecurrenceError> { - self.global_selection_impl(cost_model, Some(profiles), horizon) + self.global_selection_impl(cost_model, Some(profiles), horizon, None) + } + + /// Final selection with lifecycle-aware whole-subplan cost overrides. + /// `lifecycle` builds the overrides from normalized workload evidence and + /// calls this only after candidate legality and accuracy validation. + pub(crate) fn global_selection_with_candidate_costs( + &self, + cost_model: &dyn CostModel, + profiles: &RecurrenceProfileMap, + horizon: Option, + candidate_costs: &CandidateCostOverrides, + ) -> Result, RecurrenceError> { + self.global_selection_impl(cost_model, Some(profiles), horizon, Some(candidate_costs)) } fn global_selection_impl( @@ -2922,6 +3421,7 @@ impl PlanSpace { cost_model: &dyn CostModel, profiles: Option<&RecurrenceProfileMap>, horizon: Option, + candidate_costs: Option<&CandidateCostOverrides>, ) -> Result, RecurrenceError> { let graph = reference_graph(self); let topo = topological_order(&self.order, &graph); @@ -2929,6 +3429,7 @@ impl PlanSpace { let mut effective_uses = graph.external_root_uses.clone(); let mut chosen_share: HashMap<*const QueryExpr, ShareDecision> = HashMap::new(); let mut groups: HashMap<*const QueryExpr, SelectedGroup<'_>> = HashMap::new(); + let mut context = CompositionContext::default(); for ptr in &topo { let group = &self.groups[ptr]; @@ -2936,7 +3437,50 @@ impl PlanSpace { let effective = effective_uses.get(ptr).copied().unwrap_or(0); effective_uses.insert(*ptr, effective); - let chosen = if effective >= 2 && cse_candidate_pair(group).is_some() { + // ── Exact compositions (issue #171) ───────────────────────── + // A child an earlier parent's composition committed to is + // forced to exactly that candidate — the parent/child pair is + // one decision. Otherwise, a composition here wins only when + // its cost-units-per-second rate is *known* and beats the raw + // recompute baseline; missing statistics keep the conservative + // path below. + let mut composition_decision = None; + let forced = context + .committed_child + .get(ptr) + .and_then(|&cptr| group.candidates.iter().find(|c| std::ptr::eq(*c, cptr))); + let composed = if forced.is_some() { + None + } else { + composition_options(group, &self.groups, effective, cost_model, &context) + .into_iter() + .min_by(|a, b| a.decision.cost_rate.0.total_cmp(&b.decision.cost_rate.0)) + }; + if let Some(option) = &composed { + if let Some(child_candidate) = option.decision.child_candidate { + context.committed_child.insert( + Rc::as_ptr(option.decision.child_target), + child_candidate as *const ReplacementSubDAG, + ); + } + if let Replacement::ExactComposition(composition) = &option.candidate.replacement { + if composition.placement == CompositionPlacement::Transform { + // A chain of transforms feeds the same summary. + if let Some(parent) = context.maintaining_parent.get(ptr).cloned() { + context + .maintaining_parent + .insert(Rc::as_ptr(&composition.child_target), parent); + } + } + } + } + + let chosen = if let Some(forced) = forced { + Some(forced) + } else if let Some(option) = composed { + composition_decision = Some(option.decision); + Some(option.candidate) + } else if effective >= 2 && cse_candidate_pair(group).is_some() { let decision = if let Some(profiles) = profiles { decide_group_with_recurrence( group, @@ -2956,18 +3500,44 @@ impl PlanSpace { let logical = group .candidates .iter() - .filter(|candidate| !is_cse_candidate(candidate)) + .filter(|candidate| { + !is_cse_candidate(candidate) && !is_composition_candidate(candidate) + }) .min_by(|a, b| { - cost_model - .estimate_cost(a, &effective_target) - .total_cmp(&cost_model.estimate_cost(b, &effective_target)) + estimated_candidate_cost( + group, + a, + &effective_target, + cost_model, + candidate_costs, + ) + .total_cmp( + &estimated_candidate_cost( + group, + b, + &effective_target, + cost_model, + candidate_costs, + ), + ) }); match (cse, logical) { (Some(cse), Some(logical)) - if cost_model - .estimate_cost(logical, &effective_target) - .total_cmp(&cost_model.estimate_cost(cse, &effective_target)) - .is_lt() => + if estimated_candidate_cost( + group, + logical, + &effective_target, + cost_model, + candidate_costs, + ) + .total_cmp(&estimated_candidate_cost( + group, + cse, + &effective_target, + cost_model, + candidate_costs, + )) + .is_lt() => { Some(logical) } @@ -2987,15 +3557,32 @@ impl PlanSpace { // valid answer, just not a cross-group-aware one; this // group also contributes no Share collapse to its own // children (see `multiplier`'s `_ => effective` arm). - None => rank_group(group, cost_model).into_iter().next(), + None => rank_group_with_candidate_costs(group, cost_model, candidate_costs) + .into_iter() + .find(|candidate| !is_composition_candidate(candidate)), } } else { - rank_group(group, cost_model) + rank_group_with_candidate_costs(group, cost_model, candidate_costs) .into_iter() - .find(|candidate| !is_cse_candidate(candidate)) + .find(|candidate| { + !is_cse_candidate(candidate) && !is_composition_candidate(candidate) + }) .or_else(|| cse_candidate_pair(group).map(|(share, _)| share)) }; + // Record the maintained summary this site's bound candidate + // builds, for a child that may compose an `ExactTransform` + // beneath it. + if let (Some(Replacement::Summary(node)), QueryExpr::Aggregate { child, .. }) = + (chosen.map(|c| &c.replacement), group.target.as_ref()) + { + if let Some(summary) = maintained_summary(node) { + context + .maintaining_parent + .insert(Rc::as_ptr(child), Rc::clone(summary)); + } + } + let outgoing_multiplier = multiplier(*ptr, &effective_uses, &chosen_share); match chosen { Some(ReplacementSubDAG { @@ -3013,7 +3600,9 @@ impl PlanSpace { _ => { let selected_rewrite = match chosen.map(|candidate| &candidate.replacement) { Some(Replacement::Rewrite(rewrite)) => rewrite, - Some(Replacement::Summary(_)) | None => &group.target, + Some(Replacement::Summary(_) | Replacement::ExactComposition(_)) | None => { + &group.target + } }; for (child, edge_count) in direct_child_counts(selected_rewrite) { *effective_uses.entry(child).or_insert(0) += @@ -3029,6 +3618,7 @@ impl PlanSpace { consumer_count: group.consumer_count, effective_consumer_count: effective, chosen, + composition: composition_decision, }, ); } @@ -3036,6 +3626,7 @@ impl PlanSpace { Ok(GlobalSelection { order: self.order.clone(), groups, + materialized: RefCell::new(HashMap::new()), }) } } @@ -3411,6 +4002,7 @@ pub fn default_strategies() -> Vec> { Box::new(HydraGroupingStrategy::default_cost_model()), Box::new(SharedSubtreeStrategy), Box::new(crate::rewrite::AvgToSumOverCountStrategy), + Box::new(ExactCompositionStrategy::default_cost_model()), ] } @@ -3425,6 +4017,7 @@ pub fn default_strategies_with<'a>( Box::new(HydraGroupingStrategy::new(cost_model)), Box::new(SharedSubtreeStrategy), Box::new(crate::rewrite::AvgToSumOverCountStrategy), + Box::new(ExactCompositionStrategy::new(cost_model)), ] } @@ -3518,6 +4111,7 @@ pub fn search_workload_with_targets<'s, Id>( .as_ref() .is_some_and(|g| accuracy_model.satisfies(g, &target)), Replacement::Rewrite(_) => true, + Replacement::ExactComposition(_) => false, }); group.candidates = legal; group.rejected.extend(illegal.into_iter().map(|candidate| { @@ -3538,6 +4132,11 @@ pub fn search_workload_with_targets<'s, Id>( None, )), Replacement::Rewrite(_) => unreachable!("rewrites are never rejected here"), + Replacement::ExactComposition(_) => ( + asap_types::post_asap::ErrorMetric::AbsoluteValue, + None, + None, + ), }; RejectedCandidate { strategy: candidate.strategy, @@ -4491,7 +5090,9 @@ mod tests { .iter() .map(|r| match &r.replacement { Replacement::Summary(node) => summary_family_algorithm(node), - Replacement::Rewrite(_) => panic!("expected a Summary replacement"), + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => { + panic!("expected a Summary replacement") + } }) .collect(); assert!(kinds.contains(&SketchAlgorithm::Kll), "{kinds:?}"); @@ -4511,7 +5112,9 @@ mod tests { .iter() .map(|r| match &r.replacement { Replacement::Summary(node) => summary_family_algorithm(node), - Replacement::Rewrite(_) => panic!("expected a Summary replacement"), + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => { + panic!("expected a Summary replacement") + } }) .collect(); assert_eq!( @@ -4539,7 +5142,9 @@ mod tests { .iter() .map(|r| match &r.replacement { Replacement::Summary(node) => summary_family_algorithm(node), - Replacement::Rewrite(_) => panic!("expected a Summary replacement"), + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => { + panic!("expected a Summary replacement") + } }) .collect(); assert_eq!(kinds, vec![SketchAlgorithm::Theta, SketchAlgorithm::Kmv]); @@ -4617,7 +5222,9 @@ mod tests { .iter() .map(|r| match &r.replacement { Replacement::Summary(node) => summary_family_algorithm(node), - Replacement::Rewrite(_) => panic!("expected a Summary replacement"), + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => { + panic!("expected a Summary replacement") + } }) .collect(); assert!(kinds.contains(&SketchAlgorithm::Kll)); @@ -4625,13 +5232,13 @@ mod tests { assert_eq!(kinds.len(), 2); } - /// Enumerating candidates for the *target* node must only steer that - /// node's own decision — a nested aggregate underneath it still gets its - /// own independent (`cost_model`-ranked) enumeration, not whatever the - /// caller happened to pick for the outer target. This is the behavior - /// [`construct_summary`]'s recursion (via [`realize_child`]) - /// gets for free: only the top node's `Implementation` is ever forced - /// from outside; the child is always re-enumerated fresh. + /// Constructing the outer target's candidates never leaks the outer + /// choice into the nested aggregate — and, since issue #171's phase + /// contract, a maintained outer sketch can no longer sit above the + /// inner sketch's *readout* at all: the outer target degrades to the + /// conservative `KeepPreAsap` fallback (reported once, not once per + /// dropped family), while the inner quantile keeps its own, + /// independently cost-ranked candidates in its own `MemoGroup`. #[test] fn enumerating_the_targets_candidates_does_not_leak_into_a_nested_aggregate() { // outer: quantile(0.99, ...) over inner: quantile(0.5, m) — both @@ -4651,35 +5258,41 @@ mod tests { ) .replacements(&target); - let ddsketch = replacements - .iter() - .find(|r| { - matches!(&r.replacement, Replacement::Summary(node) - if summary_family_algorithm(node) == SketchAlgorithm::DDSketch) - }) - .expect("the outer target's DDSketch candidate must be present"); - let Replacement::Summary(node) = &ddsketch.replacement else { - unreachable!("filtered on Replacement::Summary above"); + assert_eq!(replacements.len(), 1, "{replacements:?}"); + let Replacement::Summary(node) = &replacements[0].replacement else { + unreachable!("SketchAlgorithmStrategy only returns Summary candidates"); }; - assert_eq!( - summary_family_algorithm(node), - SketchAlgorithm::DDSketch, - "the outer (target) node must be the DDSketch candidate" + assert!( + matches!(node.expr, SummaryExpr::KeepPreAsap(ref e) if Rc::ptr_eq(e, &outer)), + "a sketch over a sketch readout is data_state-illegal; expected the conservative \ + fallback, got {:?}", + node.expr ); - - let asap_types::post_asap::SummaryExpr::SummaryEstimate { summary_input, .. } = &node.expr - else { - panic!("expected SummaryEstimate root, got {:?}", node.expr); - }; - let asap_types::post_asap::SummaryExpr::SummaryAgg { child, .. } = &summary_input.expr - else { - panic!("expected SummaryAgg, got {:?}", summary_input.expr); + assert!(replacements[0].rationale.contains("readout")); + + // The inner target is still independently enumerated and ranked — + // a custom cost model that prefers DDSketch for it is honored, and + // nothing about the outer target's choice reaches it. + let space = search_workload_with( + vec![("q", Rc::clone(&outer))], + &default_strategies_with(&PreferDDSketchViaCostModel), + ); + let QueryExpr::Aggregate { child, .. } = space.roots[0].1.as_ref() else { + unreachable!() }; + let inner_group = space.group_for(child).expect("inner quantile is a target"); + let inner_kinds: Vec = inner_group + .candidates + .iter() + .filter_map(|c| match &c.replacement { + Replacement::Summary(node) => sketch_kind_of(node), + _ => None, + }) + .collect(); assert_eq!( - summary_family_algorithm(child), - SketchAlgorithm::Kll, - "the nested inner aggregate must still get the cost-model-ranked \ - default (Kll), not inherit the outer target's DDSketch candidate" + inner_kinds, + vec![SketchAlgorithm::DDSketch, SketchAlgorithm::Kll], + "the nested inner aggregate keeps its own cost-model-ranked candidates" ); } @@ -5206,7 +5819,7 @@ mod tests { assert_eq!(rewrites.len(), 2); let first_shares_target = match &rewrites[0].replacement { Replacement::Rewrite(rc) => Rc::ptr_eq(rc, &group.target), - Replacement::Summary(_) => false, + Replacement::Summary(_) | Replacement::ExactComposition(_) => false, }; assert!( first_shares_target, @@ -5242,7 +5855,7 @@ mod tests { assert_eq!(agg_group.candidates.len(), 2); let first_kind = match &agg_group.candidates[0].replacement { Replacement::Summary(node) => sketch_kind_of(node), - Replacement::Rewrite(_) => None, + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => None, }; assert_eq!(first_kind, Some(SketchAlgorithm::DDSketch)); } @@ -5438,7 +6051,7 @@ mod tests { .unwrap(); let kind = match &agg_group.chosen.unwrap().replacement { Replacement::Summary(node) => sketch_kind_of(node), - Replacement::Rewrite(_) => None, + Replacement::Rewrite(_) | Replacement::ExactComposition(_) => None, }; assert_eq!(kind, Some(SketchAlgorithm::DDSketch)); } @@ -6769,13 +7382,7 @@ mod tests { let space = search_workload_with(vec![("q", Rc::clone(&outer))], &strategies); let root = &space.roots[0].1; let group = space.group_for(root).unwrap(); - assert!(!group.rejected.is_empty()); - assert!(group.candidates.iter().all(|c| match &c.replacement { - Replacement::Summary(node) => node.guarantee.as_ref().is_some_and(|g| { - DefaultAccuracyModel.satisfies(g, &AccuracyTarget::Epsilon(0.1)) - }), - Replacement::Rewrite(_) => false, - })); + assert_eq!(group.candidates.len(), 1); let ranked = space.cost_sorted(&DefaultCostModel); let root_ranked = ranked.iter().find(|g| Rc::ptr_eq(g.target, root)).unwrap(); assert_eq!(root_ranked.candidates.len(), group.candidates.len()); @@ -6842,6 +7449,7 @@ mod tests { .as_ref() .is_some_and(ResultGuarantee::is_exact), Replacement::Rewrite(_) => true, + Replacement::ExactComposition(_) => false, })); } diff --git a/crates/asap-aware-mapping/src/lifecycle.rs b/crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs similarity index 50% rename from crates/asap-aware-mapping/src/lifecycle.rs rename to crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs index 8a4a37f8..e7b314d6 100644 --- a/crates/asap-aware-mapping/src/lifecycle.rs +++ b/crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs @@ -1,25 +1,37 @@ -//! Workload-aware physical lifecycle planning for summary state. +//! Workload-aware physical summary-maintenance lifecycle planning. //! +//! Phase validation from PR #300 answers whether a post-ASAP DAG can execute. //! This module answers how each unique `SummaryAgg` state is deployed for the //! supplied query and data workloads. Unknown evidence stays unknown and -//! therefore cannot make a long-lived lifecycle win. +//! therefore cannot make a long-lived summary maintenance lifecycle win. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::rc::Rc; -use asap_types::post_asap::{EvaluationSchedule, OutputRepresentation, SummaryMaintenanceMode}; -use asap_types::post_asap::{StateLifecycle, SummaryExpr, SummaryNode}; +use asap_types::post_asap::{ + produced_availability, validate_execution_phases, ExecutionAvailability, SummaryExpr, + SummaryMaintenanceLifecycle, SummaryNode, +}; +use asap_types::post_asap::{ + EvaluationSchedule, OutputRepresentation, SummaryMaintenanceLifecycleGuarantee, +}; +use asap_types::pre_asap::QueryExpr; use asap_types::workload::{ DataArrival, Predictability, QueryRecurrence, QueryWorkload, RepeatedDemand, TimestampMs, WorkloadError, }; use crate::cost_model::{Cost, CostModel}; -use crate::recurrence::{CostRate, EvaluationRate, Horizon, UpdateRate}; +use crate::recurrence::{ + CostRate, EvaluationRate, Horizon, RecurrenceError, RecurrenceProfile, UpdateRate, +}; +use crate::replacement::{ + CandidateCostOverrides, GlobalSelection, ImplementError, PlanSpace, Replacement, +}; -/// Runtime lifecycle shapes available to the planner. +/// Summary maintenance lifecycle shapes available to the runtime planner. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct LifecycleCapabilities { +pub struct SummaryMaintenanceLifecycleCapabilities { pub ephemeral: bool, pub prepared: bool, pub shared: bool, @@ -28,13 +40,13 @@ pub struct LifecycleCapabilities { /// Capabilities of one concrete summary family/state representation. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct SummaryLifecycleCapabilities { +pub struct SummaryMaintenanceCapabilities { pub incremental_update: bool, pub merge: bool, pub delete: bool, } -impl LifecycleCapabilities { +impl SummaryMaintenanceLifecycleCapabilities { pub const ALL: Self = Self { ephemeral: true, prepared: true, @@ -43,7 +55,7 @@ impl LifecycleCapabilities { }; } -impl Default for LifecycleCapabilities { +impl Default for SummaryMaintenanceLifecycleCapabilities { fn default() -> Self { Self::ALL } @@ -52,7 +64,7 @@ impl Default for LifecycleCapabilities { /// Primitive costs for one concrete summary state. Every field is optional: /// missing statistics produce an uncosted alternative, never a zero. #[derive(Debug, Clone, Default, PartialEq)] -pub struct LifecycleCostInputs { +pub struct SummaryMaintenanceLifecycleCostInputs { pub build_cost: Option, pub maintenance_cost_per_update: Option, pub summary_read_cost: Option, @@ -61,7 +73,7 @@ pub struct LifecycleCostInputs { } #[derive(Debug, Clone, PartialEq, Eq)] -pub enum LifecycleRejection { +pub enum SummaryMaintenanceLifecycleRejection { UnsupportedByRuntime, RequiresPredictableOneTimeQuery, RequiresMultipleReads, @@ -74,16 +86,14 @@ pub enum LifecycleRejection { } #[derive(Debug, Clone, PartialEq)] -pub struct LifecycleAlternative { - pub lifecycle: StateLifecycle, - /// How this lifecycle obtains and refreshes its summary state. - pub maintenance_mode: SummaryMaintenanceMode, +pub struct SummaryMaintenanceLifecycleAlternative { + pub summary_maintenance_lifecycle: SummaryMaintenanceLifecycle, pub total_cost: Option, - pub rejection: Option, + pub rejection: Option, pub assumptions: Vec, } -impl LifecycleAlternative { +impl SummaryMaintenanceLifecycleAlternative { fn selectable(&self) -> bool { self.rejection.is_none() && self.total_cost.is_some() } @@ -91,20 +101,17 @@ impl LifecycleAlternative { /// One unique summary-state deployment. Shared `Rc` nodes are emitted once. #[derive(Debug, Clone)] -pub struct StateDeployment { +pub struct SummaryMaintenanceDeployment { pub summary_index: usize, pub summary: Rc, - pub selected: Option, - pub selected_maintenance_mode: Option, - pub evaluation_schedule: Option, - pub output_representation: OutputRepresentation, - pub alternatives: Vec, + pub summary_maintenance_lifecycle_guarantee: Option, + pub alternatives: Vec, } #[derive(Debug, Clone)] -pub struct LifecyclePlan { +pub struct SummaryMaintenanceLifecyclePlan { pub root: Rc, - pub deployments: Vec, + pub deployments: Vec, pub horizon: Option, pub evaluation_rate: Option, pub update_rate: Option, @@ -132,9 +139,11 @@ impl<'a> WorkloadDemand<'a> { } #[derive(Debug, thiserror::Error)] -pub enum LifecyclePlanError { +pub enum SummaryMaintenanceLifecyclePlanError { #[error(transparent)] InvalidWorkload(#[from] WorkloadError), + #[error(transparent)] + InvalidExecutionPhases(#[from] asap_types::post_asap::PhaseError), #[error("optimization horizon must be finite and strictly positive")] InvalidHorizon, #[error("workload entry index {index} is out of bounds for {entry_count} entries")] @@ -145,6 +154,24 @@ pub enum LifecyclePlanError { DuplicateWorkloadEntry { index: usize }, } +#[derive(Debug, thiserror::Error)] +pub enum MaterializeSummaryMaintenanceLifecycleError { + #[error(transparent)] + Materialize(#[from] ImplementError), + #[error(transparent)] + SummaryMaintenance(#[from] SummaryMaintenanceLifecyclePlanError), +} + +/// Failure while deriving workload-aware candidate costs before global +/// selection. +#[derive(Debug, thiserror::Error)] +pub enum SummaryMaintenanceLifecycleSelectionError { + #[error(transparent)] + Recurrence(#[from] RecurrenceError), + #[error(transparent)] + SummaryMaintenance(#[from] SummaryMaintenanceLifecyclePlanError), +} + #[derive(Debug)] struct WorkloadFacts { reads: Option, @@ -160,22 +187,62 @@ struct WorkloadFacts { /// Validate a materialized plan, enumerate lifecycle alternatives for each /// unique summary state, and select the cheapest legal alternative whose cost /// is fully known. -pub fn plan_summary_lifecycles( +pub fn plan_summary_maintenance_lifecycles( root: Rc, demand: WorkloadDemand<'_>, now_ms: u64, horizon: Option, - capabilities: LifecycleCapabilities, + capabilities: SummaryMaintenanceLifecycleCapabilities, cost_model: &dyn CostModel, -) -> Result { +) -> Result { + plan_summary_maintenance_lifecycles_with_profile( + root, + demand, + now_ms, + horizon, + capabilities, + cost_model, + None, + ) +} + +/// Internal candidate-costing form. The workload binding supplies temporal +/// eligibility and data-arrival facts; `profile` supplies effective uses after +/// DAG path multiplicity has been propagated by `PlanSpace`. +fn plan_summary_maintenance_lifecycles_with_profile( + root: Rc, + demand: WorkloadDemand<'_>, + now_ms: u64, + horizon: Option, + capabilities: SummaryMaintenanceLifecycleCapabilities, + cost_model: &dyn CostModel, + profile: Option, +) -> Result { demand.workload.validate()?; + validate_execution_phases(&root)?; if horizon.is_some_and(|h| !h.0.is_finite() || h.0 <= 0.0) { - return Err(LifecyclePlanError::InvalidHorizon); + return Err(SummaryMaintenanceLifecyclePlanError::InvalidHorizon); + } + let mut facts = workload_facts(demand.workload, demand.entry_indices, now_ms, horizon)?; + if let Some(profile) = profile { + facts.one_time_invocations = u64::try_from(profile.one_shot_consumers).unwrap_or(u64::MAX); + facts.evaluation_rate = profile.evaluation_rate; + facts.update_rate = profile.update_rate; + facts.reads = match (profile.evaluation_rate, horizon) { + (Some(rate), Some(horizon)) => { + Some(profile.one_shot_consumers as f64 + rate.0 * horizon.0) + } + (Some(_), None) => None, + (None, _) if profile.one_shot_consumers > 0 => Some(profile.one_shot_consumers as f64), + // Preserve unknown recurrence from the normalized workload. An + // empty profile does not prove that the target is never read. + (None, _) => facts.reads, + }; } - let facts = workload_facts(demand.workload, demand.entry_indices, now_ms, horizon)?; let mut summaries = Vec::new(); collect_summary_aggs(&root, &mut HashSet::new(), &mut summaries); - let deployments: Vec = summaries + let components = summary_state_components(&summaries); + let mut deployments: Vec = summaries .into_iter() .enumerate() .map(|(summary_index, summary)| { @@ -183,49 +250,31 @@ pub fn plan_summary_lifecycles( &facts, horizon, capabilities, - cost_model.summary_lifecycle_capabilities(&summary), - cost_model.summary_lifecycle_cost_inputs(&summary), + cost_model.summary_maintenance_capabilities(&summary), + cost_model.summary_maintenance_lifecycle_cost_inputs(&summary), ); - let selected = alternatives - .iter() - .filter(|candidate| candidate.selectable()) - .min_by(|a, b| a.total_cost.unwrap().0.total_cmp(&b.total_cost.unwrap().0)) - .map(|candidate| (candidate.lifecycle.clone(), candidate.maintenance_mode)); - let evaluation_schedule = selected.as_ref().map(|(lifecycle, _)| match lifecycle { - StateLifecycle::Ephemeral => EvaluationSchedule::OneShot, - StateLifecycle::Prepared { .. } | StateLifecycle::Shared { .. } - if matches!( - facts.arrival, - DataArrival::ContinuouslyIngesting | DataArrival::Mixed - ) => - { - EvaluationSchedule::PerUpdate - } - StateLifecycle::Prepared { .. } => EvaluationSchedule::OneShot, - StateLifecycle::Shared { .. } => EvaluationSchedule::OnRead, - StateLifecycle::ContinuouslyMaintained => EvaluationSchedule::PerUpdate, - }); - StateDeployment { + SummaryMaintenanceDeployment { summary_index, summary, - selected: selected.as_ref().map(|(lifecycle, _)| lifecycle.clone()), - selected_maintenance_mode: selected.map(|(_, mode)| mode), - evaluation_schedule, - output_representation: OutputRepresentation::SummaryState, + summary_maintenance_lifecycle_guarantee: None, alternatives, } }) .collect(); + select_compatible_lifecycles(&mut deployments, &components, facts.arrival); let summary_total_cost = deployments.iter().try_fold(Cost::ZERO, |sum, deployment| { - let selected = deployment.selected.as_ref()?; + let selected = &deployment + .summary_maintenance_lifecycle_guarantee + .as_ref()? + .summary_maintenance_lifecycle; let cost = deployment .alternatives .iter() - .find(|alternative| &alternative.lifecycle == selected)? + .find(|alternative| &alternative.summary_maintenance_lifecycle == selected)? .total_cost?; Some(Cost(sum.0 + cost.0)) }); - Ok(LifecyclePlan { + Ok(SummaryMaintenanceLifecyclePlan { root, deployments, horizon, @@ -238,12 +287,100 @@ pub fn plan_summary_lifecycles( }) } +/// Rank semantic summary siblings using the cheapest legal +/// summary-maintenance lifecycle for each candidate before final global +/// selection. The candidate space stays compact; only cost overrides are +/// attached, so shared `Rc` identity and exact-composition commitments remain +/// the responsibility of `GlobalSelection`. +pub fn global_selection_with_summary_maintenance_lifecycles<'a, Id>( + space: &'a PlanSpace, + workload: &QueryWorkload, + root_workload_entries: &[usize], + now_ms: u64, + horizon: Option, + capabilities: SummaryMaintenanceLifecycleCapabilities, + cost_model: &dyn CostModel, +) -> Result, SummaryMaintenanceLifecycleSelectionError> { + let profiles = space.recurrence_profiles_from_workload( + workload, + root_workload_entries, + now_ms, + horizon, + )?; + let bindings = space.workload_entries_by_target(workload, root_workload_entries)?; + let mut costs = CandidateCostOverrides::default(); + for group in space.groups() { + let Some(entry_indices) = bindings.get(&Rc::as_ptr(&group.target)) else { + continue; + }; + for candidate in &group.candidates { + let Replacement::Summary(summary) = &candidate.replacement else { + continue; + }; + let plan = plan_summary_maintenance_lifecycles_with_profile( + Rc::clone(summary), + WorkloadDemand::new(workload, entry_indices), + now_ms, + horizon, + capabilities, + cost_model, + Some(profiles.for_target(&group.target)), + )?; + if !plan.deployments.is_empty() { + if let Some(total) = plan.summary_total_cost { + costs.insert(&group.target, candidate, total); + } + } + } + } + Ok(space.global_selection_with_candidate_costs(cost_model, &profiles, horizon, &costs)?) +} + +/// Materialize a globally selected phase-valid DAG and immediately attach +/// workload-aware summary maintenance deployments. +pub fn materialize_with_summary_maintenance_lifecycles( + selection: &GlobalSelection<'_>, + target: &Rc, + demand: WorkloadDemand<'_>, + now_ms: u64, + horizon: Option, + capabilities: SummaryMaintenanceLifecycleCapabilities, + cost_model: &dyn CostModel, +) -> Result, MaterializeSummaryMaintenanceLifecycleError> { + selection + .materialize(target)? + .map(|root| { + let mut plan = plan_summary_maintenance_lifecycles( + root, + demand, + now_ms, + horizon, + capabilities, + cost_model, + )?; + plan.raw_recompute_total_cost = cost_model + .raw_query_recompute_cost(target) + .zip(plan.expected_reads) + .map(|(per_read, reads)| Cost(per_read.0 * reads)); + if plan.raw_recompute_total_cost.is_some_and(|raw| { + plan.summary_total_cost + .is_none_or(|summary| raw.0 <= summary.0) + }) { + plan.root = crate::replacement::keep_pre_asap(target)?; + plan.deployments.clear(); + plan.selected_raw_recompute = true; + } + Ok(plan) + }) + .transpose() +} + fn workload_facts( workload: &QueryWorkload, workload_entry_indices: &[usize], now_ms: u64, horizon: Option, -) -> Result { +) -> Result { let mut one_time_invocations = 0u64; let mut recurring_reads = 0.0; let mut recurring_known = true; @@ -256,19 +393,19 @@ fn workload_facts( let entries: Vec<_> = workload.entries().collect(); if workload_entry_indices.is_empty() { - return Err(LifecyclePlanError::EmptyWorkloadDemand); + return Err(SummaryMaintenanceLifecyclePlanError::EmptyWorkloadDemand); } let mut seen_indices = HashSet::new(); for &index in workload_entry_indices { if !seen_indices.insert(index) { - return Err(LifecyclePlanError::DuplicateWorkloadEntry { index }); + return Err(SummaryMaintenanceLifecyclePlanError::DuplicateWorkloadEntry { index }); } - let entry = entries - .get(index) - .ok_or(LifecyclePlanError::InvalidWorkloadEntry { + let entry = entries.get(index).ok_or( + SummaryMaintenanceLifecyclePlanError::InvalidWorkloadEntry { index, entry_count: entries.len(), - })?; + }, + )?; requires_deletion |= entry.time_selection.lookback.is_some() && entry.time_selection.as_of.is_none() && matches!( @@ -384,10 +521,10 @@ fn workload_facts( fn alternatives_for( facts: &WorkloadFacts, horizon: Option, - capabilities: LifecycleCapabilities, - summary_capabilities: SummaryLifecycleCapabilities, - costs: LifecycleCostInputs, -) -> Vec { + capabilities: SummaryMaintenanceLifecycleCapabilities, + summary_capabilities: SummaryMaintenanceCapabilities, + costs: SummaryMaintenanceLifecycleCostInputs, +) -> Vec { let alternatives = vec![ ephemeral(facts, capabilities, &costs), prepared(facts, capabilities, summary_capabilities, &costs), @@ -399,15 +536,14 @@ fn alternatives_for( fn ephemeral( facts: &WorkloadFacts, - capabilities: LifecycleCapabilities, - costs: &LifecycleCostInputs, -) -> LifecycleAlternative { - let lifecycle = StateLifecycle::Ephemeral; + capabilities: SummaryMaintenanceLifecycleCapabilities, + costs: &SummaryMaintenanceLifecycleCostInputs, +) -> SummaryMaintenanceLifecycleAlternative { + let lifecycle = SummaryMaintenanceLifecycle::Ephemeral; if !capabilities.ephemeral { return rejected( lifecycle, - SummaryMaintenanceMode::DirectBuild, - LifecycleRejection::UnsupportedByRuntime, + SummaryMaintenanceLifecycleRejection::UnsupportedByRuntime, ); } let total_cost = zip_costs(&[ @@ -419,7 +555,6 @@ fn ephemeral( .map(|(per_read, reads)| Cost(per_read * reads)); costed_or_unknown( lifecycle, - SummaryMaintenanceMode::DirectBuild, total_cost, vec!["state is rebuilt per invocation".into()], ) @@ -427,43 +562,40 @@ fn ephemeral( fn prepared( facts: &WorkloadFacts, - capabilities: LifecycleCapabilities, - summary_capabilities: SummaryLifecycleCapabilities, - costs: &LifecycleCostInputs, -) -> LifecycleAlternative { + capabilities: SummaryMaintenanceLifecycleCapabilities, + summary_capabilities: SummaryMaintenanceCapabilities, + costs: &SummaryMaintenanceLifecycleCostInputs, +) -> SummaryMaintenanceLifecycleAlternative { if !facts.prepared_eligible { return rejected( - StateLifecycle::Prepared { + SummaryMaintenanceLifecycle::Prepared { activate_at: TimestampMs(0), retire_at: TimestampMs(0), }, - retained_mode(facts), - LifecycleRejection::RequiresPredictableOneTimeQuery, + SummaryMaintenanceLifecycleRejection::RequiresPredictableOneTimeQuery, ); } let Some((activate_at, retire_at)) = facts.prepared_window else { return rejected( - StateLifecycle::Prepared { + SummaryMaintenanceLifecycle::Prepared { activate_at: TimestampMs(0), retire_at: TimestampMs(0), }, - retained_mode(facts), - LifecycleRejection::RequiresPredictableOneTimeQuery, + SummaryMaintenanceLifecycleRejection::RequiresPredictableOneTimeQuery, ); }; - let lifecycle = StateLifecycle::Prepared { + let lifecycle = SummaryMaintenanceLifecycle::Prepared { activate_at, retire_at, }; if !capabilities.prepared { return rejected( lifecycle, - retained_mode(facts), - LifecycleRejection::UnsupportedByRuntime, + SummaryMaintenanceLifecycleRejection::UnsupportedByRuntime, ); } if let Some(rejection) = maintenance_capability_rejection(facts, summary_capabilities) { - return rejected(lifecycle, retained_mode(facts), rejection); + return rejected(lifecycle, rejection); } let seconds = retire_at.0.saturating_sub(activate_at.0) as f64 / 1000.0; let maintenance = maintenance_cost(facts, costs, seconds); @@ -485,7 +617,6 @@ fn prepared( }; costed_or_unknown( lifecycle, - retained_mode(facts), total_cost, vec!["activation and retirement come from the declared schedule".into()], ) @@ -494,41 +625,37 @@ fn prepared( fn shared( facts: &WorkloadFacts, horizon: Option, - capabilities: LifecycleCapabilities, - summary_capabilities: SummaryLifecycleCapabilities, - costs: &LifecycleCostInputs, -) -> LifecycleAlternative { - let lifecycle = StateLifecycle::Shared { + capabilities: SummaryMaintenanceLifecycleCapabilities, + summary_capabilities: SummaryMaintenanceCapabilities, + costs: &SummaryMaintenanceLifecycleCostInputs, +) -> SummaryMaintenanceLifecycleAlternative { + let lifecycle = SummaryMaintenanceLifecycle::Shared { retention: asap_types::workload::DurationMs(horizon.map_or(0, |h| (h.0 * 1000.0) as u64)), }; if !capabilities.shared { return rejected( lifecycle, - retained_mode(facts), - LifecycleRejection::UnsupportedByRuntime, + SummaryMaintenanceLifecycleRejection::UnsupportedByRuntime, ); } if let Some(rejection) = maintenance_capability_rejection(facts, summary_capabilities) { - return rejected(lifecycle, retained_mode(facts), rejection); + return rejected(lifecycle, rejection); } if facts.reads.is_none_or(|reads| reads <= 1.0) { return rejected( lifecycle, - retained_mode(facts), - LifecycleRejection::RequiresMultipleReads, + SummaryMaintenanceLifecycleRejection::RequiresMultipleReads, ); } let Some(horizon) = horizon else { return rejected( lifecycle, - retained_mode(facts), - LifecycleRejection::RequiresHorizon, + SummaryMaintenanceLifecycleRejection::RequiresHorizon, ); }; let total_cost = retained_cost(facts, costs, horizon.0); costed_or_unknown( lifecycle, - retained_mode(facts), total_cost, vec!["one state is shared across reads".into()], ) @@ -537,16 +664,15 @@ fn shared( fn continuous( facts: &WorkloadFacts, horizon: Option, - capabilities: LifecycleCapabilities, - summary_capabilities: SummaryLifecycleCapabilities, - costs: &LifecycleCostInputs, -) -> LifecycleAlternative { - let lifecycle = StateLifecycle::ContinuouslyMaintained; + capabilities: SummaryMaintenanceLifecycleCapabilities, + summary_capabilities: SummaryMaintenanceCapabilities, + costs: &SummaryMaintenanceLifecycleCostInputs, +) -> SummaryMaintenanceLifecycleAlternative { + let lifecycle = SummaryMaintenanceLifecycle::ContinuouslyMaintained; if !capabilities.continuously_maintained { return rejected( lifecycle, - SummaryMaintenanceMode::Incremental, - LifecycleRejection::UnsupportedByRuntime, + SummaryMaintenanceLifecycleRejection::UnsupportedByRuntime, ); } if !matches!( @@ -555,31 +681,27 @@ fn continuous( ) { return rejected( lifecycle, - SummaryMaintenanceMode::Incremental, - LifecycleRejection::RequiresContinuousData, + SummaryMaintenanceLifecycleRejection::RequiresContinuousData, ); } if facts.update_rate.is_none() { return rejected( lifecycle, - SummaryMaintenanceMode::Incremental, - LifecycleRejection::MissingOrStaleIngestionRate, + SummaryMaintenanceLifecycleRejection::MissingOrStaleIngestionRate, ); } if let Some(rejection) = maintenance_capability_rejection(facts, summary_capabilities) { - return rejected(lifecycle, SummaryMaintenanceMode::Incremental, rejection); + return rejected(lifecycle, rejection); } let Some(horizon) = horizon else { return rejected( lifecycle, - SummaryMaintenanceMode::Incremental, - LifecycleRejection::RequiresHorizon, + SummaryMaintenanceLifecycleRejection::RequiresHorizon, ); }; let total_cost = retained_cost(facts, costs, horizon.0); costed_or_unknown( lifecycle, - SummaryMaintenanceMode::Incremental, total_cost, vec!["updates are applied for the optimization horizon".into()], ) @@ -587,27 +709,31 @@ fn continuous( fn maintenance_capability_rejection( facts: &WorkloadFacts, - capabilities: SummaryLifecycleCapabilities, -) -> Option { + capabilities: SummaryMaintenanceCapabilities, +) -> Option { if matches!( facts.arrival, DataArrival::ContinuouslyIngesting | DataArrival::Mixed ) && !capabilities.incremental_update { - Some(LifecycleRejection::SummaryDoesNotSupportIncrementalUpdates) + Some(SummaryMaintenanceLifecycleRejection::SummaryDoesNotSupportIncrementalUpdates) } else if matches!( facts.arrival, DataArrival::ContinuouslyIngesting | DataArrival::Mixed ) && facts.requires_deletion && !capabilities.delete { - Some(LifecycleRejection::SummaryDoesNotSupportDeletion) + Some(SummaryMaintenanceLifecycleRejection::SummaryDoesNotSupportDeletion) } else { None } } -fn retained_cost(facts: &WorkloadFacts, costs: &LifecycleCostInputs, seconds: f64) -> Option { +fn retained_cost( + facts: &WorkloadFacts, + costs: &SummaryMaintenanceLifecycleCostInputs, + seconds: f64, +) -> Option { let reads = facts.reads?; let maintenance = maintenance_cost(facts, costs, seconds)?; Some(Cost( @@ -621,7 +747,7 @@ fn retained_cost(facts: &WorkloadFacts, costs: &LifecycleCostInputs, seconds: f6 fn maintenance_cost( facts: &WorkloadFacts, - costs: &LifecycleCostInputs, + costs: &SummaryMaintenanceLifecycleCostInputs, seconds: f64, ) -> Option { match facts.arrival { @@ -640,45 +766,32 @@ fn zip_costs(costs: &[Option]) -> Option { } fn costed_or_unknown( - lifecycle: StateLifecycle, - maintenance_mode: SummaryMaintenanceMode, + summary_maintenance_lifecycle: SummaryMaintenanceLifecycle, total_cost: Option, assumptions: Vec, -) -> LifecycleAlternative { - LifecycleAlternative { - lifecycle, - maintenance_mode, +) -> SummaryMaintenanceLifecycleAlternative { + SummaryMaintenanceLifecycleAlternative { + summary_maintenance_lifecycle, total_cost, rejection: total_cost .is_none() - .then_some(LifecycleRejection::MissingCostEvidence), + .then_some(SummaryMaintenanceLifecycleRejection::MissingCostEvidence), assumptions, } } fn rejected( - lifecycle: StateLifecycle, - maintenance_mode: SummaryMaintenanceMode, - rejection: LifecycleRejection, -) -> LifecycleAlternative { - LifecycleAlternative { - lifecycle, - maintenance_mode, + summary_maintenance_lifecycle: SummaryMaintenanceLifecycle, + rejection: SummaryMaintenanceLifecycleRejection, +) -> SummaryMaintenanceLifecycleAlternative { + SummaryMaintenanceLifecycleAlternative { + summary_maintenance_lifecycle, total_cost: None, rejection: Some(rejection), assumptions: Vec::new(), } } -fn retained_mode(facts: &WorkloadFacts) -> SummaryMaintenanceMode { - match facts.arrival { - DataArrival::ContinuouslyIngesting | DataArrival::Mixed => { - SummaryMaintenanceMode::Incremental - } - DataArrival::AtRest | DataArrival::Unknown => SummaryMaintenanceMode::DirectBuild, - } -} - fn collect_summary_aggs( node: &Rc, seen: &mut HashSet<*const SummaryNode>, @@ -709,18 +822,147 @@ fn collect_summary_aggs( collect_summary_aggs(child, seen, output); } } + SummaryExpr::UpdateTransform { child, .. } + | SummaryExpr::ReadoutPostProcess { child, .. } => { + collect_summary_aggs(child, seen, output) + } SummaryExpr::KeepPreAsap(_) => {} } } +fn evaluation_schedule( + lifecycle: &SummaryMaintenanceLifecycle, + arrival: DataArrival, +) -> EvaluationSchedule { + match lifecycle { + SummaryMaintenanceLifecycle::Ephemeral => EvaluationSchedule::OneShot, + SummaryMaintenanceLifecycle::Prepared { .. } + | SummaryMaintenanceLifecycle::Shared { .. } + if matches!( + arrival, + DataArrival::ContinuouslyIngesting | DataArrival::Mixed + ) => + { + EvaluationSchedule::PerUpdate + } + SummaryMaintenanceLifecycle::Prepared { .. } => EvaluationSchedule::OneShot, + SummaryMaintenanceLifecycle::Shared { .. } => EvaluationSchedule::OnRead, + SummaryMaintenanceLifecycle::ContinuouslyMaintained => EvaluationSchedule::PerUpdate, + } +} + +/// Summary states composed on one maintenance path must be produced on the +/// same schedule. Return a component id for each collected `SummaryAgg`. +fn summary_state_components(summaries: &[Rc]) -> Vec { + let indices: HashMap<_, _> = summaries + .iter() + .enumerate() + .map(|(index, summary)| (Rc::as_ptr(summary), index)) + .collect(); + let mut parents: Vec<_> = (0..summaries.len()).collect(); + + fn find(parents: &mut [usize], index: usize) -> usize { + if parents[index] != index { + parents[index] = find(parents, parents[index]); + } + parents[index] + } + + for (parent_index, summary) in summaries.iter().enumerate() { + let SummaryExpr::SummaryAgg { child, .. } = &summary.expr else { + continue; + }; + if produced_availability(&child.expr) != Some(ExecutionAvailability::SummaryState) { + continue; + } + let mut descendants = Vec::new(); + collect_summary_aggs(child, &mut HashSet::new(), &mut descendants); + for descendant in descendants { + let child_index = indices[&Rc::as_ptr(&descendant)]; + let parent_root = find(&mut parents, parent_index); + let child_root = find(&mut parents, child_index); + parents[child_root] = parent_root; + } + } + (0..parents.len()) + .map(|index| find(&mut parents, index)) + .collect() +} + +fn select_compatible_lifecycles( + deployments: &mut [SummaryMaintenanceDeployment], + components: &[usize], + arrival: DataArrival, +) { + let component_ids: HashSet<_> = components.iter().copied().collect(); + for component in component_ids { + let members: Vec<_> = components + .iter() + .enumerate() + .filter_map(|(index, &id)| (id == component).then_some(index)) + .collect(); + let selected_schedule = [ + EvaluationSchedule::OneShot, + EvaluationSchedule::PerUpdate, + EvaluationSchedule::OnRead, + ] + .into_iter() + .filter_map(|schedule| { + members + .iter() + .try_fold(0.0, |sum, &index| { + deployments[index] + .alternatives + .iter() + .filter(|candidate| { + candidate.selectable() + && evaluation_schedule( + &candidate.summary_maintenance_lifecycle, + arrival, + ) == schedule + }) + .map(|candidate| candidate.total_cost.unwrap().0) + .min_by(f64::total_cmp) + .map(|cost| sum + cost) + }) + .map(|cost| (schedule, cost)) + }) + .min_by(|(_, a), (_, b)| a.total_cmp(b)) + .map(|(schedule, _)| schedule); + + let Some(schedule) = selected_schedule else { + continue; + }; + for index in members { + let selected = deployments[index] + .alternatives + .iter() + .filter(|candidate| { + candidate.selectable() + && evaluation_schedule(&candidate.summary_maintenance_lifecycle, arrival) + == schedule + }) + .min_by(|a, b| a.total_cost.unwrap().0.total_cmp(&b.total_cost.unwrap().0)); + deployments[index].summary_maintenance_lifecycle_guarantee = + selected.map(|candidate| SummaryMaintenanceLifecycleGuarantee { + summary_maintenance_lifecycle: candidate.summary_maintenance_lifecycle.clone(), + evaluation_schedule: schedule, + output_representation: OutputRepresentation::SummaryState, + }); + } + } +} + #[cfg(test)] mod tests { use super::*; use asap_types::post_asap::{ - ExactKind, ExactParams, GroupingStrategy, ResultGuarantee, SummaryFamilyType, SummaryField, - SummarySchema, + ExactKind, ExactParams, GroupingStrategy, ResultGuarantee, SketchAlgorithm, + SummaryFamilyType, SummaryField, SummarySchema, }; + use asap_types::pre_asap::AggIntent; use asap_types::pre_asap::{Column, ColumnRef, DataType, QueryExpr, Reduction, Schema, Source}; + use asap_types::types::AccuracyTarget; use asap_types::workload::{ BatchEntry, DataWorkload, DurationMs, Evidence, EvidenceSource, Predictability, Query, QueryLanguage, QueryRequirements, Rate, RepeatingEntry, RepetitionInterval, TimeSelection, @@ -737,8 +979,11 @@ mod tests { candidates.to_vec() } - fn summary_lifecycle_cost_inputs(&self, _summary: &SummaryNode) -> LifecycleCostInputs { - LifecycleCostInputs { + fn summary_maintenance_lifecycle_cost_inputs( + &self, + _summary: &SummaryNode, + ) -> SummaryMaintenanceLifecycleCostInputs { + SummaryMaintenanceLifecycleCostInputs { build_cost: Some(Cost(10.0)), maintenance_cost_per_update: Some(Cost(1.0)), summary_read_cost: Some(Cost(1.0)), @@ -747,11 +992,11 @@ mod tests { } } - fn summary_lifecycle_capabilities( + fn summary_maintenance_capabilities( &self, _summary: &SummaryNode, - ) -> SummaryLifecycleCapabilities { - SummaryLifecycleCapabilities { + ) -> SummaryMaintenanceCapabilities { + SummaryMaintenanceCapabilities { incremental_update: true, merge: true, delete: true, @@ -759,6 +1004,36 @@ mod tests { } } + struct RawCheaper; + + impl CostModel for RawCheaper { + fn rank_candidates( + &self, + _intent: &asap_types::pre_asap::AggIntent, + candidates: &[asap_types::post_asap::SketchAlgorithm], + ) -> Vec { + candidates.to_vec() + } + + fn summary_maintenance_lifecycle_cost_inputs( + &self, + summary: &SummaryNode, + ) -> SummaryMaintenanceLifecycleCostInputs { + UnitCosts.summary_maintenance_lifecycle_cost_inputs(summary) + } + + fn summary_maintenance_capabilities( + &self, + summary: &SummaryNode, + ) -> SummaryMaintenanceCapabilities { + UnitCosts.summary_maintenance_capabilities(summary) + } + + fn raw_query_recompute_cost(&self, _target: &QueryExpr) -> Option { + Some(Cost(1.0)) + } + } + struct NoDelete; impl CostModel for NoDelete { @@ -770,15 +1045,18 @@ mod tests { candidates.to_vec() } - fn summary_lifecycle_cost_inputs(&self, summary: &SummaryNode) -> LifecycleCostInputs { - UnitCosts.summary_lifecycle_cost_inputs(summary) + fn summary_maintenance_lifecycle_cost_inputs( + &self, + summary: &SummaryNode, + ) -> SummaryMaintenanceLifecycleCostInputs { + UnitCosts.summary_maintenance_lifecycle_cost_inputs(summary) } - fn summary_lifecycle_capabilities( + fn summary_maintenance_capabilities( &self, _summary: &SummaryNode, - ) -> SummaryLifecycleCapabilities { - SummaryLifecycleCapabilities { + ) -> SummaryMaintenanceCapabilities { + SummaryMaintenanceCapabilities { incremental_update: true, merge: true, delete: false, @@ -786,6 +1064,90 @@ mod tests { } } + struct SummaryMaintenancePrefersDdSketch; + + impl CostModel for SummaryMaintenancePrefersDdSketch { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + // Preserve semantic mapping's KLL-first order. The lifecycle + // total below must be what changes the final choice. + candidates.to_vec() + } + + fn summary_maintenance_lifecycle_cost_inputs( + &self, + summary: &SummaryNode, + ) -> SummaryMaintenanceLifecycleCostInputs { + let build = match sketch_algorithm(summary) { + Some(SketchAlgorithm::Kll) => 100.0, + Some(SketchAlgorithm::DDSketch) => 1.0, + _ => 10.0, + }; + SummaryMaintenanceLifecycleCostInputs { + build_cost: Some(Cost(build)), + maintenance_cost_per_update: Some(Cost(1.0)), + summary_read_cost: Some(Cost(1.0)), + retention_cost_rate: Some(CostRate(0.1)), + retirement_cost: Some(Cost(1.0)), + } + } + } + + struct IncompatibleNestedCosts; + + impl CostModel for IncompatibleNestedCosts { + fn rank_candidates( + &self, + _intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + candidates.to_vec() + } + + fn summary_maintenance_lifecycle_cost_inputs( + &self, + summary: &SummaryNode, + ) -> SummaryMaintenanceLifecycleCostInputs { + let is_leaf = matches!( + summary.expr, + SummaryExpr::SummaryAgg { ref child, .. } + if matches!(child.expr, SummaryExpr::KeepPreAsap(_)) + ); + SummaryMaintenanceLifecycleCostInputs { + build_cost: Some(Cost(if is_leaf { 1.0 } else { 100.0 })), + maintenance_cost_per_update: Some(Cost(if is_leaf { 100.0 } else { 0.0 })), + summary_read_cost: Some(Cost::ZERO), + retention_cost_rate: Some(CostRate(0.0)), + retirement_cost: Some(Cost::ZERO), + } + } + + fn summary_maintenance_capabilities( + &self, + _summary: &SummaryNode, + ) -> SummaryMaintenanceCapabilities { + SummaryMaintenanceCapabilities { + incremental_update: true, + merge: true, + delete: true, + } + } + } + + fn sketch_algorithm(node: &SummaryNode) -> Option { + match &node.expr { + SummaryExpr::SummaryEstimate { summary_input, .. } => sketch_algorithm(summary_input), + SummaryExpr::SummaryAgg { + family: SummaryFamilyType::Sketch(kind, _), + .. + } => Some(kind.algorithm().clone()), + _ => None, + } + } + fn query_root() -> Rc { query_root_for("m") } @@ -807,6 +1169,30 @@ mod tests { }) } + fn sum_query() -> Rc { + Rc::new(QueryExpr::Aggregate { + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::Sum { col: None }], + output_names: vec![], + having: None, + child: query_root(), + }) + } + + fn quantile_query() -> Rc { + Rc::new(QueryExpr::Aggregate { + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::Quantile { + col: None, + q: 0.99, + accuracy: AccuracyTarget::Epsilon(0.1), + }], + output_names: vec![], + having: None, + child: query_root(), + }) + } + fn summary() -> Rc { let child = Rc::new(SummaryNode { expr: SummaryExpr::KeepPreAsap(query_root()), @@ -837,6 +1223,29 @@ mod tests { }) } + fn nested_summary() -> Rc { + let child = summary(); + let family = SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); + Rc::new(SummaryNode { + expr: SummaryExpr::SummaryAgg { + child, + family: family.clone(), + col: ColumnRef::Named("state".into()), + reduction: Reduction::by(vec![]), + grouping: GroupingStrategy::default(), + }, + schema: SummarySchema { + fields: vec![SummaryField { + name: "state".into(), + dtype: family, + nullable: false, + }], + time_index: None, + }, + guarantee: Some(ResultGuarantee::exact("nested sum")), + }) + } + fn batch(predictability: Predictability) -> BatchEntry { BatchEntry { query: Query("sum(m)".into()), @@ -891,9 +1300,18 @@ mod tests { } } + fn selected_summary_maintenance_lifecycle( + deployment: &SummaryMaintenanceDeployment, + ) -> Option<&SummaryMaintenanceLifecycle> { + deployment + .summary_maintenance_lifecycle_guarantee + .as_ref() + .map(|guarantee| &guarantee.summary_maintenance_lifecycle) + } + #[test] fn unpredictable_one_time_at_rest_selects_ephemeral() { - let plan = plan_summary_lifecycles( + let plan = plan_summary_maintenance_lifecycles( summary(), WorkloadDemand::new( &workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()), @@ -901,18 +1319,23 @@ mod tests { ), 1_000, None, - LifecycleCapabilities::ALL, + SummaryMaintenanceLifecycleCapabilities::ALL, &UnitCosts, ) .unwrap(); assert_eq!(plan.deployments.len(), 1); assert_eq!( - plan.deployments[0].selected, - Some(StateLifecycle::Ephemeral) + selected_summary_maintenance_lifecycle(&plan.deployments[0]), + Some(&SummaryMaintenanceLifecycle::Ephemeral) ); + let guarantee = plan.deployments[0] + .summary_maintenance_lifecycle_guarantee + .as_ref() + .unwrap(); + assert_eq!(guarantee.evaluation_schedule, EvaluationSchedule::OneShot); assert_eq!( - plan.deployments[0].selected_maintenance_mode, - Some(SummaryMaintenanceMode::DirectBuild) + guarantee.output_representation, + OutputRepresentation::SummaryState ); assert_eq!( plan.deployments[0].alternatives[0].total_cost, @@ -926,12 +1349,12 @@ mod tests { known_at: Some(TimestampMs(1_000)), }); entry.execute_at = Some(TimestampMs(11_000)); - let plan = plan_summary_lifecycles( + let plan = plan_summary_maintenance_lifecycles( summary(), WorkloadDemand::new(&workload(vec![entry], vec![], at_rest()), &[0]), 1_000, None, - LifecycleCapabilities::ALL, + SummaryMaintenanceLifecycleCapabilities::ALL, &UnitCosts, ) .unwrap(); @@ -940,41 +1363,65 @@ mod tests { assert_eq!(prepared.total_cost, Some(Cost(13.0))); } + #[test] + fn nested_summary_lifecycles_have_compatible_evaluation_schedules() { + let workload = workload(vec![], vec![repeating()], continuous(1_000, 20_000)); + let plan = plan_summary_maintenance_lifecycles( + nested_summary(), + WorkloadDemand::new(&workload, &[0]), + 1_000, + Some(Horizon(10.0)), + SummaryMaintenanceLifecycleCapabilities::ALL, + &IncompatibleNestedCosts, + ) + .unwrap(); + + assert_eq!(plan.deployments.len(), 2); + let schedules: HashSet<_> = plan + .deployments + .iter() + .map(|deployment| { + deployment + .summary_maintenance_lifecycle_guarantee + .as_ref() + .unwrap() + .evaluation_schedule + }) + .collect(); + assert_eq!(schedules.len(), 1); + } + #[test] fn repeated_at_rest_selects_shared_without_inventing_updates() { - let plan = plan_summary_lifecycles( + let plan = plan_summary_maintenance_lifecycles( summary(), WorkloadDemand::new(&workload(vec![], vec![repeating()], at_rest()), &[0]), 1_000, Some(Horizon(10.0)), - LifecycleCapabilities::ALL, + SummaryMaintenanceLifecycleCapabilities::ALL, &UnitCosts, ) .unwrap(); assert_eq!( - plan.deployments[0].selected, - Some(StateLifecycle::Shared { + selected_summary_maintenance_lifecycle(&plan.deployments[0]), + Some(&SummaryMaintenanceLifecycle::Shared { retention: DurationMs(10_000) }) ); - assert_eq!( - plan.deployments[0].selected_maintenance_mode, - Some(SummaryMaintenanceMode::DirectBuild) - ); assert_eq!( plan.deployments[0].alternatives[3].rejection, - Some(LifecycleRejection::RequiresContinuousData) + Some(SummaryMaintenanceLifecycleRejection::RequiresContinuousData) ); assert_eq!(plan.update_rate, None); } #[test] fn repeated_continuous_workload_can_select_continuous_maintenance() { - let capabilities = LifecycleCapabilities { + let capabilities = SummaryMaintenanceLifecycleCapabilities { shared: false, - ..LifecycleCapabilities::ALL + ..SummaryMaintenanceLifecycleCapabilities::ALL }; - let plan = plan_summary_lifecycles( + let plan = plan_summary_maintenance_lifecycles( summary(), WorkloadDemand::new( &workload(vec![], vec![repeating()], continuous(1_000, 60_000)), @@ -987,12 +1434,8 @@ mod tests { ) .unwrap(); assert_eq!( - plan.deployments[0].selected, - Some(StateLifecycle::ContinuouslyMaintained) - ); - assert_eq!( - plan.deployments[0].selected_maintenance_mode, - Some(SummaryMaintenanceMode::Incremental) + selected_summary_maintenance_lifecycle(&plan.deployments[0]), + Some(&SummaryMaintenanceLifecycle::ContinuouslyMaintained) ); assert_eq!(plan.evaluation_rate, Some(EvaluationRate(1.0))); assert_eq!(plan.update_rate, Some(UpdateRate(1.0))); @@ -1000,7 +1443,7 @@ mod tests { #[test] fn stale_ingestion_evidence_cannot_enable_continuous_maintenance() { - let plan = plan_summary_lifecycles( + let plan = plan_summary_maintenance_lifecycles( summary(), WorkloadDemand::new( &workload(vec![], vec![repeating()], continuous(1_000, 1_000)), @@ -1008,20 +1451,20 @@ mod tests { ), 3_000, Some(Horizon(10.0)), - LifecycleCapabilities::ALL, + SummaryMaintenanceLifecycleCapabilities::ALL, &UnitCosts, ) .unwrap(); assert_eq!( plan.deployments[0].alternatives[3].rejection, - Some(LifecycleRejection::MissingOrStaleIngestionRate) + Some(SummaryMaintenanceLifecycleRejection::MissingOrStaleIngestionRate) ); assert_eq!(plan.update_rate, None); } #[test] fn unknown_costs_do_not_make_a_long_lived_lifecycle_win() { - let plan = plan_summary_lifecycles( + let plan = plan_summary_maintenance_lifecycles( summary(), WorkloadDemand::new( &workload(vec![], vec![repeating()], continuous(1_000, 60_000)), @@ -1029,11 +1472,14 @@ mod tests { ), 1_000, Some(Horizon(10.0)), - LifecycleCapabilities::ALL, + SummaryMaintenanceLifecycleCapabilities::ALL, &crate::cost_model::DefaultCostModel, ) .unwrap(); - assert_eq!(plan.deployments[0].selected, None); + assert_eq!( + selected_summary_maintenance_lifecycle(&plan.deployments[0]), + None + ); assert!(plan.deployments[0] .alternatives .iter() @@ -1042,7 +1488,7 @@ mod tests { #[test] fn unrelated_workload_entries_do_not_create_reuse_for_a_target() { - let plan = plan_summary_lifecycles( + let plan = plan_summary_maintenance_lifecycles( summary(), WorkloadDemand::new( &workload( @@ -1054,17 +1500,17 @@ mod tests { ), 1_000, Some(Horizon(10.0)), - LifecycleCapabilities::ALL, + SummaryMaintenanceLifecycleCapabilities::ALL, &UnitCosts, ) .unwrap(); assert_eq!( - plan.deployments[0].selected, - Some(StateLifecycle::Ephemeral) + selected_summary_maintenance_lifecycle(&plan.deployments[0]), + Some(&SummaryMaintenanceLifecycle::Ephemeral) ); assert_eq!( plan.deployments[0].alternatives[2].rejection, - Some(LifecycleRejection::RequiresMultipleReads) + Some(SummaryMaintenanceLifecycleRejection::RequiresMultipleReads) ); } @@ -1076,12 +1522,12 @@ mod tests { TimestampMs(5_000), TimestampMs(20_000), ]); - let plan = plan_summary_lifecycles( + let plan = plan_summary_maintenance_lifecycles( summary(), WorkloadDemand::new(&workload(vec![], vec![entry], at_rest()), &[0]), 1_000, Some(Horizon(10.0)), - LifecycleCapabilities::ALL, + SummaryMaintenanceLifecycleCapabilities::ALL, &UnitCosts, ) .unwrap(); @@ -1092,26 +1538,26 @@ mod tests { fn demand_binding_rejects_empty_and_duplicate_entries() { let workload = workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()); assert!(matches!( - plan_summary_lifecycles( + plan_summary_maintenance_lifecycles( summary(), WorkloadDemand::new(&workload, &[]), 1_000, None, - LifecycleCapabilities::ALL, + SummaryMaintenanceLifecycleCapabilities::ALL, &UnitCosts, ), - Err(LifecyclePlanError::EmptyWorkloadDemand) + Err(SummaryMaintenanceLifecyclePlanError::EmptyWorkloadDemand) )); assert!(matches!( - plan_summary_lifecycles( + plan_summary_maintenance_lifecycles( summary(), WorkloadDemand::new(&workload, &[0, 0]), 1_000, None, - LifecycleCapabilities::ALL, + SummaryMaintenanceLifecycleCapabilities::ALL, &UnitCosts, ), - Err(LifecyclePlanError::DuplicateWorkloadEntry { index: 0 }) + Err(SummaryMaintenanceLifecyclePlanError::DuplicateWorkloadEntry { index: 0 }) )); } @@ -1126,18 +1572,18 @@ mod tests { vec![], at_rest(), ); - let plan = plan_summary_lifecycles( + let plan = plan_summary_maintenance_lifecycles( summary(), WorkloadDemand::new(&workload, &[0, 1]), 1_000, Some(Horizon(10.0)), - LifecycleCapabilities::ALL, + SummaryMaintenanceLifecycleCapabilities::ALL, &UnitCosts, ) .unwrap(); assert_eq!( plan.deployments[0].alternatives[1].rejection, - Some(LifecycleRejection::RequiresPredictableOneTimeQuery) + Some(SummaryMaintenanceLifecycleRejection::RequiresPredictableOneTimeQuery) ); } @@ -1149,7 +1595,7 @@ mod tests { lookback: Some(DurationMs(60_000)), as_of: None, }; - let plan = plan_summary_lifecycles( + let plan = plan_summary_maintenance_lifecycles( summary(), WorkloadDemand::new( &workload(vec![], vec![entry], continuous(1_000, 60_000)), @@ -1157,16 +1603,95 @@ mod tests { ), 1_000, Some(Horizon(10.0)), - LifecycleCapabilities::ALL, + SummaryMaintenanceLifecycleCapabilities::ALL, &NoDelete, ) .unwrap(); assert_eq!( plan.deployments[0].alternatives[3].rejection, - Some(LifecycleRejection::SummaryDoesNotSupportDeletion) + Some(SummaryMaintenanceLifecycleRejection::SummaryDoesNotSupportDeletion) ); } + #[test] + fn lifecycle_cost_can_fall_back_to_raw_recomputation() { + let target = sum_query(); + let space = crate::replacement::search_workload(vec![("q", Rc::clone(&target))]); + let selection = space.global_selection(&RawCheaper); + let workload = workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()); + let plan = materialize_with_summary_maintenance_lifecycles( + &selection, + &space.roots[0].1, + WorkloadDemand::new(&workload, &[0]), + 1_000, + None, + SummaryMaintenanceLifecycleCapabilities::ALL, + &RawCheaper, + ) + .unwrap() + .unwrap(); + assert!(plan.selected_raw_recompute); + assert_eq!(plan.raw_recompute_total_cost, Some(Cost(1.0))); + assert!(plan.deployments.is_empty()); + assert!(matches!(plan.root.expr, SummaryExpr::KeepPreAsap(_))); + } + + #[test] + fn lifecycle_cost_reorders_semantic_summary_candidates_before_materialization() { + let target = quantile_query(); + let space = crate::replacement::search_workload(vec![("q", target)]); + let workload = workload(vec![batch(Predictability::AdHoc)], vec![], at_rest()); + + let selection = global_selection_with_summary_maintenance_lifecycles( + &space, + &workload, + &[0], + 1_000, + None, + SummaryMaintenanceLifecycleCapabilities::ALL, + &SummaryMaintenancePrefersDdSketch, + ) + .unwrap(); + let materialized = selection.materialize(&space.roots[0].1).unwrap().unwrap(); + + assert_eq!( + sketch_algorithm(&materialized), + Some(SketchAlgorithm::DDSketch) + ); + } + + #[test] + fn lifecycle_cost_counts_one_shared_summary_node_once() { + let shared = summary(); + let root = Rc::new(SummaryNode { + expr: SummaryExpr::SummaryMerge { + children: vec![Rc::clone(&shared), Rc::clone(&shared)], + }, + schema: shared.schema.clone(), + guarantee: None, + }); + let workload = workload( + vec![batch(Predictability::AdHoc), batch(Predictability::AdHoc)], + vec![], + at_rest(), + ); + let horizon = Some(Horizon(10.0)); + let plan = plan_summary_maintenance_lifecycles( + root, + WorkloadDemand::new(&workload, &[0, 1]), + 1_000, + horizon, + SummaryMaintenanceLifecycleCapabilities::ALL, + &UnitCosts, + ) + .unwrap(); + assert_eq!(plan.deployments.len(), 1); + assert!(matches!( + selected_summary_maintenance_lifecycle(&plan.deployments[0]), + Some(SummaryMaintenanceLifecycle::Shared { .. }) + )); + } + #[test] fn normalized_workload_drives_plan_space_recurrence_profiles() { let root = query_root(); diff --git a/crates/types/src/post_asap/mod.rs b/crates/types/src/post_asap/mod.rs index 5c572b05..83c31be3 100644 --- a/crates/types/src/post_asap/mod.rs +++ b/crates/types/src/post_asap/mod.rs @@ -29,7 +29,7 @@ pub mod expr; pub mod guarantee; -pub mod lifecycle; +pub mod summary_maintenance_lifecycle; pub mod query_time; pub mod schema; pub mod sketch; @@ -40,7 +40,10 @@ pub use guarantee::{ AccuracyError, BoundExpr, CompositionOperator, ErrorMetric, GuaranteeSource, ProbabilityExpr, ResultGuarantee, }; -pub use lifecycle::{EvaluationSchedule, OutputRepresentation, StateLifecycle}; +pub use summary_maintenance_lifecycle::{ + EvaluationSchedule, OutputRepresentation, SummaryMaintenanceLifecycle, + SummaryMaintenanceLifecycleGuarantee, +}; pub use query_time::{ classic_cms_sizing, cms_posterior_error_bound, count_sketch_posterior_error_bound, cu_sketch_posterior_error_bound, traditional_a_priori_bound, diff --git a/crates/types/src/post_asap/lifecycle.rs b/crates/types/src/post_asap/summary_maintenance_lifecycle.rs similarity index 50% rename from crates/types/src/post_asap/lifecycle.rs rename to crates/types/src/post_asap/summary_maintenance_lifecycle.rs index 995c2f79..09521bf5 100644 --- a/crates/types/src/post_asap/lifecycle.rs +++ b/crates/types/src/post_asap/summary_maintenance_lifecycle.rs @@ -1,7 +1,9 @@ -//! Physical lifecycle vocabulary for summary state. +//! Physical summary-maintenance lifecycle vocabulary. //! //! These choices are attached by physical planning; a `SummaryAgg` does not -//! imply continuous maintenance by itself. +//! imply continuous maintenance by itself. "Summary maintenance lifecycle" +//! is deliberately narrower than the end-to-end data lifecycle (collection, +//! transmission, storage, and analytics). use crate::workload::{DurationMs, TimestampMs}; @@ -24,7 +26,7 @@ pub enum OutputRepresentation { /// How long one planned summary state deployment exists. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum StateLifecycle { +pub enum SummaryMaintenanceLifecycle { Ephemeral, Prepared { activate_at: TimestampMs, @@ -35,3 +37,15 @@ pub enum StateLifecycle { }, ContinuouslyMaintained, } + +/// The lifecycle commitment emitted for one materialized summary deployment. +/// +/// This names the summary-maintenance promise explicitly so consumers do not +/// confuse it with guarantees about the broader data lifecycle. Accuracy is a +/// separate [`super::ResultGuarantee`]. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct SummaryMaintenanceLifecycleGuarantee { + pub summary_maintenance_lifecycle: SummaryMaintenanceLifecycle, + pub evaluation_schedule: EvaluationSchedule, + pub output_representation: OutputRepresentation, +} From dfc3f8c558ca9f2c8f8a048ae465f901a356f3be Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 30 Aug 2026 12:20:55 -0600 Subject: [PATCH 2/3] fix(cost): dispatch exact composition by placement --- crates/asap-aware-mapping/src/cost_model.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index 8fbbb464..df060801 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -136,7 +136,7 @@ impl MixedExecutionCapabilities { }; pub fn supports(self, placement: CompositionPlacement) -> bool { - match phase { + match placement { CompositionPlacement::PostProcess => self.exact_post_process, CompositionPlacement::Transform => self.exact_update_transform, } @@ -216,7 +216,7 @@ impl ExactCompositionCostInputs { /// The rate for whichever phase `phase` names — /// [`postprocess_plan_cost_rate`] or [`pretransform_plan_cost_rate`]. pub fn composed_plan_cost_rate(&self, placement: CompositionPlacement) -> Option { - match phase { + match placement { CompositionPlacement::PostProcess => postprocess_plan_cost_rate(self), CompositionPlacement::Transform => pretransform_plan_cost_rate(self), } From c517a1a24a8692c6680ad89806ab8136149af72c Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 30 Aug 2026 15:19:59 -0600 Subject: [PATCH 3/3] refactor(lifecycle): select and materialize maintenance modes --- crates/asap-aware-mapping/src/cost_model.rs | 364 +-------- crates/asap-aware-mapping/src/lib.rs | 18 +- crates/asap-aware-mapping/src/replacement.rs | 704 +++--------------- .../src/summary_maintenance_lifecycle.rs | 65 +- crates/types/src/post_asap/mod.rs | 10 +- .../summary_maintenance_lifecycle.rs | 2 + 6 files changed, 187 insertions(+), 976 deletions(-) diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index df060801..556afe89 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -35,14 +35,11 @@ //! ## CSE sharing (issue #237, #223 stage 4) //! //! [`CseCandidate`]/[`ShareDecision`]/[`CostModel::cse_share_decision`] below -//! provide the context-free fallback for whether a CSE-detected shared subtree +//! decide whether a CSE-detected shared subtree //! ([`asap_types::pre_asap::cse::share_common_subtrees`], issue #223 stages //! 1-2, PR #235) is actually worth sharing, via a real Volcano/Cascades-style -//! cost comparison rather than a fixed rule. Workload-aware selection uses -//! [`CostModel::cse_share_decision_with_recurrence`]; the target design also -//! expands each share candidate with its legal summary-maintenance lifecycles -//! before whole-plan ranking. See -//! `docs/design_docs/cost-model.md` for the full design discussion (why +//! cost comparison rather than a fixed rule. See +//! `docs/design_docs/cse-cost-model-decision.md` for the full design discussion (why //! cost-based, why not a full plan-search engine, the layering constraint //! that forces detection to stay cost-agnostic). //! [`PlanSpace::cost_sorted`](crate::replacement::PlanSpace::cost_sorted) @@ -59,10 +56,8 @@ use asap_types::pre_asap::agg_intent::AggIntent; use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::query_expr::QueryExpr; -use crate::exact_composition::{CompositionPlacement, ExactComposition}; use crate::recurrence::{ - self, CostRate, EvaluationRate, Horizon, RecurrenceCostExplanation, RecurrenceError, - RecurrenceProfile, + self, Horizon, RecurrenceCostExplanation, RecurrenceError, RecurrenceProfile, }; use crate::replacement::{ realize_child, Implementation, Replacement, ReplacementProvenance, ReplacementSubDAG, @@ -72,209 +67,6 @@ use crate::summary_maintenance_lifecycle::{ SummaryMaintenanceCapabilities, SummaryMaintenanceLifecycleCostInputs, }; -// ── Recurring-cost vocabulary for mixed exact/summary plans (issue #171) ── - -/// The unit a recurring cost is expressed in. One variant today; an enum so -/// a JSON/DAG export names the unit explicitly instead of a consumer -/// assuming it, and so a future per-resource unit can be added without -/// changing every hook's signature. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum CostUnit { - /// Abstract cost units per wall-clock second — the common currency - /// every recurring alternative (maintain-and-read vs. recompute-per-eval) - /// is compared in. - CostUnitsPerSecond, -} - -impl CostUnit { - /// Stable name for export (`"cost_units_per_second"`). - pub fn as_str(self) -> &'static str { - match self { - Self::CostUnitsPerSecond => "cost_units_per_second", - } - } -} - -/// Who produced a set of [`ExactCompositionCostInputs`], and under which -/// model version — carried into every composed decision's explanation and -/// DAG export so a reviewer can tell a deployment's measured numbers from -/// a placeholder. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CostProvenance { - /// The cost model's own name (e.g. `"DefaultCostModel"`). - pub model: String, - /// The model's own version string, whatever scheme it uses. - pub version: String, -} - -/// Which mixed-execution shapes the downstream runtime can actually -/// execute (issue #171). [`crate::exact_composition::ExactCompositionStrategy`] -/// proposes an `ExactPostProcess` candidate only when -/// `exact_post_process` is set, and an `ExactTransform` candidate only -/// when `exact_update_transform` is — a runtime that cannot run an exact -/// operator on the update path must never be handed one. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub struct MixedExecutionCapabilities { - /// The runtime can apply an exact operator to summary readouts at - /// query evaluation time. - pub exact_post_process: bool, - /// The runtime can apply an exact row transform on the update path, - /// feeding its output into maintained summary state. - pub exact_update_transform: bool, -} - -impl MixedExecutionCapabilities { - /// Neither shape supported. - pub const NONE: Self = Self { - exact_post_process: false, - exact_update_transform: false, - }; - /// Both shapes supported. - pub const ALL: Self = Self { - exact_post_process: true, - exact_update_transform: true, - }; - - pub fn supports(self, placement: CompositionPlacement) -> bool { - match placement { - CompositionPlacement::PostProcess => self.exact_post_process, - CompositionPlacement::Transform => self.exact_update_transform, - } - } -} - -/// What [`CostModel::exact_composition_cost_inputs`] is asked about: one -/// composed alternative at one site, paired with the concrete summary it -/// composes with. -#[derive(Debug, Clone, Copy)] -pub struct ExactCompositionCostRequest<'a> { - /// The pre-ASAP target the composed candidate replaces. - pub target: &'a QueryExpr, - /// The composition itself — phase, operator, child target. - pub composition: &'a ExactComposition, - /// For [`CompositionPlacement::PostProcess`]: the child target's *selected* - /// summary readout candidate the exact operator consumes. For - /// [`CompositionPlacement::Transform`]: the maintained summary *above* the - /// transform that consumes its output (the `SummaryAgg` this transform - /// feeds). Either way, the summary whose maintenance/read cost the - /// formula charges. - pub summary: &'a SummaryNode, - /// How many times this site actually runs once ancestors' own choices - /// are accounted for (see `PlanSpace::global_selection`). - pub effective_consumer_count: usize, -} - -/// Every input the issue #171 cost formulas need, each individually -/// optional: **an unknown stays `None` — never a zero** — so a formula -/// with a missing input yields no rate at all rather than a spuriously -/// cheap one, and global selection then keeps the conservative -/// `KeepPreAsap` behavior. A deployment model that wants defaults supplies -/// them explicitly by overriding [`CostModel::exact_composition_cost_inputs`]. -#[derive(Debug, Clone, PartialEq)] -pub struct ExactCompositionCostInputs { - /// Exact operator cost per row it processes — per readout row for a - /// post-process, per input row for an update-path transform. - pub exact_cost_per_row: Option, - /// Rows the exact operator consumes per evaluation (post-process) or - /// per update (transform). - pub expected_input_rows: Option, - /// Rows the exact operator emits per evaluation/update. - pub expected_output_rows: Option, - /// Cost of one update to the composed-with summary's maintained state. - pub summary_maintenance_cost_per_update: Option, - /// Cost of one readout of that summary at evaluation time. - pub summary_read_cost: Option, - /// Update (ingest) events per second reaching this site. - pub update_rate: Option, - /// Evaluations per second across every consumer of this site. - pub evaluation_rate: Option, - /// Cost of one full raw recompute of the target from pre-ASAP data — - /// the `KeepPreAsap` baseline's per-evaluation cost. - pub raw_recompute_cost: Option, - pub unit: CostUnit, - pub provenance: CostProvenance, -} - -impl ExactCompositionCostInputs { - /// Every input unknown, attributed to `provenance` — what a model that - /// has no statistics for a site returns. - pub fn unknown(provenance: CostProvenance) -> Self { - Self { - exact_cost_per_row: None, - expected_input_rows: None, - expected_output_rows: None, - summary_maintenance_cost_per_update: None, - summary_read_cost: None, - update_rate: None, - evaluation_rate: None, - raw_recompute_cost: None, - unit: CostUnit::CostUnitsPerSecond, - provenance, - } - } - - /// The rate for whichever phase `phase` names — - /// [`postprocess_plan_cost_rate`] or [`pretransform_plan_cost_rate`]. - pub fn composed_plan_cost_rate(&self, placement: CompositionPlacement) -> Option { - match placement { - CompositionPlacement::PostProcess => postprocess_plan_cost_rate(self), - CompositionPlacement::Transform => pretransform_plan_cost_rate(self), - } - } -} - -/// Outer exact post-process over a maintained summary: -/// -/// ```text -/// postprocess_plan_cost_rate = -/// update_rate * summary_maintenance_cost_per_update -/// + evaluation_rate * (summary_read_cost -/// + output_rows_per_eval * exact_postprocess_cost_per_row) -/// ``` -/// -/// `None` if any input is unknown — see [`ExactCompositionCostInputs`]. -pub fn postprocess_plan_cost_rate(inputs: &ExactCompositionCostInputs) -> Option { - let maintenance = inputs.update_rate? * inputs.summary_maintenance_cost_per_update?; - let per_eval = - inputs.summary_read_cost? + inputs.expected_output_rows? * inputs.exact_cost_per_row?; - let evaluation = inputs.evaluation_rate?.0 * per_eval; - finite_rate(maintenance + evaluation) -} - -/// Outer maintained summary over an exact update-time transform: -/// -/// ```text -/// pretransform_plan_cost_rate = -/// update_rate * (exact_transform_cost_per_input_row -/// + summary_maintenance_cost_per_update) -/// + evaluation_rate * summary_read_cost -/// ``` -/// -/// `None` if any input is unknown — see [`ExactCompositionCostInputs`]. -pub fn pretransform_plan_cost_rate(inputs: &ExactCompositionCostInputs) -> Option { - let per_update = inputs.exact_cost_per_row? + inputs.summary_maintenance_cost_per_update?; - let maintenance = inputs.update_rate? * per_update; - let evaluation = inputs.evaluation_rate?.0 * inputs.summary_read_cost?; - finite_rate(maintenance + evaluation) -} - -/// The raw/pre-ASAP fallback baseline: -/// -/// ```text -/// raw_recompute_cost_rate = evaluation_rate * raw_recompute_cost -/// ``` -/// -/// `None` if either input is unknown — see [`ExactCompositionCostInputs`]. -pub fn raw_recompute_cost_rate(inputs: &ExactCompositionCostInputs) -> Option { - finite_rate(inputs.evaluation_rate?.0 * inputs.raw_recompute_cost?) -} - -fn finite_rate(units_per_second: f64) -> Option { - units_per_second - .is_finite() - .then_some(CostRate(units_per_second)) -} - /// A CSE-detected, legality-gated shared subtree with two or more consumers /// — the unit [`CostModel::cse_share_decision`] decides over. Built by /// [`PlanSpace::cost_sorted`](crate::replacement::PlanSpace::cost_sorted) @@ -282,7 +74,7 @@ fn finite_rate(units_per_second: f64) -> Option { /// needs a representative bound node for a subtree that /// [`asap_types::pre_asap::cse::share_common_subtrees`] already collapsed /// onto one `Rc` for two or more workload roots. See -/// `docs/design_docs/cost-model.md`. +/// `docs/design_docs/cse-cost-model-decision.md`. pub struct CseCandidate<'a> { /// The shared pre-ASAP subtree itself. pub subtree: &'a QueryExpr, @@ -363,12 +155,12 @@ pub fn default_cse_recompute_cost(subtree: &QueryExpr) -> Cost { Cost(asap_types::pre_asap::cse::dag_node_count(subtree) as f64) } -/// Default context-free [`CostModel::cse_shared_maintenance_cost`]: a small +/// Default [`CostModel::cse_shared_maintenance_cost`]: a small /// per-[`SummaryFamilyType`] weight, scaled to the same order of magnitude /// as [`default_cse_recompute_cost`]'s typical output (a small node /// count, not a byte length), reflecting that families differ in how -/// expensive they are to maintain as shared state — an exact accumulator is -/// the cheapest (an O(1) merge), +/// expensive they are to keep *continuously updated* for the life of a +/// workload — an exact accumulator is the cheapest (an O(1) merge), /// sketches/samples cost more (a whole data structure to update per new /// row), wavelets/fitted models cost the most (coefficient/parameter /// maintenance). These weights are illustrative, not measured — a @@ -516,22 +308,19 @@ pub trait CostModel { /// Estimate the one-time cost of recomputing `candidate.subtree` /// independently at a single use site. Default: /// [`default_cse_recompute_cost`] (a structural-size proxy). See - /// `docs/design_docs/cost-model.md`. + /// `docs/design_docs/cse-cost-model-decision.md`. fn cse_recompute_cost(&self, candidate: &CseCandidate) -> Cost { default_cse_recompute_cost(candidate.subtree) } - /// Estimate a context-free proxy for maintaining `candidate.bound_summary` - /// as shared state. This fallback has no query recurrence, data arrival, - /// or horizon; workload-aware selection uses - /// [`Self::cse_share_decision_with_recurrence`], and full physical - /// selection additionally uses [`Self::summary_maintenance_lifecycle_cost_inputs`]. + /// Estimate the cost of maintaining `candidate.bound_summary` as one + /// continuously-updated shared summary for the life of the workload. /// Default: [`default_cse_shared_maintenance_cost`] (a per-family /// weight table), applied to whichever field of /// `candidate.bound_summary`'s output schema actually carries summary /// state (falls back to the cheapest, `Plain`, weight if none does — /// e.g. `bound_summary` is a passthrough `KeepPreAsap` node with nothing - /// summary-shaped to maintain). See `docs/design_docs/cost-model.md`. + /// summary-shaped to maintain). See `docs/design_docs/cse-cost-model-decision.md`. fn cse_shared_maintenance_cost(&self, candidate: &CseCandidate) -> Cost { let family = candidate .bound_summary @@ -550,7 +339,7 @@ pub trait CostModel { /// Decide whether to reuse one shared `SummaryNode` across every /// consumer of `candidate`, or bind each occurrence independently — a /// Volcano/Cascades-style cost comparison (issue #237, #223 stage 4; see - /// `docs/design_docs/cost-model.md`): share iff the estimated cost of + /// `docs/design_docs/cse-cost-model-decision.md`): share iff the estimated cost of /// maintaining one shared summary is no greater than the estimated total /// cost of recomputing it independently everywhere it's used. /// @@ -719,54 +508,10 @@ pub trait CostModel { f64::NAN } - /// Which mixed exact/summary execution shapes the downstream runtime - /// advertises (issue #171). Gates candidate *generation* in - /// [`crate::exact_composition::ExactCompositionStrategy`]: a shape the - /// runtime can't execute is never proposed, so it can't be selected - /// either. - /// - /// Default: [`MixedExecutionCapabilities::ALL`]. The built-in model - /// describes no particular runtime, and leaving both shapes *visible* - /// in `PlanSpace` (for explanations and the DAG viewer) is the more - /// informative default; selection is still gated separately by - /// [`Self::exact_composition_cost_inputs`], whose default supplies no - /// statistics, so nothing is ever *committed* to under the built-in - /// model. A deployment whose runtime lacks a shape narrows this. - fn mixed_execution_capabilities(&self) -> MixedExecutionCapabilities { - MixedExecutionCapabilities::ALL - } - - /// The statistics the issue #171 recurring-cost formulas need for one - /// composed alternative — see [`ExactCompositionCostInputs`] for each - /// input and [`postprocess_plan_cost_rate`]/ - /// [`pretransform_plan_cost_rate`]/[`raw_recompute_cost_rate`] for how - /// they combine. One structured hook rather than eight scalar ones, so - /// a deployment answers them all from one place (and can attach its own - /// [`CostProvenance`]). - /// - /// Default: every input unknown ([`ExactCompositionCostInputs::unknown`]) - /// — unknown is never zero, and with no rate derivable - /// `PlanSpace::global_selection` keeps the conservative `KeepPreAsap` - /// behavior for the site. A deployment that wants defaults must supply - /// them here explicitly. - fn exact_composition_cost_inputs( - &self, - request: &ExactCompositionCostRequest<'_>, - ) -> ExactCompositionCostInputs { - let _ = request; - ExactCompositionCostInputs::unknown(CostProvenance { - model: "CostModel::exact_composition_cost_inputs (default)".into(), - version: "unknown".into(), - }) - } - /// Primitive build, update, read, retention, and retirement costs used to - /// compare physical summary-state lifecycles. This is part of the same - /// cost model as candidate ranking and recurrence; summary maintenance - /// lifecycle planning does not introduce a second optimizer. - /// - /// The default leaves every value unknown, which prevents a long-lived - /// deployment from winning through optimistic zeroes. + /// compare physical summary-state lifecycles. Unknown values stay + /// unknown, preventing long-lived deployments from winning through + /// optimistic zeroes. fn summary_maintenance_lifecycle_cost_inputs( &self, _summary: &SummaryNode, @@ -784,8 +529,8 @@ pub trait CostModel { } /// Cost of evaluating `target` directly from its logical/raw inputs once. - /// When known, summary-maintenance-aware materialization compares this - /// fallback with the aggregate cost of the selected summary deployments. + /// When known, lifecycle-aware materialization compares this fallback with + /// the aggregate cost of the selected summary deployments. fn raw_query_recompute_cost(&self, _target: &QueryExpr) -> Option { None } @@ -946,12 +691,6 @@ impl CostModel for DefaultCostModel { (self.cse_recompute_cost(&cse) * consumer_count).0 } } - // A composed candidate is costed in cost-units-per-second by - // `PlanSpace::global_selection` against the child decision it - // is committed with — a different unit from this structural - // estimate, and unknowable here without that child. `NaN` - // keeps it from ever out-ranking a real estimate by accident. - Replacement::ExactComposition(_) => f64::NAN, } } } @@ -1096,73 +835,6 @@ mod tests { ); } - // ── Recurring-cost formulas (issue #171) ───────────────────────────── - - fn known_inputs() -> ExactCompositionCostInputs { - ExactCompositionCostInputs { - exact_cost_per_row: Some(0.1), - expected_input_rows: Some(50.0), - expected_output_rows: Some(10.0), - summary_maintenance_cost_per_update: Some(0.01), - summary_read_cost: Some(1.0), - update_rate: Some(100.0), - evaluation_rate: Some(EvaluationRate(2.0)), - raw_recompute_cost: Some(100.0), - unit: CostUnit::CostUnitsPerSecond, - provenance: CostProvenance { - model: "test".into(), - version: "1".into(), - }, - } - } - - #[test] - fn composition_formulas_match_the_issue_definitions() { - let inputs = known_inputs(); - // 100 * 0.01 + 2 * (1 + 10 * 0.1) = 1 + 4 = 5 - assert_eq!(postprocess_plan_cost_rate(&inputs).unwrap().0, 5.0); - // 100 * (0.1 + 0.01) + 2 * 1 = 11 + 2 = 13 - assert!((pretransform_plan_cost_rate(&inputs).unwrap().0 - 13.0).abs() < 1e-9); - // 2 * 100 - assert_eq!(raw_recompute_cost_rate(&inputs).unwrap().0, 200.0); - assert_eq!( - crate::recurrence::total_cost(CostRate(5.0), Horizon(10.0), Cost(3.0)), - Cost(53.0) - ); - } - - #[test] - fn a_missing_input_yields_no_rate_not_zero() { - let mut inputs = known_inputs(); - inputs.summary_maintenance_cost_per_update = None; - assert_eq!(postprocess_plan_cost_rate(&inputs), None); - assert_eq!(pretransform_plan_cost_rate(&inputs), None); - // The baseline doesn't need maintenance and is still known. - assert!(raw_recompute_cost_rate(&inputs).is_some()); - let unknown = ExactCompositionCostInputs::unknown(known_inputs().provenance); - assert_eq!(raw_recompute_cost_rate(&unknown), None); - } - - #[test] - fn default_model_advertises_capabilities_but_no_statistics() { - assert_eq!( - DefaultCostModel.mixed_execution_capabilities(), - MixedExecutionCapabilities::ALL - ); - assert!(MixedExecutionCapabilities::NONE - .supports(CompositionPlacement::PostProcess) - .not()); - } - - trait Not { - fn not(self) -> bool; - } - impl Not for bool { - fn not(self) -> bool { - !self - } - } - // ── CSE sharing (issue #237, #223 stage 4) ────────────────────────── use asap_types::post_asap::{ diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 04dafdde..ed1570e4 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -184,7 +184,6 @@ pub mod accuracy; pub mod accuracy_reconciliation; pub mod cost_model; -pub mod exact_composition; pub mod explanation; pub mod grouping; pub mod recurrence; @@ -200,12 +199,7 @@ pub use accuracy::{ PropagationStats, WorkloadAccuracyEvidence, }; pub use accuracy_reconciliation::AccuracyReconciliationStrategy; -pub use cost_model::{ - postprocess_plan_cost_rate, pretransform_plan_cost_rate, raw_recompute_cost_rate, CostModel, - CostProvenance, CostUnit, DefaultCostModel, ExactCompositionCostInputs, - ExactCompositionCostRequest, MixedExecutionCapabilities, -}; -pub use exact_composition::{CompositionPlacement, ExactComposition, ExactCompositionStrategy}; +pub use cost_model::{CostModel, DefaultCostModel}; pub use explanation::{ explain_replacements, explain_replacements_with, ExplanationKind, ReplacementExplanation, }; @@ -217,11 +211,11 @@ pub use recurrence::{ }; pub use replacement::{ default_strategies, default_strategies_with, search_workload, search_workload_with, - search_workload_with_targets, summary_candidates, CompositionDecision, GlobalSelection, - ImplementError, Implementation, Matcher, MemoGroup, PlanSpace, Proposals, RankedGroup, - RecurrenceProfileMap, RejectedCandidate, Replacement, ReplacementProvenance, - ReplacementStrategy, ReplacementSubDAG, SelectedGroup, SharedSubtreeStrategy, - SketchAlgorithmStrategy, TargetSubDAG, MAX_SEARCH_ITERATIONS, + search_workload_with_targets, summary_candidates, GlobalSelection, ImplementError, + Implementation, Matcher, MemoGroup, PlanSpace, Proposals, RankedGroup, RecurrenceProfileMap, + RejectedCandidate, Replacement, ReplacementProvenance, ReplacementStrategy, ReplacementSubDAG, + SelectedGroup, SharedSubtreeStrategy, SketchAlgorithmStrategy, TargetSubDAG, + MAX_SEARCH_ITERATIONS, }; pub use rewrite::AvgToSumOverCountStrategy; pub use summary_maintenance_lifecycle::{ diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index 62951124..2c19330c 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -241,7 +241,7 @@ //! //! [`PlanSpace::cost_sorted`] is the `sorted_by(cost_model)` step, and it //! reuses this crate's existing [`CostModel`] trait rather than inventing a -//! second cost interface (`docs/design_docs/cost-model.md`, +//! second cost interface (`docs/design_docs/cse-cost-model-decision.md`, //! issue #237, explicitly reasoned about *why* a narrow, direct cost //! comparison was enough for the CSE share/recompute decision alone, and //! flagged that a real search engine — this module — is where that stops @@ -345,17 +345,15 @@ //! multi-group joint optimization beyond this per-site recurrence is left //! for whenever that changes. -use std::cell::RefCell; use std::collections::{HashMap, HashSet, VecDeque}; +use asap_types::post_asap::{AccuracyError, CompositionOperator, GuaranteeSource, ResultGuarantee}; use asap_types::post_asap::{ - validate_execution_data_states_at, ExactKind, ExactOperatorSchemaError, ExactParams, - ExecutionDataState, ExecutionDataStateError, GroupingStrategy, SamplingKind, SamplingParams, - SketchAlgorithm, SketchKind, SketchParams, SketchQuery as PostAsapSketchQuery, StatModelKind, - StatModelParams, SummaryExpr, SummaryFamilyType, SummaryField, SummaryNode, SummarySchema, - WaveletKind, WaveletParams, + ExactKind, ExactParams, GroupingStrategy, SamplingKind, SamplingParams, SketchAlgorithm, + SketchKind, SketchParams, SketchQuery as PostAsapSketchQuery, StatModelKind, StatModelParams, + SummaryExpr, SummaryFamilyType, SummaryField, SummaryNode, SummarySchema, WaveletKind, + WaveletParams, }; -use asap_types::post_asap::{AccuracyError, CompositionOperator, GuaranteeSource, ResultGuarantee}; use asap_types::pre_asap::agg_intent::{agg_is_mergeable, AggIntent}; use asap_types::pre_asap::cse::{share_common_subtrees, structural_hash, HashCache}; use asap_types::pre_asap::expr_ir::ColumnRef; @@ -374,15 +372,10 @@ use crate::accuracy::{ KLL_RANK_ERROR_EXPONENT_99, }; use crate::accuracy_reconciliation::AccuracyReconciliationStrategy; -use crate::cost_model::{ - raw_recompute_cost_rate, Cost, CostModel, CseCandidate, DefaultCostModel, - ExactCompositionCostInputs, ExactCompositionCostRequest, ShareDecision, -}; -use crate::exact_composition::{CompositionPlacement, ExactComposition, ExactCompositionStrategy}; +use crate::cost_model::{Cost, CostModel, CseCandidate, DefaultCostModel, ShareDecision}; use crate::grouping::HydraGroupingStrategy; use crate::recurrence::{ - evaluation_rate_of, CostRate, Horizon, RecurrenceError, RecurrenceProfile, RootRecurrence, - UpdateRate, + evaluation_rate_of, Horizon, RecurrenceError, RecurrenceProfile, RootRecurrence, UpdateRate, }; use crate::rollup::RollupStrategy; use crate::topk_reuse::TopKLimitReuseStrategy; @@ -405,15 +398,6 @@ pub enum ImplementError { /// records it as a [`RejectedCandidate`] instead of a candidate. #[error("accuracy-illegal candidate: {0}")] Accuracy(#[from] AccuracyError), - /// A constructed plan violates the update/readout phase contract - /// (issue #171) — e.g. a summary readout placed beneath a maintained - /// `SummaryAgg`. Detected at construction, never at runtime. - #[error("execution-data_state violation in post-ASAP plan: {0}")] - ExecutionDataState(#[from] ExecutionDataStateError), - /// An `ExactOperator`'s output schema could not be derived over its - /// child — the child carries summary state the operator can't read. - #[error("exact operator schema derivation failed: {0}")] - ExactOperatorSchema(#[from] ExactOperatorSchemaError), } /// A pre-ASAP sub-DAG a [`ReplacementStrategy`] knows how to replace. @@ -473,15 +457,6 @@ pub enum Replacement { /// different from the target's own `root` (e.g. sharing vs. not sharing /// a subtree) but semantically equivalent to it. Rewrite(Rc), - /// An exact operator composed over another target's *own* selected - /// decision across an explicit update/readout boundary (issue #171): - /// `ExactPostProcess` over a child's summary readout, or - /// `ExactTransform` feeding a maintained summary above. Carries only a - /// reference to the child target — [`PlanSpace::global_selection`] - /// commits the compatible parent/child pair and - /// [`GlobalSelection::materialize`] links it into one validated - /// `SummaryNode`. See [`crate::exact_composition`]. - ExactComposition(ExactComposition), } /// One candidate replacement for a [`TargetSubDAG`], plus a human-readable @@ -523,12 +498,6 @@ pub enum ReplacementProvenance { /// regardless, so pricing it like a full independent rebuild would be /// the wrong shape of cost, not just the wrong number. AccuracyReconciliation, - /// [`Replacement::ExactComposition`] with - /// [`CompositionPlacement::PostProcess`] (issue #171). - ExactPostProcess, - /// [`Replacement::ExactComposition`] with - /// [`CompositionPlacement::Transform`] (issue #171). - ExactTransform, } /// A candidate a strategy considered for a target but refused to propose on @@ -554,7 +523,6 @@ pub struct RejectedCandidate { pub struct Proposals { pub candidates: Vec, pub rejected: Vec, - domain_error: Option, } /// A replacement strategy: given a [`TargetSubDAG`], does this strategy have @@ -600,7 +568,6 @@ pub trait ReplacementStrategy { Proposals { candidates: self.replacements(target), rejected: Vec::new(), - domain_error: None, } } } @@ -1404,22 +1371,6 @@ impl<'a> SketchAlgorithmStrategy<'a> { ); } } - if proposals.candidates.is_empty() { - if let Some(error) = &proposals.domain_error { - if let Ok(node) = keep_pre_asap(root) { - proposals.candidates.push(ReplacementSubDAG { - strategy: "SketchAlgorithmStrategy", - replacement: Replacement::Summary(node), - provenance: ReplacementProvenance::SummaryImplementation, - rationale: format!( - "{} stays pre-ASAP because summary construction crosses an illegal \ - execution-data_state boundary ({error})", - describe_intent(intent) - ), - }); - } - } - } proposals } } @@ -1441,10 +1392,7 @@ impl Proposals { description: rationale, error, }), - Err(ImplementError::ExecutionDataState(error)) => { - self.domain_error.get_or_insert(error); - } - Err(ImplementError::Schema(_) | ImplementError::ExactOperatorSchema(_)) => {} + Err(ImplementError::Schema(_)) => {} } } } @@ -1592,10 +1540,10 @@ pub(crate) fn realize_child_with( .. }) => Ok(node), Some(ReplacementSubDAG { - replacement: Replacement::Rewrite(_) | Replacement::ExactComposition(_), + replacement: Replacement::Rewrite(_), .. }) => { - unreachable!("SketchAlgorithmStrategy never returns a Rewrite/composition candidate") + unreachable!("SketchAlgorithmStrategy never returns a Rewrite candidate") } // No candidate at all: `root` isn't `bindable_intent` shape (or its // intent has no realization `implementations_for_with` can't @@ -1816,10 +1764,6 @@ fn construct_summary_agg( // finalized value does. An exact accumulator's state is its value. guarantee: if estimate { None } else { guarantee.clone() }, }); - // Phase contract (issue #171): a maintained summary consumes update-path - // values or exact accumulator state — never a query-time readout. A - // typed error here, at construction; the caller decides the fallback. - validate_execution_data_states_at(&agg, ExecutionDataState::MAINTENANCE_SUMMARY)?; match query { // The readout: downstream of the estimate the schema is the plain // pre-ASAP row shape again (the summary-state type does not @@ -2137,13 +2081,10 @@ impl MemoGroup { (Replacement::Summary(existing_node), Replacement::Summary(node)) => { is_duplicate_summary(existing_node, node) } - ( - Replacement::ExactComposition(existing), - Replacement::ExactComposition(candidate), - ) => existing.same_as(candidate), - // Different `Replacement` variants are never the same - // candidate. - _ => false, + // A `Rewrite` and a `Summary` are never the same candidate — + // they're different `Replacement` variants entirely. + (Replacement::Rewrite(_), Replacement::Summary(_)) + | (Replacement::Summary(_), Replacement::Rewrite(_)) => false, } }); if is_duplicate { @@ -2240,9 +2181,7 @@ pub struct PlanSpace { order: Vec<*const QueryExpr>, } -/// Lifecycle-aware whole-subplan costs keyed by target and candidate pointer. -/// Built by `lifecycle` before final selection; kept internal so pointer keys -/// never become part of the public planner API. +/// Lifecycle-aware whole-subplan costs keyed by target and candidate identity. #[derive(Default)] pub(crate) struct CandidateCostOverrides { costs: HashMap<(*const QueryExpr, *const ReplacementSubDAG), Cost>, @@ -2255,15 +2194,13 @@ impl CandidateCostOverrides { candidate: &ReplacementSubDAG, cost: Cost, ) { - self.costs.insert( - (Rc::as_ptr(target), candidate as *const ReplacementSubDAG), - cost, - ); + self.costs + .insert((Rc::as_ptr(target), candidate as *const _), cost); } fn get(&self, target: &Rc, candidate: &ReplacementSubDAG) -> Option { self.costs - .get(&(Rc::as_ptr(target), candidate as *const ReplacementSubDAG)) + .get(&(Rc::as_ptr(target), candidate as *const _)) .copied() } } @@ -2687,10 +2624,8 @@ impl PlanSpace { self.recurrence_profiles(&recurrences, update_rate) } - /// Map every discovered target to the normalized workload entries whose - /// roots can reach it. Each entry appears at most once per target even - /// when a root has several paths to that target; path multiplicity is a - /// separate recurrence/effective-use concern. + /// Associate every discovered target with the normalized workload entries + /// whose roots can reach it. pub(crate) fn workload_entries_by_target( &self, workload: &QueryWorkload, @@ -2867,7 +2802,7 @@ fn rank_group<'a>(group: &'a MemoGroup, cost_model: &dyn CostModel) -> Vec<&'a R .iter() .map(|c| match &c.replacement { Replacement::Summary(node) => sketch_kind_of(node), - Replacement::Rewrite(_) | Replacement::ExactComposition(_) => None, + Replacement::Rewrite(_) => None, }) .collect(); if let Some(kinds) = kinds { @@ -2875,7 +2810,7 @@ fn rank_group<'a>(group: &'a MemoGroup, cost_model: &dyn CostModel) -> Vec<&'a R ranked.sort_by_key(|c| { let kind = match &c.replacement { Replacement::Summary(node) => sketch_kind_of(node), - Replacement::Rewrite(_) | Replacement::ExactComposition(_) => None, + Replacement::Rewrite(_) => None, }; kind.and_then(|k| order.iter().position(|o| *o == k)) .unwrap_or(usize::MAX) @@ -2898,59 +2833,6 @@ fn rank_group<'a>(group: &'a MemoGroup, cost_model: &dyn CostModel) -> Vec<&'a R ranked } -/// Apply lifecycle-aware costs to summary siblings after the ordinary -/// strategy-specific ordering. Known lifecycle totals sort before unknown -/// totals; non-summary alternatives keep their existing relative order and -/// continue through their dedicated CSE/composition selection paths. -fn rank_group_with_candidate_costs<'a>( - group: &'a MemoGroup, - cost_model: &dyn CostModel, - overrides: Option<&CandidateCostOverrides>, -) -> Vec<&'a ReplacementSubDAG> { - let mut ranked = rank_group(group, cost_model); - let Some(overrides) = overrides else { - return ranked; - }; - let positions: Vec = ranked - .iter() - .enumerate() - .filter_map(|(index, candidate)| { - matches!(candidate.replacement, Replacement::Summary(_)).then_some(index) - }) - .collect(); - let mut summaries: Vec<_> = positions.iter().map(|&index| ranked[index]).collect(); - summaries.sort_by(|a, b| { - match ( - overrides.get(&group.target, a), - overrides.get(&group.target, b), - ) { - (Some(a), Some(b)) => a.0.total_cmp(&b.0), - (Some(_), None) => std::cmp::Ordering::Less, - (None, Some(_)) => std::cmp::Ordering::Greater, - (None, None) => std::cmp::Ordering::Equal, - } - }); - for (index, candidate) in positions.into_iter().zip(summaries) { - ranked[index] = candidate; - } - ranked -} - -fn estimated_candidate_cost( - group: &MemoGroup, - candidate: &ReplacementSubDAG, - target: &TargetSubDAG<'_>, - cost_model: &dyn CostModel, - overrides: Option<&CandidateCostOverrides>, -) -> f64 { - overrides - .and_then(|costs| costs.get(&group.target, candidate)) - .map_or_else( - || cost_model.estimate_cost(candidate, target), - |cost| cost.0, - ) -} - /// For a group whose candidates are all [`Replacement::Rewrite`] (the /// [`SharedSubtreeStrategy`] shape): does [`CostModel::cse_share_decision`] /// prefer the candidate that shares `group.target`'s own `Rc` (`true`), or @@ -3056,32 +2938,6 @@ pub struct SelectedGroup<'a> { /// registered strategy proposed anything for (mirrors /// [`MemoGroup::candidates`] being possibly empty). pub chosen: Option<&'a ReplacementSubDAG>, - /// When `chosen` is a [`Replacement::ExactComposition`]: the child - /// decision it was committed together with, and the cost comparison - /// that justified it — the explicit target-to-decision provenance - /// chain (issue #171). - pub composition: Option>, -} - -/// Why [`PlanSpace::global_selection`] committed an exact composition at a -/// site: which child candidate it composes with, and the -/// cost-units-per-second comparison against the raw fallback that it won. -#[derive(Debug)] -pub struct CompositionDecision<'a> { - /// The child target the composed operator consumes. - pub child_target: &'a Rc, - /// For a post-process: the child's own candidate committed alongside - /// (the summary readout the operator folds). `None` for an update-path - /// transform, whose input is raw update data — its cost is charged to - /// the maintained summary *above* it instead. - pub child_candidate: Option<&'a ReplacementSubDAG>, - /// The composed plan's recurring rate — `postprocess_plan_cost_rate` - /// or `pretransform_plan_cost_rate`. - pub cost_rate: CostRate, - /// `raw_recompute_cost_rate` — the `KeepPreAsap` baseline it beat. - pub baseline_rate: CostRate, - /// The statistics (and their provenance) both rates were computed from. - pub inputs: ExactCompositionCostInputs, } /// [`PlanSpace::global_selection`]'s result: one [`SelectedGroup`] per @@ -3091,10 +2947,6 @@ pub struct CompositionDecision<'a> { pub struct GlobalSelection<'a> { order: Vec<*const QueryExpr>, groups: HashMap<*const QueryExpr, SelectedGroup<'a>>, - /// [`Self::materialize`]'s memo — one bound node per target for the - /// life of this selection, so two parents composing over one shared - /// child get the *same* `Rc`. - materialized: RefCell>>, } impl<'a> GlobalSelection<'a> { @@ -3110,271 +2962,22 @@ impl<'a> GlobalSelection<'a> { self.groups.get(&Rc::as_ptr(target)) } - /// Link this selection's per-site decisions into one data_state-validated - /// post-ASAP DAG rooted at `target` — the one place a committed - /// composition's child *reference* becomes an actual `Rc` - /// edge (issue #171). `None` if `target` is not a discovered site. - /// - /// Per site: a [`Replacement::ExactComposition`] composes over its - /// child target's own materialization; a [`Replacement::Summary`] is - /// re-linked so its `SummaryAgg` child is the child target's own - /// materialization whenever that is phase-legal beneath maintenance - /// (so a child that chose an `ExactTransform` actually ends up under - /// the summary); a [`Replacement::Rewrite`] or an unmatched site stays - /// the conservative `KeepPreAsap`. Memoized by target identity, so a - /// shared inner summary is one `Rc` no matter how many roots reach it. + /// Materialize the selected replacement at `target`. Exact operators + /// that remain in pre-ASAP IR are preserved by `KeepPreAsap`; logical + /// summary candidates are already fully bound post-ASAP nodes. pub fn materialize( &self, target: &Rc, ) -> Result>, ImplementError> { - if !self.groups.contains_key(&Rc::as_ptr(target)) { + let Some(selected) = self.for_target(target) else { return Ok(None); - } - self.materialize_inner(target).map(Some) - } - - fn materialize_inner(&self, target: &Rc) -> Result, ImplementError> { - let ptr = Rc::as_ptr(target); - if let Some(node) = self.materialized.borrow().get(&ptr) { - return Ok(Rc::clone(node)); - } - let node = match self - .groups - .get(&ptr) - .and_then(|sel| sel.chosen) - .map(|c| &c.replacement) - { - None => keep_pre_asap(target)?, - Some(Replacement::Rewrite(rewritten)) => keep_pre_asap(rewritten)?, - Some(Replacement::Summary(node)) => self.relink_summary(node, target)?, - Some(Replacement::ExactComposition(composition)) => { - let child = self.materialize_inner(&composition.child_target)?; - let child = if composition.accepts_child(&child) { - child - } else { - keep_pre_asap(&composition.child_target)? - }; - composition.compose(child)? - } - }; - self.materialized.borrow_mut().insert(ptr, Rc::clone(&node)); - Ok(node) - } - - /// Re-link a bound `Summary` candidate's `SummaryAgg` child to the - /// child target's own materialization when that is legal beneath - /// maintenance; otherwise keep the candidate exactly as constructed. - fn relink_summary( - &self, - node: &Rc, - target: &Rc, - ) -> Result, ImplementError> { - let QueryExpr::Aggregate { - child: pre_child, .. - } = target.as_ref() - else { - return Ok(Rc::clone(node)); - }; - if !self.groups.contains_key(&Rc::as_ptr(pre_child)) { - return Ok(Rc::clone(node)); - } - let new_child = self.materialize_inner(pre_child)?; - Ok(relink_agg_child(node, &new_child)) - } -} - -/// Rebuild `node` (a `SummaryAgg`, possibly under a `SummaryEstimate`) with -/// `new_child` as the `SummaryAgg`'s child, if the result still validates -/// as maintained state; otherwise return `node` unchanged. -fn relink_agg_child(node: &Rc, new_child: &Rc) -> Rc { - match &node.expr { - SummaryExpr::SummaryEstimate { - summary_input, - query, - } => { - let inner = relink_agg_child(summary_input, new_child); - if Rc::ptr_eq(&inner, summary_input) { - return Rc::clone(node); - } - Rc::new(SummaryNode { - expr: SummaryExpr::SummaryEstimate { - summary_input: inner, - query: query.clone(), - }, - schema: node.schema.clone(), - guarantee: node.guarantee.clone(), - }) - } - SummaryExpr::SummaryAgg { - child, - family, - col, - reduction, - grouping, - } => { - if Rc::ptr_eq(child, new_child) { - return Rc::clone(node); - } - let rebuilt = Rc::new(SummaryNode { - expr: SummaryExpr::SummaryAgg { - child: Rc::clone(new_child), - family: family.clone(), - col: col.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - }, - schema: node.schema.clone(), - guarantee: node.guarantee.clone(), - }); - match validate_execution_data_states_at( - &rebuilt, - ExecutionDataState::MAINTENANCE_SUMMARY, - ) { - Ok(_) => rebuilt, - Err(_) => Rc::clone(node), - } - } - _ => Rc::clone(node), - } -} - -/// The maintained `SummaryAgg` a bound `Summary` candidate builds (under -/// its `SummaryEstimate` readout, if any) — the summary an `ExactTransform` -/// beneath it feeds, for `pretransform_plan_cost_rate`. -fn maintained_summary(node: &Rc) -> Option<&Rc> { - match &node.expr { - SummaryExpr::SummaryEstimate { summary_input, .. } => maintained_summary(summary_input), - SummaryExpr::SummaryAgg { .. } => Some(node), - _ => None, - } -} - -fn is_composition_candidate(candidate: &ReplacementSubDAG) -> bool { - matches!(candidate.replacement, Replacement::ExactComposition(_)) -} - -/// Everything [`PlanSpace::global_selection`] threads between sites for -/// exact compositions (issue #171): child candidates already committed by -/// an earlier parent, and the maintained summary above each site. -#[derive(Default)] -struct CompositionContext { - /// child target ptr → the child's candidate an ancestor's composition - /// already committed to (a later parent must compose with the *same* - /// one, and the child's own selection is forced to it). - committed_child: HashMap<*const QueryExpr, *const ReplacementSubDAG>, - /// site ptr → the maintained `SummaryAgg` directly above it, when its - /// parent chose a bound `Summary` — what an `ExactTransform` here feeds. - maintaining_parent: HashMap<*const QueryExpr, Rc>, -} - -/// One eligible composed alternative at a site, before the cheapest wins. -struct CompositionOption<'a> { - candidate: &'a ReplacementSubDAG, - decision: CompositionDecision<'a>, -} - -/// Every [`Replacement::ExactComposition`] candidate of `group` whose -/// composed-plan rate is *known* and beats the raw-recompute baseline — -/// costed against each compatible child candidate already in `PlanSpace` -/// (or the one an earlier parent committed). Unknown statistics yield no -/// option at all: the conservative `KeepPreAsap` path stays. -fn composition_options<'a>( - group: &'a MemoGroup, - groups: &'a HashMap<*const QueryExpr, MemoGroup>, - effective: usize, - cost_model: &dyn CostModel, - context: &CompositionContext, -) -> Vec> { - let mut options = Vec::new(); - for candidate in &group.candidates { - let Replacement::ExactComposition(composition) = &candidate.replacement else { - continue; }; - let child_ptr = Rc::as_ptr(&composition.child_target); - let Some(child_group) = groups.get(&child_ptr) else { - continue; - }; - let already_committed = context.committed_child.get(&child_ptr).copied(); - let cost = |summary: &SummaryNode, shared: bool| { - let request = ExactCompositionCostRequest { - target: &group.target, - composition, - summary, - effective_consumer_count: effective, - }; - let mut inputs = cost_model.exact_composition_cost_inputs(&request); - if shared { - // Shared state is counted once: an earlier parent already - // pays this child's maintenance, so the marginal cost here - // is zero — a *known* zero, unlike an unknown input. - if let Some(maintenance) = inputs.summary_maintenance_cost_per_update.as_mut() { - *maintenance = 0.0; - } - } - let rate = inputs.composed_plan_cost_rate(composition.placement)?; - let baseline = raw_recompute_cost_rate(&inputs)?; - (rate < baseline).then_some((rate, baseline, inputs)) - }; - match composition.placement { - CompositionPlacement::PostProcess => { - let child_candidates: Vec<&'a ReplacementSubDAG> = match already_committed { - // SAFETY-free: the pointer was taken from `groups`'s own - // candidate storage, which outlives this borrow. - Some(ptr) => child_group - .candidates - .iter() - .filter(|c| std::ptr::eq(*c, ptr)) - .collect(), - None => child_group.candidates.iter().collect(), - }; - for child_candidate in child_candidates { - let Replacement::Summary(summary) = &child_candidate.replacement else { - continue; - }; - if !composition.accepts_child(summary) { - continue; - } - let Some((rate, baseline, inputs)) = cost(summary, already_committed.is_some()) - else { - continue; - }; - options.push(CompositionOption { - candidate, - decision: CompositionDecision { - child_target: &composition.child_target, - child_candidate: Some(child_candidate), - cost_rate: rate, - baseline_rate: baseline, - inputs, - }, - }); - } - } - CompositionPlacement::Transform => { - // An update-path transform only pays off beneath a - // maintained summary; with nothing above it, its output is - // never read and the raw fallback is the same computation. - let Some(parent) = context.maintaining_parent.get(&Rc::as_ptr(&group.target)) - else { - continue; - }; - let Some((rate, baseline, inputs)) = cost(parent, false) else { - continue; - }; - options.push(CompositionOption { - candidate, - decision: CompositionDecision { - child_target: &composition.child_target, - child_candidate: None, - cost_rate: rate, - baseline_rate: baseline, - inputs, - }, - }); - } + match selected.chosen.map(|candidate| &candidate.replacement) { + Some(Replacement::Summary(node)) => Ok(Some(Rc::clone(node))), + Some(Replacement::Rewrite(rewritten)) => keep_pre_asap(rewritten).map(Some), + None => keep_pre_asap(target).map(Some), } } - options } impl PlanSpace { @@ -3403,17 +3006,14 @@ impl PlanSpace { self.global_selection_impl(cost_model, Some(profiles), horizon, None) } - /// Final selection with lifecycle-aware whole-subplan cost overrides. - /// `lifecycle` builds the overrides from normalized workload evidence and - /// calls this only after candidate legality and accuracy validation. pub(crate) fn global_selection_with_candidate_costs( &self, cost_model: &dyn CostModel, profiles: &RecurrenceProfileMap, horizon: Option, - candidate_costs: &CandidateCostOverrides, + costs: &CandidateCostOverrides, ) -> Result, RecurrenceError> { - self.global_selection_impl(cost_model, Some(profiles), horizon, Some(candidate_costs)) + self.global_selection_impl(cost_model, Some(profiles), horizon, Some(costs)) } fn global_selection_impl( @@ -3429,7 +3029,6 @@ impl PlanSpace { let mut effective_uses = graph.external_root_uses.clone(); let mut chosen_share: HashMap<*const QueryExpr, ShareDecision> = HashMap::new(); let mut groups: HashMap<*const QueryExpr, SelectedGroup<'_>> = HashMap::new(); - let mut context = CompositionContext::default(); for ptr in &topo { let group = &self.groups[ptr]; @@ -3437,49 +3036,20 @@ impl PlanSpace { let effective = effective_uses.get(ptr).copied().unwrap_or(0); effective_uses.insert(*ptr, effective); - // ── Exact compositions (issue #171) ───────────────────────── - // A child an earlier parent's composition committed to is - // forced to exactly that candidate — the parent/child pair is - // one decision. Otherwise, a composition here wins only when - // its cost-units-per-second rate is *known* and beats the raw - // recompute baseline; missing statistics keep the conservative - // path below. - let mut composition_decision = None; - let forced = context - .committed_child - .get(ptr) - .and_then(|&cptr| group.candidates.iter().find(|c| std::ptr::eq(*c, cptr))); - let composed = if forced.is_some() { - None - } else { - composition_options(group, &self.groups, effective, cost_model, &context) - .into_iter() - .min_by(|a, b| a.decision.cost_rate.0.total_cmp(&b.decision.cost_rate.0)) - }; - if let Some(option) = &composed { - if let Some(child_candidate) = option.decision.child_candidate { - context.committed_child.insert( - Rc::as_ptr(option.decision.child_target), - child_candidate as *const ReplacementSubDAG, - ); - } - if let Replacement::ExactComposition(composition) = &option.candidate.replacement { - if composition.placement == CompositionPlacement::Transform { - // A chain of transforms feeds the same summary. - if let Some(parent) = context.maintaining_parent.get(ptr).cloned() { - context - .maintaining_parent - .insert(Rc::as_ptr(&composition.child_target), parent); - } - } - } - } - - let chosen = if let Some(forced) = forced { - Some(forced) - } else if let Some(option) = composed { - composition_decision = Some(option.decision); - Some(option.candidate) + let lifecycle_choice = candidate_costs.and_then(|costs| { + group + .candidates + .iter() + .filter_map(|candidate| { + costs + .get(&group.target, candidate) + .map(|cost| (candidate, cost)) + }) + .min_by(|(_, a), (_, b)| a.0.total_cmp(&b.0)) + .map(|(candidate, _)| candidate) + }); + let chosen = if lifecycle_choice.is_some() { + lifecycle_choice } else if effective >= 2 && cse_candidate_pair(group).is_some() { let decision = if let Some(profiles) = profiles { decide_group_with_recurrence( @@ -3500,44 +3070,18 @@ impl PlanSpace { let logical = group .candidates .iter() - .filter(|candidate| { - !is_cse_candidate(candidate) && !is_composition_candidate(candidate) - }) + .filter(|candidate| !is_cse_candidate(candidate)) .min_by(|a, b| { - estimated_candidate_cost( - group, - a, - &effective_target, - cost_model, - candidate_costs, - ) - .total_cmp( - &estimated_candidate_cost( - group, - b, - &effective_target, - cost_model, - candidate_costs, - ), - ) + cost_model + .estimate_cost(a, &effective_target) + .total_cmp(&cost_model.estimate_cost(b, &effective_target)) }); match (cse, logical) { (Some(cse), Some(logical)) - if estimated_candidate_cost( - group, - logical, - &effective_target, - cost_model, - candidate_costs, - ) - .total_cmp(&estimated_candidate_cost( - group, - cse, - &effective_target, - cost_model, - candidate_costs, - )) - .is_lt() => + if cost_model + .estimate_cost(logical, &effective_target) + .total_cmp(&cost_model.estimate_cost(cse, &effective_target)) + .is_lt() => { Some(logical) } @@ -3557,32 +3101,15 @@ impl PlanSpace { // valid answer, just not a cross-group-aware one; this // group also contributes no Share collapse to its own // children (see `multiplier`'s `_ => effective` arm). - None => rank_group_with_candidate_costs(group, cost_model, candidate_costs) - .into_iter() - .find(|candidate| !is_composition_candidate(candidate)), + None => rank_group(group, cost_model).into_iter().next(), } } else { - rank_group_with_candidate_costs(group, cost_model, candidate_costs) + rank_group(group, cost_model) .into_iter() - .find(|candidate| { - !is_cse_candidate(candidate) && !is_composition_candidate(candidate) - }) + .find(|candidate| !is_cse_candidate(candidate)) .or_else(|| cse_candidate_pair(group).map(|(share, _)| share)) }; - // Record the maintained summary this site's bound candidate - // builds, for a child that may compose an `ExactTransform` - // beneath it. - if let (Some(Replacement::Summary(node)), QueryExpr::Aggregate { child, .. }) = - (chosen.map(|c| &c.replacement), group.target.as_ref()) - { - if let Some(summary) = maintained_summary(node) { - context - .maintaining_parent - .insert(Rc::as_ptr(child), Rc::clone(summary)); - } - } - let outgoing_multiplier = multiplier(*ptr, &effective_uses, &chosen_share); match chosen { Some(ReplacementSubDAG { @@ -3600,9 +3127,7 @@ impl PlanSpace { _ => { let selected_rewrite = match chosen.map(|candidate| &candidate.replacement) { Some(Replacement::Rewrite(rewrite)) => rewrite, - Some(Replacement::Summary(_) | Replacement::ExactComposition(_)) | None => { - &group.target - } + Some(Replacement::Summary(_)) | None => &group.target, }; for (child, edge_count) in direct_child_counts(selected_rewrite) { *effective_uses.entry(child).or_insert(0) += @@ -3618,7 +3143,6 @@ impl PlanSpace { consumer_count: group.consumer_count, effective_consumer_count: effective, chosen, - composition: composition_decision, }, ); } @@ -3626,7 +3150,6 @@ impl PlanSpace { Ok(GlobalSelection { order: self.order.clone(), groups, - materialized: RefCell::new(HashMap::new()), }) } } @@ -4002,7 +3525,6 @@ pub fn default_strategies() -> Vec> { Box::new(HydraGroupingStrategy::default_cost_model()), Box::new(SharedSubtreeStrategy), Box::new(crate::rewrite::AvgToSumOverCountStrategy), - Box::new(ExactCompositionStrategy::default_cost_model()), ] } @@ -4017,7 +3539,6 @@ pub fn default_strategies_with<'a>( Box::new(HydraGroupingStrategy::new(cost_model)), Box::new(SharedSubtreeStrategy), Box::new(crate::rewrite::AvgToSumOverCountStrategy), - Box::new(ExactCompositionStrategy::new(cost_model)), ] } @@ -4111,7 +3632,6 @@ pub fn search_workload_with_targets<'s, Id>( .as_ref() .is_some_and(|g| accuracy_model.satisfies(g, &target)), Replacement::Rewrite(_) => true, - Replacement::ExactComposition(_) => false, }); group.candidates = legal; group.rejected.extend(illegal.into_iter().map(|candidate| { @@ -4132,11 +3652,6 @@ pub fn search_workload_with_targets<'s, Id>( None, )), Replacement::Rewrite(_) => unreachable!("rewrites are never rejected here"), - Replacement::ExactComposition(_) => ( - asap_types::post_asap::ErrorMetric::AbsoluteValue, - None, - None, - ), }; RejectedCandidate { strategy: candidate.strategy, @@ -5090,9 +4605,7 @@ mod tests { .iter() .map(|r| match &r.replacement { Replacement::Summary(node) => summary_family_algorithm(node), - Replacement::Rewrite(_) | Replacement::ExactComposition(_) => { - panic!("expected a Summary replacement") - } + Replacement::Rewrite(_) => panic!("expected a Summary replacement"), }) .collect(); assert!(kinds.contains(&SketchAlgorithm::Kll), "{kinds:?}"); @@ -5112,9 +4625,7 @@ mod tests { .iter() .map(|r| match &r.replacement { Replacement::Summary(node) => summary_family_algorithm(node), - Replacement::Rewrite(_) | Replacement::ExactComposition(_) => { - panic!("expected a Summary replacement") - } + Replacement::Rewrite(_) => panic!("expected a Summary replacement"), }) .collect(); assert_eq!( @@ -5142,9 +4653,7 @@ mod tests { .iter() .map(|r| match &r.replacement { Replacement::Summary(node) => summary_family_algorithm(node), - Replacement::Rewrite(_) | Replacement::ExactComposition(_) => { - panic!("expected a Summary replacement") - } + Replacement::Rewrite(_) => panic!("expected a Summary replacement"), }) .collect(); assert_eq!(kinds, vec![SketchAlgorithm::Theta, SketchAlgorithm::Kmv]); @@ -5222,9 +4731,7 @@ mod tests { .iter() .map(|r| match &r.replacement { Replacement::Summary(node) => summary_family_algorithm(node), - Replacement::Rewrite(_) | Replacement::ExactComposition(_) => { - panic!("expected a Summary replacement") - } + Replacement::Rewrite(_) => panic!("expected a Summary replacement"), }) .collect(); assert!(kinds.contains(&SketchAlgorithm::Kll)); @@ -5232,13 +4739,13 @@ mod tests { assert_eq!(kinds.len(), 2); } - /// Constructing the outer target's candidates never leaks the outer - /// choice into the nested aggregate — and, since issue #171's phase - /// contract, a maintained outer sketch can no longer sit above the - /// inner sketch's *readout* at all: the outer target degrades to the - /// conservative `KeepPreAsap` fallback (reported once, not once per - /// dropped family), while the inner quantile keeps its own, - /// independently cost-ranked candidates in its own `MemoGroup`. + /// Enumerating candidates for the *target* node must only steer that + /// node's own decision — a nested aggregate underneath it still gets its + /// own independent (`cost_model`-ranked) enumeration, not whatever the + /// caller happened to pick for the outer target. This is the behavior + /// [`construct_summary`]'s recursion (via [`realize_child`]) + /// gets for free: only the top node's `Implementation` is ever forced + /// from outside; the child is always re-enumerated fresh. #[test] fn enumerating_the_targets_candidates_does_not_leak_into_a_nested_aggregate() { // outer: quantile(0.99, ...) over inner: quantile(0.5, m) — both @@ -5258,41 +4765,35 @@ mod tests { ) .replacements(&target); - assert_eq!(replacements.len(), 1, "{replacements:?}"); - let Replacement::Summary(node) = &replacements[0].replacement else { - unreachable!("SketchAlgorithmStrategy only returns Summary candidates"); + let ddsketch = replacements + .iter() + .find(|r| { + matches!(&r.replacement, Replacement::Summary(node) + if summary_family_algorithm(node) == SketchAlgorithm::DDSketch) + }) + .expect("the outer target's DDSketch candidate must be present"); + let Replacement::Summary(node) = &ddsketch.replacement else { + unreachable!("filtered on Replacement::Summary above"); }; - assert!( - matches!(node.expr, SummaryExpr::KeepPreAsap(ref e) if Rc::ptr_eq(e, &outer)), - "a sketch over a sketch readout is data_state-illegal; expected the conservative \ - fallback, got {:?}", - node.expr - ); - assert!(replacements[0].rationale.contains("readout")); - - // The inner target is still independently enumerated and ranked — - // a custom cost model that prefers DDSketch for it is honored, and - // nothing about the outer target's choice reaches it. - let space = search_workload_with( - vec![("q", Rc::clone(&outer))], - &default_strategies_with(&PreferDDSketchViaCostModel), + assert_eq!( + summary_family_algorithm(node), + SketchAlgorithm::DDSketch, + "the outer (target) node must be the DDSketch candidate" ); - let QueryExpr::Aggregate { child, .. } = space.roots[0].1.as_ref() else { - unreachable!() + + let asap_types::post_asap::SummaryExpr::SummaryEstimate { summary_input, .. } = &node.expr + else { + panic!("expected SummaryEstimate root, got {:?}", node.expr); + }; + let asap_types::post_asap::SummaryExpr::SummaryAgg { child, .. } = &summary_input.expr + else { + panic!("expected SummaryAgg, got {:?}", summary_input.expr); }; - let inner_group = space.group_for(child).expect("inner quantile is a target"); - let inner_kinds: Vec = inner_group - .candidates - .iter() - .filter_map(|c| match &c.replacement { - Replacement::Summary(node) => sketch_kind_of(node), - _ => None, - }) - .collect(); assert_eq!( - inner_kinds, - vec![SketchAlgorithm::DDSketch, SketchAlgorithm::Kll], - "the nested inner aggregate keeps its own cost-model-ranked candidates" + summary_family_algorithm(child), + SketchAlgorithm::Kll, + "the nested inner aggregate must still get the cost-model-ranked \ + default (Kll), not inherit the outer target's DDSketch candidate" ); } @@ -5819,7 +5320,7 @@ mod tests { assert_eq!(rewrites.len(), 2); let first_shares_target = match &rewrites[0].replacement { Replacement::Rewrite(rc) => Rc::ptr_eq(rc, &group.target), - Replacement::Summary(_) | Replacement::ExactComposition(_) => false, + Replacement::Summary(_) => false, }; assert!( first_shares_target, @@ -5855,7 +5356,7 @@ mod tests { assert_eq!(agg_group.candidates.len(), 2); let first_kind = match &agg_group.candidates[0].replacement { Replacement::Summary(node) => sketch_kind_of(node), - Replacement::Rewrite(_) | Replacement::ExactComposition(_) => None, + Replacement::Rewrite(_) => None, }; assert_eq!(first_kind, Some(SketchAlgorithm::DDSketch)); } @@ -6051,7 +5552,7 @@ mod tests { .unwrap(); let kind = match &agg_group.chosen.unwrap().replacement { Replacement::Summary(node) => sketch_kind_of(node), - Replacement::Rewrite(_) | Replacement::ExactComposition(_) => None, + Replacement::Rewrite(_) => None, }; assert_eq!(kind, Some(SketchAlgorithm::DDSketch)); } @@ -7382,7 +6883,13 @@ mod tests { let space = search_workload_with(vec![("q", Rc::clone(&outer))], &strategies); let root = &space.roots[0].1; let group = space.group_for(root).unwrap(); - assert_eq!(group.candidates.len(), 1); + assert!(!group.rejected.is_empty()); + assert!(group.candidates.iter().all(|c| match &c.replacement { + Replacement::Summary(node) => node.guarantee.as_ref().is_some_and(|g| { + DefaultAccuracyModel.satisfies(g, &AccuracyTarget::Epsilon(0.1)) + }), + Replacement::Rewrite(_) => false, + })); let ranked = space.cost_sorted(&DefaultCostModel); let root_ranked = ranked.iter().find(|g| Rc::ptr_eq(g.target, root)).unwrap(); assert_eq!(root_ranked.candidates.len(), group.candidates.len()); @@ -7449,7 +6956,6 @@ mod tests { .as_ref() .is_some_and(ResultGuarantee::is_exact), Replacement::Rewrite(_) => true, - Replacement::ExactComposition(_) => false, })); } diff --git a/crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs b/crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs index e7b314d6..d69fd27e 100644 --- a/crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs +++ b/crates/asap-aware-mapping/src/summary_maintenance_lifecycle.rs @@ -1,6 +1,5 @@ //! Workload-aware physical summary-maintenance lifecycle planning. //! -//! Phase validation from PR #300 answers whether a post-ASAP DAG can execute. //! This module answers how each unique `SummaryAgg` state is deployed for the //! supplied query and data workloads. Unknown evidence stays unknown and //! therefore cannot make a long-lived summary maintenance lifecycle win. @@ -9,11 +8,8 @@ use std::collections::{HashMap, HashSet}; use std::rc::Rc; use asap_types::post_asap::{ - produced_availability, validate_execution_phases, ExecutionAvailability, SummaryExpr, - SummaryMaintenanceLifecycle, SummaryNode, -}; -use asap_types::post_asap::{ - EvaluationSchedule, OutputRepresentation, SummaryMaintenanceLifecycleGuarantee, + EvaluationSchedule, OutputRepresentation, SummaryExpr, SummaryMaintenanceLifecycle, + SummaryMaintenanceLifecycleGuarantee, SummaryMaintenanceMode, SummaryNode, }; use asap_types::pre_asap::QueryExpr; use asap_types::workload::{ @@ -142,8 +138,6 @@ impl<'a> WorkloadDemand<'a> { pub enum SummaryMaintenanceLifecyclePlanError { #[error(transparent)] InvalidWorkload(#[from] WorkloadError), - #[error(transparent)] - InvalidExecutionPhases(#[from] asap_types::post_asap::PhaseError), #[error("optimization horizon must be finite and strictly positive")] InvalidHorizon, #[error("workload entry index {index} is out of bounds for {entry_count} entries")] @@ -219,7 +213,6 @@ fn plan_summary_maintenance_lifecycles_with_profile( profile: Option, ) -> Result { demand.workload.validate()?; - validate_execution_phases(&root)?; if horizon.is_some_and(|h| !h.0.is_finite() || h.0 <= 0.0) { return Err(SummaryMaintenanceLifecyclePlanError::InvalidHorizon); } @@ -822,10 +815,6 @@ fn collect_summary_aggs( collect_summary_aggs(child, seen, output); } } - SummaryExpr::UpdateTransform { child, .. } - | SummaryExpr::ReadoutPostProcess { child, .. } => { - collect_summary_aggs(child, seen, output) - } SummaryExpr::KeepPreAsap(_) => {} } } @@ -872,7 +861,14 @@ fn summary_state_components(summaries: &[Rc]) -> Vec { let SummaryExpr::SummaryAgg { child, .. } = &summary.expr else { continue; }; - if produced_availability(&child.expr) != Some(ExecutionAvailability::SummaryState) { + if !matches!( + child.expr, + SummaryExpr::SummaryAgg { .. } + | SummaryExpr::SummaryJoin { .. } + | SummaryExpr::SummarySubtract { .. } + | SummaryExpr::SummaryDelete { .. } + | SummaryExpr::SummaryMerge { .. } + ) { continue; } let mut descendants = Vec::new(); @@ -946,6 +942,10 @@ fn select_compatible_lifecycles( deployments[index].summary_maintenance_lifecycle_guarantee = selected.map(|candidate| SummaryMaintenanceLifecycleGuarantee { summary_maintenance_lifecycle: candidate.summary_maintenance_lifecycle.clone(), + summary_maintenance_mode: maintenance_mode( + &candidate.summary_maintenance_lifecycle, + arrival, + ), evaluation_schedule: schedule, output_representation: OutputRepresentation::SummaryState, }); @@ -953,6 +953,23 @@ fn select_compatible_lifecycles( } } +fn maintenance_mode( + lifecycle: &SummaryMaintenanceLifecycle, + arrival: DataArrival, +) -> SummaryMaintenanceMode { + match lifecycle { + SummaryMaintenanceLifecycle::Ephemeral => SummaryMaintenanceMode::DirectBuild, + SummaryMaintenanceLifecycle::ContinuouslyMaintained => SummaryMaintenanceMode::Incremental, + SummaryMaintenanceLifecycle::Prepared { .. } + | SummaryMaintenanceLifecycle::Shared { .. } => match arrival { + DataArrival::ContinuouslyIngesting | DataArrival::Mixed => { + SummaryMaintenanceMode::Incremental + } + DataArrival::AtRest | DataArrival::Unknown => SummaryMaintenanceMode::DirectBuild, + }, + } +} + #[cfg(test)] mod tests { use super::*; @@ -1333,6 +1350,10 @@ mod tests { .as_ref() .unwrap(); assert_eq!(guarantee.evaluation_schedule, EvaluationSchedule::OneShot); + assert_eq!( + guarantee.summary_maintenance_mode, + SummaryMaintenanceMode::DirectBuild + ); assert_eq!( guarantee.output_representation, OutputRepresentation::SummaryState @@ -1408,6 +1429,14 @@ mod tests { retention: DurationMs(10_000) }) ); + assert_eq!( + plan.deployments[0] + .summary_maintenance_lifecycle_guarantee + .as_ref() + .unwrap() + .summary_maintenance_mode, + SummaryMaintenanceMode::DirectBuild + ); assert_eq!( plan.deployments[0].alternatives[3].rejection, Some(SummaryMaintenanceLifecycleRejection::RequiresContinuousData) @@ -1437,6 +1466,14 @@ mod tests { selected_summary_maintenance_lifecycle(&plan.deployments[0]), Some(&SummaryMaintenanceLifecycle::ContinuouslyMaintained) ); + assert_eq!( + plan.deployments[0] + .summary_maintenance_lifecycle_guarantee + .as_ref() + .unwrap() + .summary_maintenance_mode, + SummaryMaintenanceMode::Incremental + ); assert_eq!(plan.evaluation_rate, Some(EvaluationRate(1.0))); assert_eq!(plan.update_rate, Some(UpdateRate(1.0))); } diff --git a/crates/types/src/post_asap/mod.rs b/crates/types/src/post_asap/mod.rs index 83c31be3..3b424bba 100644 --- a/crates/types/src/post_asap/mod.rs +++ b/crates/types/src/post_asap/mod.rs @@ -29,21 +29,17 @@ pub mod expr; pub mod guarantee; -pub mod summary_maintenance_lifecycle; pub mod query_time; pub mod schema; pub mod sketch; pub mod summary_maintenance; +pub mod summary_maintenance_lifecycle; pub use expr::{SummaryExpr, SummaryNode}; pub use guarantee::{ AccuracyError, BoundExpr, CompositionOperator, ErrorMetric, GuaranteeSource, ProbabilityExpr, ResultGuarantee, }; -pub use summary_maintenance_lifecycle::{ - EvaluationSchedule, OutputRepresentation, SummaryMaintenanceLifecycle, - SummaryMaintenanceLifecycleGuarantee, -}; pub use query_time::{ classic_cms_sizing, cms_posterior_error_bound, count_sketch_posterior_error_bound, cu_sketch_posterior_error_bound, traditional_a_priori_bound, @@ -55,3 +51,7 @@ pub use sketch::{ SketchParams, SketchQuery, StatModelKind, StatModelParams, WaveletKind, WaveletParams, }; pub use summary_maintenance::SummaryMaintenanceMode; +pub use summary_maintenance_lifecycle::{ + EvaluationSchedule, OutputRepresentation, SummaryMaintenanceLifecycle, + SummaryMaintenanceLifecycleGuarantee, +}; diff --git a/crates/types/src/post_asap/summary_maintenance_lifecycle.rs b/crates/types/src/post_asap/summary_maintenance_lifecycle.rs index 09521bf5..966b2cd5 100644 --- a/crates/types/src/post_asap/summary_maintenance_lifecycle.rs +++ b/crates/types/src/post_asap/summary_maintenance_lifecycle.rs @@ -5,6 +5,7 @@ //! is deliberately narrower than the end-to-end data lifecycle (collection, //! transmission, storage, and analytics). +use super::SummaryMaintenanceMode; use crate::workload::{DurationMs, TimestampMs}; /// When an operator is evaluated. This is independent of whether it owns @@ -46,6 +47,7 @@ pub enum SummaryMaintenanceLifecycle { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct SummaryMaintenanceLifecycleGuarantee { pub summary_maintenance_lifecycle: SummaryMaintenanceLifecycle, + pub summary_maintenance_mode: SummaryMaintenanceMode, pub evaluation_schedule: EvaluationSchedule, pub output_representation: OutputRepresentation, }