From a5aabd848b70996a9919406db363a10a4a355b61 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 14 Oct 2026 23:14:23 +0000 Subject: [PATCH 01/14] feat(types): summary-capability facts in the family-capability table Each family lists the readouts one of its states answers (Readout: total, item frequency, any quantile rank, distinct count, L2, entropy, top-k), so a planner rule can check that one state serves every consumer's statistic: UnivMon answers its total and every frequency moment, KLL every rank, CountSketch item frequencies and L2, the heap sketches top-k and item frequencies. Exact accumulators are read as finalized values and answer no readout. --- .../types/src/post_asap/family_capability.rs | 223 +++++++++++++++++- crates/types/src/post_asap/mod.rs | 4 +- 2 files changed, 223 insertions(+), 4 deletions(-) diff --git a/crates/types/src/post_asap/family_capability.rs b/crates/types/src/post_asap/family_capability.rs index a6d4c5ac8..25f087ee2 100644 --- a/crates/types/src/post_asap/family_capability.rs +++ b/crates/types/src/post_asap/family_capability.rs @@ -3,9 +3,10 @@ //! //! The IR stays permissive: a `SummaryMerge` over any state family is //! structurally valid. The planner rules and the executor consult this -//! table to decide whether a composition is sound. Today the table holds -//! the mergeability facts; the summary-capability facts (which statistics -//! a family answers) are added as further fields of [`FamilyCapability`]. +//! table to decide whether a composition is sound. It holds the +//! mergeability facts and the summary-capability facts: which readouts +//! ([`Readout`]) one state of the family answers, so one state can serve +//! every consumer whose statistic it answers. use super::sketch::{ExactKind, GroupingStrategy, HydraKind, SketchAlgorithm, SketchStatistic}; use crate::pre_asap::schema::FieldDataType; @@ -20,6 +21,44 @@ pub struct FamilyCapability { /// (`SummarySubtract`, reserved): the subtrahend's selection must lie /// inside the minuend's, and the result is the state of the difference. pub subtractable: bool, + /// The readouts one state of the family answers, whatever its + /// parameters; none for a family read only as a finalized value (exact + /// accumulators) or one not listed. + pub readouts: &'static [Readout], +} + +/// One kind of readout (`SummaryEstimate` statistic) of a summary state. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Readout { + /// The total of the updates: a bare `PointCount` (no item value). + Total, + /// One item's frequency: a `PointCount` with an item value. + ItemFrequency, + /// The value at any quantile rank. + Quantile, + /// The number of distinct items. + Distinct, + /// The L2 norm of the item frequencies. + FrequencyL2, + /// The entropy of the item frequencies. + FrequencyEntropy, + /// The heaviest items, any `k` up to the state's heap. + TopK, +} + +impl Readout { + /// The kind of readout `statistic` is. + pub fn of(statistic: &SketchStatistic) -> Self { + match statistic { + SketchStatistic::PointCount { value: None, .. } => Self::Total, + SketchStatistic::PointCount { value: Some(_), .. } => Self::ItemFrequency, + SketchStatistic::Quantile { .. } => Self::Quantile, + SketchStatistic::Cardinality => Self::Distinct, + SketchStatistic::FrequencyL2 => Self::FrequencyL2, + SketchStatistic::FrequencyEntropy => Self::FrequencyEntropy, + SketchStatistic::TopK { .. } => Self::TopK, + } + } } /// How a family's states merge. @@ -66,9 +105,21 @@ pub enum MergedError { } impl FamilyCapability { + /// Whether one state of the family answers `statistic`. + pub fn answers(&self, statistic: &SketchStatistic) -> bool { + self.readouts.contains(&Readout::of(statistic)) + } + /// The capability of `family`. Families not listed have no capability /// (they fail closed). pub fn of(family: &FieldDataType) -> Self { + Self { + readouts: readouts(family), + ..Self::merge_facts(family) + } + } + + fn merge_facts(family: &FieldDataType) -> Self { use ExactKind as E; use SelectionRelation::{Disjoint, DisjointInTime, OverlapAllowed}; use SketchAlgorithm as S; @@ -150,6 +201,7 @@ impl FamilyCapability { Self { merge, subtractable, + readouts: &[], } } @@ -163,3 +215,168 @@ impl FamilyCapability { }) } } + +/// The readouts one state of `family` answers. +fn readouts(family: &FieldDataType) -> &'static [Readout] { + use Readout::*; + use SketchAlgorithm as S; + match family { + FieldDataType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { kind, .. }) => { + match kind { + // Per subpopulation, as the inner family answers. + HydraKind::HydraCms | HydraKind::HydraCountSketch => &[ItemFrequency], + HydraKind::HydraKll => &[Quantile], + } + } + FieldDataType::Sketch(kind, GroupingStrategy::PerSubpopulationInstance) => { + match kind.algorithm() { + // One frequency vector serves its total and every moment. + S::UnivMon => &[Total, Distinct, FrequencyL2, FrequencyEntropy], + // A quantile sketch answers every rank. + S::Kll => &[Quantile], + S::DDSketch => &[Quantile, Total], + S::Cms => &[ItemFrequency, Total], + // Each row's squared counters estimate F2 (AMS). + S::CountSketch => &[ItemFrequency, FrequencyL2], + S::CmsWithHeap | S::CountSketchWithHeap => &[TopK, ItemFrequency], + S::Hll | S::Kmv | S::Theta => &[Distinct], + } + } + FieldDataType::ExactAggregate(..) + | FieldDataType::Plain(_) + | FieldDataType::Sample(..) + | FieldDataType::Wavelet(..) + | FieldDataType::StatModel(..) => &[], + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::post_asap::{ExactParams, SketchKind, SketchParams}; + use crate::pre_asap::expr_ir::ColumnRef; + + fn sketch(algorithm: SketchAlgorithm, params: SketchParams) -> FieldDataType { + FieldDataType::Sketch( + SketchKind::new(algorithm, params), + GroupingStrategy::default(), + ) + } + + fn total() -> SketchStatistic { + SketchStatistic::PointCount { + key: ColumnRef::SampleValue, + value: None, + } + } + + fn item() -> SketchStatistic { + SketchStatistic::PointCount { + key: ColumnRef::Named("src_ip".into()), + value: Some("10.0.0.1".into()), + } + } + + /// One UnivMon answers its total, the distinct count, L2 and entropy, + /// so it can serve all four; it answers no quantile or top-k. + #[test] + fn univmon_answers_its_total_and_every_frequency_moment() { + let univmon = FamilyCapability::of(&sketch( + SketchAlgorithm::UnivMon, + SketchParams::UnivMon { + heap_size: 256, + sketch_rows: 5, + sketch_cols: 1024, + layers: 16, + distinct_failure_probability: None, + }, + )); + for statistic in [ + total(), + SketchStatistic::Cardinality, + SketchStatistic::FrequencyL2, + SketchStatistic::FrequencyEntropy, + ] { + assert!(univmon.answers(&statistic), "{statistic:?}"); + } + for statistic in [ + SketchStatistic::Quantile { q: 0.5 }, + SketchStatistic::TopK { k: 3 }, + ] { + assert!(!univmon.answers(&statistic), "{statistic:?}"); + } + } + + /// A quantile sketch answers every rank; heap sketches answer top-k and + /// item frequencies; CountSketch answers item frequencies and L2. + #[test] + fn summary_capability_facts() { + let kll = sketch( + SketchAlgorithm::Kll, + SketchParams::Kll { + k: 200, + failure_probability: None, + }, + ); + let count_sketch = sketch( + SketchAlgorithm::CountSketch, + SketchParams::CountSketch { + width: 64, + depth: 5, + }, + ); + let cms_heap = sketch( + SketchAlgorithm::CmsWithHeap, + SketchParams::CmsWithHeap { + width: 64, + depth: 5, + heap_size: 10, + }, + ); + let hll = sketch(SketchAlgorithm::Hll, SketchParams::Hll { precision: 12 }); + for (family, yes, no) in [ + ( + &kll, + vec![ + SketchStatistic::Quantile { q: 0.5 }, + SketchStatistic::Quantile { q: 0.99 }, + ], + vec![SketchStatistic::Cardinality, total()], + ), + ( + &count_sketch, + vec![item(), SketchStatistic::FrequencyL2], + vec![ + SketchStatistic::FrequencyEntropy, + SketchStatistic::TopK { k: 3 }, + ], + ), + ( + &cms_heap, + vec![SketchStatistic::TopK { k: 10 }, item()], + vec![SketchStatistic::FrequencyL2], + ), + ( + &hll, + vec![SketchStatistic::Cardinality], + vec![SketchStatistic::FrequencyL2, item()], + ), + ] { + let capability = FamilyCapability::of(family); + for statistic in yes { + assert!(capability.answers(&statistic), "{family:?} {statistic:?}"); + } + for statistic in no { + assert!(!capability.answers(&statistic), "{family:?} {statistic:?}"); + } + } + } + + /// An exact accumulator is read as its finalized value, not a readout. + #[test] + fn exact_accumulators_answer_no_readout() { + let sum = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); + assert!(FamilyCapability::of(&sum).readouts.is_empty()); + assert!(!FamilyCapability::of(&sum).answers(&total())); + } +} diff --git a/crates/types/src/post_asap/mod.rs b/crates/types/src/post_asap/mod.rs index 93dba180e..6da106382 100644 --- a/crates/types/src/post_asap/mod.rs +++ b/crates/types/src/post_asap/mod.rs @@ -37,7 +37,9 @@ pub use execution_data_state::{ lift_plain, DataPrimitive, ExactOperationSchemaError, ExecutionDataState, ExecutionDataStateError, ExecutionTiming, Materialization, Medium, }; -pub use family_capability::{FamilyCapability, MergeCapability, MergedError, SelectionRelation}; +pub use family_capability::{ + FamilyCapability, MergeCapability, MergedError, Readout, SelectionRelation, +}; pub use guarantee::{ AccuracyError, AccuracyGuarantee, BoundExpr, CompositionOperator, ErrorMetric, GuaranteeSource, ProbabilityExpr, From 3dc53756dcf5e8b58bb223ebafd993471439131c Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 15 Oct 2026 02:32:04 +0000 Subject: [PATCH 02/14] feat(logical-asap-optimization): Pass 2 summary-capability rule Composition sizes each consumer's summary build alone, for its own target. The summary-capability rule groups the builds of one candidate that chose the same family over the same input sub-DAG (window, WHERE and time range included), item and weight, grouping and filter, whose every reader is an estimate the family answers (the family-capability table's readouts), and that differ only in size. For every choice, per group, of no sharing or one subset of two or more members whose sizes differ (partial sharing included), the subset becomes one build sized with each parameter's largest value among the members (a KLL's failure probability and DDSketch's alpha: the smallest), and every guarantee above it is composed again, so plan selection checks each consumer against the shared state. Builds of equal size stay the identical-expression rule's; unshared builds are untouched and no consumer changes family. Stage 1 lists each composed candidate, its identical-expression candidates, then each summary-capability candidate followed by its own identical-expression candidates, every distinct DAG once. The Pass 2 record (Pass2Candidate) names the shared summaries ("UnivMon for Q1, Q3") and the shared identical groups. Example 2 (PromQL stand-in): with #702's sizing the three UnivMons differ (Q1 a distinct-count heap on 19 layers, Q2 the baseline shape for entropy, Q3 a one-layer Count Sketch for L2), so the rule offers one UnivMon for each pair and one for all three, each sized with the members' largest heap, rows, columns and layers and the smallest recorded distinct-count failure probability; the third query keeps its own options. A shared UnivMon whose rows and columns together exceed asap_sketchlib's hash budget (rows * (ceil(log2 cols) + 1) <= 128) is not offered, since the library would not build it. The shared UnivMon stays one build in Stage 2. --- .../src/accuracy/estimators/mod.rs | 220 ++++++++ .../asap-aware-mapping/src/asap_aware_cse.rs | 531 +++++++++++++++++- crates/asap-aware-mapping/src/pass/mod.rs | 4 +- .../src/pass/stage1_to_stage3.rs | 203 +++++-- .../src/per_computation_candidates.rs | 26 + .../tests/planner_layering_example2.rs | 42 +- 6 files changed, 967 insertions(+), 59 deletions(-) diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/mod.rs b/crates/asap-aware-mapping/src/accuracy/estimators/mod.rs index 922385dcd..516512f71 100644 --- a/crates/asap-aware-mapping/src/accuracy/estimators/mod.rs +++ b/crates/asap-aware-mapping/src/accuracy/estimators/mod.rs @@ -128,6 +128,7 @@ pub(crate) fn size_params( } } } + /// `P(Binomial(n, p) ≥ k)`, summed term by term in log space and rounded /// up by a relative 1e-12 so rounding never certifies a smaller tail. pub(crate) fn binomial_upper_tail(n: u32, p: f64, k: u32) -> f64 { @@ -165,6 +166,131 @@ pub(crate) fn column_collision(cols: u32) -> f64 { } } +/// The parameters of one state shared by consumers of the same algorithm, +/// each sized alone with one of `params` (Pass 2's summary-capability +/// rule): each parameter's largest value, except where a smaller value is +/// stricter, where it takes the smallest (a KLL's failure probability, +/// DDSketch's α). Every sizing formula is monotonic in its parameters, so +/// the shared state meets each consumer's target. `None` for no parameters, +/// parameters of different algorithms, or a shared UnivMon the library +/// would not build ([`univmon::hash_independent`]). +pub(crate) fn shared_size_params(params: &[&SketchParams]) -> Option { + use SketchParams as P; + let (first, rest) = params.split_first()?; + let shared = rest.iter().try_fold((*first).clone(), |a, b| { + Some(match (a, *b) { + ( + P::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, + distinct_failure_probability, + }, + P::UnivMon { + heap_size: h, + sketch_rows: r, + sketch_cols: c, + layers: l, + distinct_failure_probability: f, + }, + ) => P::UnivMon { + heap_size: heap_size.max(*h), + sketch_rows: sketch_rows.max(*r), + sketch_cols: sketch_cols.max(*c), + layers: layers.max(*l), + // `None`: not sized for a distinct count. + distinct_failure_probability: min_recorded(distinct_failure_probability, *f), + }, + ( + P::Kll { + k, + failure_probability, + }, + P::Kll { + k: k2, + failure_probability: f2, + }, + ) => P::Kll { + k: k.max(*k2), + // `None` is the calibration's δ, above any recorded one. + failure_probability: min_recorded(failure_probability, *f2), + }, + (P::Cms { width, depth }, P::Cms { width: w, depth: d }) => P::Cms { + width: width.max(*w), + depth: depth.max(*d), + }, + (P::CountSketch { width, depth }, P::CountSketch { width: w, depth: d }) => { + P::CountSketch { + width: width.max(*w), + depth: depth.max(*d), + } + } + ( + P::CmsWithHeap { + width, + depth, + heap_size, + }, + P::CmsWithHeap { + width: w, + depth: d, + heap_size: h, + }, + ) => P::CmsWithHeap { + width: width.max(*w), + depth: depth.max(*d), + heap_size: heap_size.max(*h), + }, + ( + P::CountSketchWithHeap { + width, + depth, + heap_size, + }, + P::CountSketchWithHeap { + width: w, + depth: d, + heap_size: h, + }, + ) => P::CountSketchWithHeap { + width: width.max(*w), + depth: depth.max(*d), + heap_size: heap_size.max(*h), + }, + (P::Hll { precision }, P::Hll { precision: p }) => P::Hll { + precision: precision.max(*p), + }, + (P::DDSketch { alpha }, P::DDSketch { alpha: a }) => P::DDSketch { + alpha: alpha.min(*a), + }, + (P::Kmv { k }, P::Kmv { k: k2 }) => P::Kmv { k: k.max(*k2) }, + (P::Theta { k }, P::Theta { k: k2 }) => P::Theta { k: k.max(*k2) }, + _ => return None, + }) + })?; + // The largest rows of one UnivMon with the largest columns of another + // can pass the hash budget `asap_sketchlib` builds within. + match shared { + P::UnivMon { + sketch_rows, + sketch_cols, + layers, + .. + } if !univmon::hash_independent(sketch_rows, sketch_cols, layers) => None, + shared => Some(shared), + } +} + +/// The smaller of two recorded failure probabilities; an unrecorded one +/// (`None`) yields to a recorded one. +fn min_recorded(a: Option, b: Option) -> Option { + match (a, b) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + } +} + /// Accuracy-dependent candidate budget, not a completeness theorem. The /// membership model must still certify the selected set independently. pub(crate) fn topk_capacity(k: usize, eps: f64) -> u32 { @@ -181,3 +307,97 @@ pub(crate) fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { } (x.ceil() as u32).clamp(lo, hi) } + +#[cfg(test)] +mod shared_size_tests { + use super::*; + + /// The shared size takes each parameter's largest value, and the + /// smallest where smaller is stricter (KLL δ, DDSketch α); sizes of + /// different algorithms have none. + #[test] + fn shared_size_takes_each_parameters_strictest_value() { + assert_eq!( + shared_size_params(&[&univmon(256, 5, 1024, 16), &univmon(1, 7, 4096, 1)]), + Some(univmon(256, 7, 4096, 16)) + ); + // The distinct count's failure probability, where one was sized + // for, takes the smallest. + let distinct = |heap_size, delta| SketchParams::UnivMon { + heap_size, + sketch_rows: 1, + sketch_cols: 1, + layers: 19, + distinct_failure_probability: Some(delta), + }; + let Some(SketchParams::UnivMon { + heap_size, + distinct_failure_probability, + .. + }) = shared_size_params(&[ + &univmon(256, 5, 1024, 16), + &distinct(4096, 0.01), + &distinct(2048, 0.001), + ]) + else { + unreachable!() + }; + assert_eq!( + (heap_size, distinct_failure_probability), + (4096, Some(0.001)) + ); + let kll = |k, failure_probability| SketchParams::Kll { + k, + failure_probability, + }; + let SketchParams::Kll { + k, + failure_probability, + } = shared_size_params(&[ + &kll(200, None), + &kll(100, Some(0.001)), + &kll(150, Some(0.005)), + ]) + .unwrap() + else { + unreachable!() + }; + assert_eq!((k, failure_probability), (200, Some(0.001))); + assert_eq!( + shared_size_params(&[ + &SketchParams::DDSketch { alpha: 0.01 }, + &SketchParams::DDSketch { alpha: 0.001 } + ]), + Some(SketchParams::DDSketch { alpha: 0.001 }) + ); + assert_eq!( + shared_size_params(&[&kll(200, None), &SketchParams::Hll { precision: 12 }]), + None + ); + assert_eq!(shared_size_params(&[]), None); + } + + fn univmon(heap_size: u32, sketch_rows: u32, sketch_cols: u32, layers: u8) -> SketchParams { + SketchParams::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, + distinct_failure_probability: None, + } + } + + /// Each UnivMon fits `asap_sketchlib`'s hash budget, but the largest + /// rows of one with the largest columns of the other do not + /// (`12 · (⌈log2 2^16⌉ + 1) = 204 > 128`): the library would not build + /// the shared shape, so there is none. + #[test] + fn no_shared_univmon_past_the_hash_budget() { + assert!(univmon::hash_independent(12, 512, 1)); + assert!(univmon::hash_independent(5, 1 << 16, 1)); + assert_eq!( + shared_size_params(&[&univmon(1, 12, 512, 1), &univmon(1, 5, 1 << 16, 1)]), + None + ); + } +} diff --git a/crates/asap-aware-mapping/src/asap_aware_cse.rs b/crates/asap-aware-mapping/src/asap_aware_cse.rs index 359dccf1a..35e23b3d8 100644 --- a/crates/asap-aware-mapping/src/asap_aware_cse.rs +++ b/crates/asap-aware-mapping/src/asap_aware_cse.rs @@ -2,7 +2,12 @@ //! common-subexpression elimination, as defined in //! `docs/design_docs/proposals/planner-layering.md` (Algorithm 2). //! -//! Only the identical-expression rule is implemented. A *sharing group* of a +//! Two of its rules are implemented: the identical-expression rule and the +//! summary-capability rule. +//! +//! # Identical-expression rule +//! +//! A *sharing group* of a //! whole-workload candidate is a set of two or more of its nodes with //! identical input and computation semantics that are legal to share: the //! `ir::cse` interning merges only ASAP nodes and nodes with a unique key. @@ -17,7 +22,26 @@ //! same candidate as the subset with that group; such a candidate is //! returned once, under the subset of the groups it actually shares. //! -//! The summary-capability and window-composition rules are not implemented. +//! # Summary-capability rule +//! +//! Composition sizes each consumer's summary build alone, for its own +//! accuracy target. The *members* of a [`CapabilityGroup`] are summary +//! builds of one candidate that chose the same family (same +//! [`SketchAlgorithm`]) over the same input sub-DAG (window, `WHERE` and +//! time range included), item and weight, grouping and filter, and whose +//! every reader is an estimate the family answers +//! ([`FamilyCapability::answers`]); they differ only in their sizes. +//! [`summary_capability_variants`] returns one candidate per choice, for +//! each group, of no sharing or one subset of two or more of its members +//! whose sizes differ (Q1: partial sharing too): the subset becomes one +//! build serving all its readers, sized with each parameter's largest value +//! among them ([`shared_size_params`]: each consumer sized alone, the shared +//! build the per-parameter maximum, a KLL's failure probability the +//! minimum), and every guarantee above it is composed again, so plan +//! selection checks each consumer against the shared state. Members of +//! equal size are the identical-expression rule's; builds that are not +//! shared are untouched, and a consumer never changes family. The window- +//! composition rule is not implemented. use std::collections::HashMap; #[cfg(test)] @@ -25,7 +49,13 @@ use std::collections::HashSet; use std::rc::Rc; use asap_types::ir::cse::share_common_sub_dags; -use asap_types::ir::{OperatorNode, QueryRoot}; +use asap_types::ir::{ASAPOp, Operator, OperatorNode, QueryRoot}; +use asap_types::post_asap::{ + FamilyCapability, FieldDataType, GroupingStrategy, SketchAlgorithm, SketchKind, +}; + +use crate::accuracy::estimators::shared_size_params; +use crate::per_computation_candidates::{with_composed_guarantees, PerComputationCandidateError}; /// A Pass 2 candidate: the candidate's roots with the sharing groups /// `shared_groups` (indices into [`sharing_groups`]) shared. @@ -254,6 +284,329 @@ impl Sharing<'_> { } } +/// One group of the summary-capability rule: summary builds of one family +/// over the same input data that differ only in size, in the order a +/// children-first walk of the roots meets them. +#[derive(Debug, Clone)] +pub struct CapabilityGroup { + pub family: SketchAlgorithm, + pub members: Vec>, +} + +/// One shared build of a summary-capability candidate: the members +/// (indices into the group's [`CapabilityGroup::members`]) of group +/// `group` that read one state. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SharedSummary { + pub group: usize, + pub members: Vec, +} + +/// A Pass 2 candidate of the summary-capability rule: the candidate's +/// roots with the builds in `shared` made one build each, sized for all +/// their consumers, and every guarantee composed again. +#[derive(Debug, Clone)] +pub struct SummaryCapabilityVariant { + pub shared: Vec, + pub roots: Vec, +} + +/// The summary-capability groups of the candidate with query roots +/// `roots`: groups of at least two builds whose sizes are not all equal. +pub fn capability_groups(roots: &[QueryRoot]) -> Vec { + let mut nodes = Vec::new(); + let mut seen = std::collections::HashSet::new(); + for node in roots.iter().flat_map(root_operators) { + collect_children_first(node, &mut seen, &mut nodes); + } + // Every node that reads each build, by the build's address. + let mut readers: HashMap<*const OperatorNode, Vec<&Rc>> = HashMap::new(); + for node in &nodes { + for child in node.children() { + readers.entry(Rc::as_ptr(child)).or_default().push(node); + } + } + let mut groups: Vec = Vec::new(); + for node in &nodes { + let Some(family) = shareable_family(node, readers.get(&Rc::as_ptr(node))) else { + continue; + }; + match groups + .iter_mut() + .find(|g| g.family == family && same_input_data(&g.members[0], node)) + { + Some(group) => group.members.push(Rc::clone(node)), + None => groups.push(CapabilityGroup { + family, + members: vec![Rc::clone(node)], + }), + } + } + groups.retain(|g| !all_equal(g.members.iter())); + groups +} + +fn collect_children_first( + node: &Rc, + seen: &mut std::collections::HashSet<*const OperatorNode>, + out: &mut Vec>, +) { + if !seen.insert(Rc::as_ptr(node)) { + return; + } + for child in node.children() { + collect_children_first(child, seen, out); + } + out.push(Rc::clone(node)); +} + +/// The family of a per-subpopulation sketch build whose every reader is an +/// estimate the family answers; `None` for any other node. +fn shareable_family( + node: &OperatorNode, + readers: Option<&Vec<&Rc>>, +) -> Option { + let Some(ASAPOp::SummaryAgg { + family: family @ FieldDataType::Sketch(kind, GroupingStrategy::PerSubpopulationInstance), + .. + }) = node.asap() + else { + return None; + }; + let capability = FamilyCapability::of(family); + let answered = |reader: &&Rc| { + matches!( + reader.asap(), + Some(ASAPOp::SummaryEstimate { query, .. }) if capability.answers(query) + ) + }; + let readers = readers?; + readers + .iter() + .all(answered) + .then(|| kind.algorithm().clone()) +} + +/// Whether two builds of one family read the same data the same way: the +/// same input sub-DAG, item and weight, grouping and filter. +fn same_input_data(a: &OperatorNode, b: &OperatorNode) -> bool { + match (a.asap(), b.asap()) { + ( + Some(ASAPOp::SummaryAgg { + child, + input, + reduction, + grouping, + filter, + .. + }), + Some(ASAPOp::SummaryAgg { + child: child_b, + input: input_b, + reduction: reduction_b, + grouping: grouping_b, + filter: filter_b, + .. + }), + ) => { + (Rc::ptr_eq(child, child_b) || child == child_b) + && input == input_b + && reduction == reduction_b + && grouping == grouping_b + && filter == filter_b + } + _ => false, + } +} + +fn all_equal<'a>(mut nodes: impl Iterator>) -> bool { + let Some(first) = nodes.next() else { + return true; + }; + nodes.all(|node| Rc::ptr_eq(node, first) || **node == **first) +} + +/// Every Pass 2 candidate of the summary-capability rule for the candidate +/// with query roots `roots` and groups `groups` ([`capability_groups`]): +/// one per choice, for each group, of no sharing or one subset of two or +/// more members whose sizes are not all equal, other than sharing nothing. +/// The first group varies slowest; within a group, subsets come in +/// binary-counter order (the first member the lowest bit). +pub fn summary_capability_variants( + roots: &[QueryRoot], + groups: &[CapabilityGroup], +) -> Result, PerComputationCandidateError> { + // Each group's options: `None` (no sharing), then its subsets. + let options: Vec>>> = groups + .iter() + .map(|group| { + let n = group.members.len(); + let mut options = vec![None]; + for mask in 1u64..(1u64 << n.min(63)) { + let members: Vec = (0..n).filter(|&m| mask >> m & 1 == 1).collect(); + if members.len() >= 2 && !all_equal(members.iter().map(|&m| &group.members[m])) { + options.push(Some(members)); + } + } + options + }) + .collect(); + let mut choice = vec![0usize; groups.len()]; + let mut variants = Vec::new(); + loop { + // Mixed-radix increment, the last group varying fastest. + let mut carried = true; + for (digit, group_options) in choice.iter_mut().zip(&options).rev() { + *digit += 1; + if *digit < group_options.len() { + carried = false; + break; + } + *digit = 0; + } + if carried { + break; + } + let shared: Vec = choice + .iter() + .enumerate() + .filter_map(|(group, &option)| { + options[group][option] + .clone() + .map(|members| SharedSummary { group, members }) + }) + .collect(); + if let Some(roots) = share_summaries(roots, groups, &shared)? { + variants.push(SummaryCapabilityVariant { shared, roots }); + } + } + Ok(variants) +} + +/// `roots` with each of `shared` made one build sized for all its members, +/// and every guarantee composed again; `None` when the members' sizes have +/// no shared size. +fn share_summaries( + roots: &[QueryRoot], + groups: &[CapabilityGroup], + shared: &[SharedSummary], +) -> Result>, PerComputationCandidateError> { + let mut replaced: HashMap<*const OperatorNode, Rc> = HashMap::new(); + for summary in shared { + let members: Vec<&Rc> = summary + .members + .iter() + .map(|&m| &groups[summary.group].members[m]) + .collect(); + let Some(build) = sized_for_all(&members)? else { + return Ok(None); + }; + for member in members { + replaced.insert(Rc::as_ptr(member), Rc::clone(&build)); + } + } + let mut rebuilt: HashMap<*const OperatorNode, Rc> = HashMap::new(); + let mut rebuild = |node: &Rc| rebuild(node, &replaced, &mut rebuilt); + let mut out = Vec::new(); + for root in roots { + out.push(match root { + QueryRoot::Operator(node) => QueryRoot::Operator(rebuild(node)?), + QueryRoot::Scalar(expr) => { + let mut nodes = HashMap::new(); + for node in expr.operator_refs() { + nodes.insert(Rc::as_ptr(node), rebuild(node)?); + } + QueryRoot::Scalar( + expr.map_operator_refs(&mut |node| nodes[&Rc::as_ptr(node)].clone()), + ) + } + }); + } + Ok(Some(with_composed_guarantees(&out)?)) +} + +/// One build serving every member: the first member with the members' +/// shared size ([`shared_size_params`]). +fn sized_for_all( + members: &[&Rc], +) -> Result>, PerComputationCandidateError> { + let kinds: Vec<&SketchKind> = members + .iter() + .filter_map(|member| match member.asap() { + Some(ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(kind, _), + .. + }) => Some(kind), + _ => None, + }) + .collect(); + let params: Vec<_> = kinds.iter().map(|kind| kind.params()).collect(); + let Some(params) = shared_size_params(¶ms) else { + return Ok(None); + }; + let first = members[0]; + let Some(ASAPOp::SummaryAgg { + child, + family: FieldDataType::Sketch(kind, grouping_strategy), + input, + reduction, + grouping, + filter, + }) = first.asap() + else { + return Ok(None); + }; + let operator = Operator::ASAP(ASAPOp::SummaryAgg { + child: Rc::clone(child), + family: FieldDataType::Sketch( + SketchKind::new(kind.algorithm().clone(), params), + grouping_strategy.clone(), + ), + input: input.clone(), + reduction: reduction.clone(), + grouping: grouping.clone(), + filter: filter.clone(), + }); + let mut build = OperatorNode::new(operator)?; + // Keep the member's explicit output names. + for (field, retained) in build.schema.fields.iter_mut().zip(&first.schema.fields) { + field.name = retained.name.clone(); + field.table = retained.table.clone(); + } + Ok(Some(Rc::new(build))) +} + +/// `node` with every replaced build below it swapped for its shared build; +/// the same `Rc` when nothing below changed. +fn rebuild( + node: &Rc, + replaced: &HashMap<*const OperatorNode, Rc>, + rebuilt: &mut HashMap<*const OperatorNode, Rc>, +) -> Result, PerComputationCandidateError> { + if let Some(done) = rebuilt.get(&Rc::as_ptr(node)) { + return Ok(Rc::clone(done)); + } + let out = match replaced.get(&Rc::as_ptr(node)) { + Some(build) => Rc::clone(build), + None => { + let mut changed = false; + let mut children = HashMap::new(); + for child in node.children() { + let new = rebuild(child, replaced, rebuilt)?; + changed |= !Rc::ptr_eq(&new, child); + children.insert(Rc::as_ptr(child), new); + } + if changed { + Rc::new(node.with_new_children(|child| children[&Rc::as_ptr(child)].clone())?) + } else { + Rc::clone(node) + } + } + }; + rebuilt.insert(Rc::as_ptr(node), Rc::clone(&out)); + Ok(out) +} + fn root_operators(root: &QueryRoot) -> Vec<&Rc> { match root { QueryRoot::Operator(node) => vec![node], @@ -454,6 +807,178 @@ mod tests { ); } + /// The composed candidate of `queries` (each at its own ε, δ = 0.01) + /// in which every target chose `family`. + fn chose(queries: &[(&str, f64)], family: SketchAlgorithm) -> Vec { + use crate::per_computation_candidates::{ + compose_logical_candidate, generate_per_computation_candidates, ComputationCandidate, + }; + let roots = queries + .iter() + .enumerate() + .map(|(i, (q, epsilon))| { + let root = lower_promql( + q, + asap_types::types::RequestedAccuracy::epsilon_delta(*epsilon, 0.01) + .with_metric(asap_types::types::ErrorMetric::Rank), + ); + ( + i, + QueryRoot::Operator(with_promql_series_identity(&root).unwrap()), + ) + }) + .collect(); + let inventory = generate_per_computation_candidates(roots).unwrap(); + let choice: Vec = inventory + .targets + .iter() + .map(|t| { + t.choices + .candidates + .iter() + .position(|c| *c == ComputationCandidate::Summary(family.clone())) + .expect("the family is a candidate") + }) + .collect(); + compose_logical_candidate(&inventory, &choice) + .unwrap() + .into_iter() + .map(|(_, root)| root) + .collect() + } + + /// The summary builds `roots` read, by address, with each one's KLL `k`. + fn kll_builds(roots: &[QueryRoot]) -> Vec<(*const OperatorNode, u32)> { + let mut out: Vec<(*const OperatorNode, u32)> = Vec::new(); + for node in roots + .iter() + .flat_map(root_operators) + .flat_map(OperatorNode::reachable) + { + if let Some(ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(kind, _), + .. + }) = node.asap() + { + let asap_types::post_asap::SketchParams::Kll { k, .. } = kind.params() else { + panic!("{kind:?}") + }; + if !out.iter().any(|(p, _)| *p == Rc::as_ptr(&node)) { + out.push((Rc::as_ptr(&node), *k)); + } + } + } + out + } + + fn root_bound(root: &QueryRoot) -> f64 { + let QueryRoot::Operator(node) = root else { + panic!("operator root") + }; + node.guarantee + .as_ref() + .and_then(|g| g.spec.error.evaluate()) + .expect("a constant bound") + } + + /// p50 at ε = 0.01 and p99 at ε = 0.001 over one window, each with its + /// own KLL: the rule shares one KLL with the larger `k`, and p50's + /// guarantee is the shared KLL's, tighter than its own. + #[test] + fn quantiles_at_different_targets_share_one_kll_with_the_larger_k() { + let roots = chose( + &[ + ("quantile_over_time(0.5, lat[5m])", 0.01), + ("quantile_over_time(0.99, lat[5m])", 0.001), + ], + SketchAlgorithm::Kll, + ); + let own = kll_builds(&roots); + assert_eq!(own.len(), 2); + assert!(own[0].1 < own[1].1); + let groups = capability_groups(&roots); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].family, SketchAlgorithm::Kll); + let variants = summary_capability_variants(&roots, &groups).unwrap(); + assert_eq!(variants.len(), 1); + assert_eq!( + variants[0].shared, + [SharedSummary { + group: 0, + members: vec![0, 1] + }] + ); + let shared = kll_builds(&variants[0].roots); + assert_eq!(shared.len(), 1, "one KLL serves both"); + assert_eq!(shared[0].1, own[1].1); + assert!(root_bound(&variants[0].roots[0]) < root_bound(&roots[0])); + assert_eq!(root_bound(&variants[0].roots[1]), root_bound(&roots[1])); + } + + /// Three consumers at three targets: every subset of two or more of + /// them shares one build (three pairs and all three), the others keep + /// their own. + #[test] + fn every_subset_of_two_or_more_consumers_shares_one_build() { + let roots = chose( + &[ + ("quantile_over_time(0.5, lat[5m])", 0.05), + ("quantile_over_time(0.9, lat[5m])", 0.01), + ("quantile_over_time(0.99, lat[5m])", 0.001), + ], + SketchAlgorithm::Kll, + ); + let groups = capability_groups(&roots); + let variants = summary_capability_variants(&roots, &groups).unwrap(); + assert_eq!( + variants + .iter() + .map(|v| v.shared[0].members.clone()) + .collect::>(), + [vec![0, 1], vec![0, 2], vec![1, 2], vec![0, 1, 2]] + ); + for variant in &variants { + let builds = kll_builds(&variant.roots); + assert_eq!(builds.len(), 4 - variant.shared[0].members.len()); + } + } + + /// Equal targets give identical builds, which are the + /// identical-expression rule's; a different window or family is a + /// different input or capability, never shared. + #[test] + fn equal_sizes_other_windows_and_other_families_are_not_capability_groups() { + let equal = chose( + &[ + ("quantile_over_time(0.5, lat[5m])", 0.01), + ("quantile_over_time(0.99, lat[5m])", 0.01), + ], + SketchAlgorithm::Kll, + ); + assert!(capability_groups(&equal).is_empty()); + assert!(!identical_expression_variants(&equal, &sharing_groups(&equal)).is_empty()); + let windows = chose( + &[ + ("quantile_over_time(0.5, lat[5m])", 0.01), + ("quantile_over_time(0.99, lat[10m])", 0.001), + ], + SketchAlgorithm::Kll, + ); + assert!(capability_groups(&windows).is_empty()); + let families = chose( + &[ + ("quantile_over_time(0.5, lat[5m])", 0.01), + ("quantile_over_time(0.99, lat[5m])", 0.001), + ], + SketchAlgorithm::DDSketch, + ); + assert_eq!(capability_groups(&families).len(), 1); + assert_eq!( + capability_groups(&families)[0].family, + SketchAlgorithm::DDSketch + ); + } + /// Each candidate's label (the groups it shares) and its roots agree: /// applying the label gives the same roots, sharing exactly the /// labelled groups. diff --git a/crates/asap-aware-mapping/src/pass/mod.rs b/crates/asap-aware-mapping/src/pass/mod.rs index 3f37907a8..664d4a6a6 100644 --- a/crates/asap-aware-mapping/src/pass/mod.rs +++ b/crates/asap-aware-mapping/src/pass/mod.rs @@ -36,8 +36,8 @@ use crate::stage3_plan_selection::{Selection, SelectionError}; pub use stage1_to_stage3::{ run_stage1_to_stage3, run_stage1_to_stage3_from_pass1, run_stage2_and_stage3, - stage1_logical_asap_optimization, IdenticalExpressionCandidate, LogicalCandidate, - PrunedCandidate, Stage1Candidate, Stage1ToStage3Pass, Stage1ToStage3Run, + stage1_logical_asap_optimization, LogicalCandidate, Pass2Candidate, Pass2Of, PrunedCandidate, + Stage1Candidate, Stage1ToStage3Pass, Stage1ToStage3Run, }; static DEFAULT_COST_MODEL: DefaultCostModel = DefaultCostModel; diff --git a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs index 458e0fce8..4c6a817b2 100644 --- a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs +++ b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs @@ -10,16 +10,19 @@ //! generation, then composition of one whole-workload candidate per choice //! of a per-computation candidate for every target (the Cartesian product, //! in mixed-radix order, the last target varying fastest), each followed -//! by its Pass 2 candidates from the identical-expression rule +//! by its Pass 2 candidates: those of the identical-expression rule //! ([`identical_expression_variants`]: one per subset of its sharing -//! groups; the independent candidate is kept); +//! groups), then those of the summary-capability rule +//! ([`summary_capability_variants`]: one per choice of shared summary +//! builds), each followed by its own identical-expression candidates, +//! every distinct candidate once; the independent candidate is kept; //! 2. Stage 2, physical ASAP-aware optimization of every Stage 1 candidate //! ([`optimize_physical_asap`]); //! 3. Stage 3, plan selection over the physical candidates ([`select_plan`]). //! //! Stage 0, the query-language frontends, runs before it (the `asap-planner` -//! facade lowers the workload). Pass 2's summary-capability and -//! window-composition rules are not planned yet. +//! facade lowers the workload). Pass 2's window-composition rule is not +//! planned yet. //! //! Every combination is enumerated and plan selection prices every //! candidate; nothing is capped. A search that avoids building every @@ -27,6 +30,7 @@ use std::rc::Rc; +use asap_types::ir::flat::flatten; use asap_types::ir::schema_support::with_promql_series_identity; use asap_types::ir::{OperatorNode, QueryRoot}; use asap_types::post_asap::SketchAlgorithm; @@ -34,7 +38,9 @@ use asap_types::types::{AccuracyTarget, RequestedAccuracy}; use asap_types::workload::{DataWorkload, QueryLanguage, QueryRecurrence}; use super::{OptimizationInput, OptimizationPass, OptimizeError, PlanOutput, QueryPlan}; -use crate::asap_aware_cse::identical_expression_variants; +use crate::asap_aware_cse::{ + capability_groups, identical_expression_variants, sharing_groups, summary_capability_variants, +}; use crate::per_computation_candidates::{ compose_logical_candidate, generate_per_computation_candidates, ComputationCandidate, PerComputationCandidateError, PerComputationCandidates, @@ -57,14 +63,17 @@ pub struct LogicalCandidate { pub roots: Result, PerComputationCandidateError>, } -/// A Stage 1 candidate Pass 2's identical-expression rule added: `id` is -/// `from_logical` with the nodes of each group in `shared` (named by -/// operator kind, children first) made one node. It has `from_logical`'s -/// `choice`. +/// A Stage 1 candidate Pass 2 added: `id` is `from_logical` with each +/// summary build set in `summaries` made one build sized for all its +/// consumers (the summary-capability rule, e.g. "UnivMon for Q1, Q3"), +/// then the nodes of each group in `shared` (named by operator kind, +/// children first) made one node (the identical-expression rule). It has +/// `from_logical`'s `choice`. #[derive(Debug, Clone, PartialEq)] -pub struct IdenticalExpressionCandidate { +pub struct Pass2Candidate { pub id: String, pub from_logical: String, + pub summaries: Vec, pub shared: Vec, } @@ -90,9 +99,9 @@ pub struct Stage1ToStage3Run { /// the size of the Cartesian product of the targets' candidates, /// saturating at `usize::MAX`. Pass 2 adds to them. pub combinations: usize, - /// Stage 1, Pass 2: the candidates the identical-expression rule added, - /// in enumeration order. - pub stage1_pass2_identical_expression: Vec, + /// Stage 1, Pass 2: the candidates the identical-expression and + /// summary-capability rules added, in enumeration order. + pub stage1_pass2: Vec, /// Stage 1: every whole-workload logical candidate, in enumeration /// order: each composed Pass 1 candidate, followed by its Pass 2 /// candidates. @@ -159,17 +168,30 @@ pub struct Stage1Candidate { /// Pass 2 candidate has its Pass 1 candidate's. pub choice: Vec, pub roots: Result, PerComputationCandidateError>, - /// For a Pass 2 candidate: the position, in the enumeration, of the - /// Pass 1 candidate it shares the identical sub-DAGs of, and the shared - /// groups (operator kinds, children first). - pub pass2_of: Option<(usize, Vec)>, + /// For a Pass 2 candidate: what it shares of which Pass 1 candidate. + pub pass2_of: Option, +} + +/// What a Pass 2 candidate shares of a Pass 1 candidate. +#[derive(Debug, Clone, PartialEq)] +pub struct Pass2Of { + /// The position, in the enumeration, of the Pass 1 candidate. + pub from: usize, + /// The summary builds made one build each (summary-capability rule). + pub summaries: Vec, + /// The identical groups shared (operator kinds, children first). + pub shared: Vec, } /// Stage 1 (logical ASAP-aware optimization) over the Pass 1 inventory, one /// candidate at a time: each composed Pass 1 candidate (every choice of a /// per-computation candidate for every target, the last target varying -/// fastest), followed by its Pass 2 identical-expression candidates. Lazy -/// and uncapped: a caller that limits the candidates stops consuming it. +/// fastest), followed by its Pass 2 candidates: the identical-expression +/// rule's, then each summary-capability candidate followed by its own +/// identical-expression candidates. A Pass 2 candidate equal to an earlier +/// one of the same Pass 1 candidate (same nodes, same sharing) is listed +/// once. Lazy and uncapped: a caller that limits the candidates stops +/// consuming it. pub fn stage1_logical_asap_optimization( per_computation: &PerComputationCandidates, ) -> impl Iterator + '_ { @@ -191,32 +213,34 @@ pub fn stage1_logical_asap_optimization( let display_name = display_name(per_computation, &query_of_target, &this); let roots = compose_logical_candidate(per_computation, &this) .map(|roots| roots.into_iter().map(|(_, root)| root).collect::>()); - let (groups, variants) = match &roots { - Ok(roots) => { - let groups = crate::asap_aware_cse::sharing_groups(roots); - let variants = identical_expression_variants(roots, &groups); - (groups, variants) - } - Err(_) => (Vec::new(), Vec::new()), - }; let from = position + queue.len(); + let pass2 = match &roots { + Ok(roots) => pass2_candidates(roots), + Err(_) => Vec::new(), + }; queue.push_back(Stage1Candidate { display_name: display_name.clone(), choice: this.clone(), roots, pass2_of: None, }); - for variant in variants { - let shared: Vec = variant - .shared_groups - .iter() - .map(|&g| format!("{} #{}", groups[g].nodes[0].operator.kind_name(), g + 1)) - .collect(); + for (summaries, shared, roots) in pass2 { + let mut name = display_name.clone(); + if !summaries.is_empty() { + name += &format!(" · one {}", summaries.join(", one ")); + } + if !shared.is_empty() { + name += &format!(" · shared {}", shared.join(", ")); + } queue.push_back(Stage1Candidate { - display_name: format!("{display_name} · shared {}", shared.join(", ")), + display_name: name, choice: this.clone(), - roots: Ok(variant.roots), - pass2_of: Some((from, shared)), + roots, + pass2_of: Some(Pass2Of { + from, + summaries, + shared, + }), }); } } @@ -225,6 +249,102 @@ pub fn stage1_logical_asap_optimization( }) } +/// A Pass 2 candidate: the summary builds it shares (summary-capability +/// rule), the identical groups it shares, and its roots. +type Pass2 = ( + Vec, + Vec, + Result, PerComputationCandidateError>, +); + +/// The Pass 2 candidates of the composed candidate with roots `roots`, in +/// enumeration order, each distinct candidate once. +fn pass2_candidates(roots: &[QueryRoot]) -> Vec { + let mut found = Pass2List { + seen: vec![flatten(roots).0], + out: Vec::new(), + }; + found.identical_variants(&[], roots); + let groups = capability_groups(roots); + let variants = match summary_capability_variants(roots, &groups) { + Ok(variants) => variants, + Err(error) => { + found.out.push((Vec::new(), Vec::new(), Err(error))); + return found.out; + } + }; + for variant in variants { + let summaries: Vec = variant + .shared + .iter() + .map(|summary| { + let group = &groups[summary.group]; + let mut queries: Vec = summary + .members + .iter() + .flat_map(|&m| readers_of(roots, &group.members[m])) + .collect(); + queries.sort_unstable(); + queries.dedup(); + let queries: Vec = queries.iter().map(|q| format!("Q{}", q + 1)).collect(); + format!("{:?} for {}", group.family, queries.join(", ")) + }) + .collect(); + found.push(&summaries, Vec::new(), &variant.roots); + found.identical_variants(&summaries, &variant.roots); + } + found.out +} + +/// Pass 2 candidates found so far, and the flattened DAG of every candidate +/// listed (the Pass 1 candidate's included), so each is listed once. +struct Pass2List { + seen: Vec, + out: Vec, +} + +impl Pass2List { + fn push(&mut self, summaries: &[String], shared: Vec, roots: &[QueryRoot]) { + let flat = flatten(roots).0; + if self.seen.contains(&flat) { + return; + } + self.seen.push(flat); + self.out + .push((summaries.to_vec(), shared, Ok(roots.to_vec()))); + } + + /// The identical-expression candidates of `roots`, which shares + /// `summaries`. + fn identical_variants(&mut self, summaries: &[String], roots: &[QueryRoot]) { + let groups = sharing_groups(roots); + for variant in identical_expression_variants(roots, &groups) { + let shared = variant + .shared_groups + .iter() + .map(|&g| format!("{} #{}", groups[g].nodes[0].operator.kind_name(), g + 1)) + .collect(); + self.push(summaries, shared, &variant.roots); + } + } +} + +/// The query roots (indices) from which `node` is reachable. +fn readers_of(roots: &[QueryRoot], node: &Rc) -> Vec { + roots + .iter() + .enumerate() + .filter(|(_, root)| { + root_operators(root).into_iter().any(|op| { + OperatorNode::reachable(op) + .iter() + .any(|n| Rc::ptr_eq(n, node)) + }) + }) + .map(|(q, _)| q) + .collect() +} + /// [`run_stage1_to_stage3`] after Pass 1: enumerates every Stage 1 /// candidate of `per_computation` (the Pass 1 inventory of the frontends' /// roots, [`stage1_logical_asap_optimization`]), so a caller can read the @@ -275,11 +395,12 @@ pub fn run_stage2_and_stage3( }), } } - if let Some((from, shared)) = candidate.pass2_of { - pass2.push(IdenticalExpressionCandidate { + if let Some(of) = candidate.pass2_of { + pass2.push(Pass2Candidate { id: id.clone(), - from_logical: format!("L{}", from + 1), - shared, + from_logical: format!("L{}", of.from + 1), + summaries: of.summaries, + shared: of.shared, }); } logical.push(LogicalCandidate { @@ -310,7 +431,7 @@ pub fn run_stage2_and_stage3( stage3_selected_logical: selected_logical, stage1_pass1_per_computation: per_computation, combinations, - stage1_pass2_identical_expression: pass2, + stage1_pass2: pass2, stage1_logical_asap_optimization: logical, stage2_physical_asap_optimization: physical, stage2_pruned: pruned, diff --git a/crates/asap-aware-mapping/src/per_computation_candidates.rs b/crates/asap-aware-mapping/src/per_computation_candidates.rs index 71457a40d..63d405771 100644 --- a/crates/asap-aware-mapping/src/per_computation_candidates.rs +++ b/crates/asap-aware-mapping/src/per_computation_candidates.rs @@ -819,6 +819,32 @@ fn rewrite( Ok(rebuilt) } +/// `roots` with every node's guarantee composed again +/// ([`composed_guarantee`]), children first; a sub-DAG shared between roots +/// stays shared. Pass 2 calls it after re-sizing a shared summary, so each +/// consumer's guarantee is the shared state's. +pub(crate) fn with_composed_guarantees( + roots: &[QueryRoot], +) -> Result, PerComputationCandidateError> { + let mut memo = HashMap::new(); + roots + .iter() + .map(|root| { + Ok(match root { + QueryRoot::Operator(node) => QueryRoot::Operator(with_guarantees(node, &mut memo)?), + QueryRoot::Scalar(expr) => { + for node in expr.operator_refs() { + with_guarantees(node, &mut memo)?; + } + QueryRoot::Scalar( + expr.map_operator_refs(&mut |node| memo[&Rc::as_ptr(node)].clone()), + ) + } + }) + }) + .collect() +} + /// `node` and everything below it, each node with its [`composed_guarantee`]. fn with_guarantees( node: &Rc, diff --git a/crates/integration-tests/tests/planner_layering_example2.rs b/crates/integration-tests/tests/planner_layering_example2.rs index b0c87134d..50b7c5498 100644 --- a/crates/integration-tests/tests/planner_layering_example2.rs +++ b/crates/integration-tests/tests/planner_layering_example2.rs @@ -382,7 +382,6 @@ fn stage1_logical_asap_optimization_keeps_every_independent_combination() { /// The summary-capability rule adds one UnivMon build feeding a distinct-count, an entropy and an L2 estimate. #[test] -#[ignore = "needs the Pass 2 summary-capability rule for frequency moments (#595, #596)"] fn stage1_logical_asap_optimization_summary_capability_adds_one_univmon_for_all_three() { let run = pipeline(); let shared = run.logical.iter().find(|c| { @@ -402,7 +401,6 @@ fn stage1_logical_asap_optimization_summary_capability_adds_one_univmon_for_all_ /// For each pair, a candidate shares one UnivMon between the two while the third keeps each of its own options. #[test] -#[ignore = "needs the Pass 2 summary-capability rule for frequency moments (#595, #596)"] fn stage1_logical_asap_optimization_summary_capability_adds_pairwise_shared_univmons() { let run = pipeline(); for (pair, third) in [([0, 1], 2), ([0, 2], 1), ([1, 2], 0)] { @@ -416,23 +414,42 @@ fn stage1_logical_asap_optimization_summary_capability_adds_pairwise_shared_univ } } -/// A UnivMon shared by several statistics is sized for the strictest consumer. +/// A UnivMon shared by several statistics is sized for every consumer: +/// each consumer is sized alone, and the shared build takes each +/// parameter's largest value among them. #[test] -#[ignore = "needs the Pass 2 summary-capability rule for frequency moments (#595, #596)"] -fn stage1_logical_asap_optimization_shared_univmon_is_sized_for_the_strictest_consumer() { +fn stage1_logical_asap_optimization_shared_univmon_takes_each_parameters_largest_own_size() { let run = pipeline(); - let strictest = |queries: &[usize]| { - *queries - .iter() - .min_by(|a, b| ACCURACY[**a].0.total_cmp(&ACCURACY[**b].0)) - .unwrap() + let own = |q| match own_univmon(&run, q) { + SketchParams::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, + .. + } => [heap_size, sketch_rows, sketch_cols, u32::from(layers)], + other => panic!("{other:?}"), }; let mut seen = 0; for queries in [vec![0, 1, 2], vec![0, 1], vec![0, 2], vec![1, 2]] { + let mut want = [0u32; 4]; + for &q in &queries { + for (w, p) in want.iter_mut().zip(own(q)) { + *w = (*w).max(p); + } + } + let want = SketchParams::UnivMon { + heap_size: want[0], + sketch_rows: want[1], + sketch_cols: want[2], + layers: want[3] as u8, + // Ignored by equality. + distinct_failure_probability: None, + }; for c in &run.logical { if let Some(params) = shared_univmon(c, &queries) { seen += 1; - assert_eq!(params, own_univmon(&run, strictest(&queries)), "{}", c.id); + assert_eq!(params, want, "{} {queries:?}", c.id); } } } @@ -475,7 +492,6 @@ fn stage2_physical_asap_optimization_keeps_every_logical_candidate() { /// Physical ASAP-aware optimization keeps the shared UnivMon as one build node read by all three queries. #[test] -#[ignore = "needs the Pass 2 summary-capability rule for frequency moments (#595, #596)"] fn stage2_physical_asap_optimization_keeps_the_shared_univmon_as_one_build() { let run = pipeline(); let mut seen = 0; @@ -743,7 +759,7 @@ fn stage3_plan_selection_accepts_univmon_l2() { /// One materialized UnivMon for all three costs no more than three separate UnivMons. #[test] -#[ignore = "needs one UnivMon for all three (#595, #596), a UnivMon accuracy model (#621) and materializing it (#604): not materialized, it is charged once per consuming query"] +#[ignore = "needs a UnivMon entropy accuracy model and materializing the shared UnivMon (#604): not materialized, it is charged once per consuming query"] fn stage3_plan_selection_shared_univmon_costs_no_more_than_three() { let run = pipeline(); let cost = |c: &Logical| run.cost(&run.physical_of(c).next().unwrap().id); From 4c92b038f7d137e3d9f366b16ad2bb844417ddac Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 14 Oct 2026 23:47:31 +0000 Subject: [PATCH 03/14] feat(frontend-sql): recognize the SQL frequency L2 and entropy idioms #509 Example 2 writes L2(src_ip) and Entropy(src_ip) as SQL idioms over grouped unit counts. The SQL lowering (single query and batch) now names them at the query root as FrequencyL2 / FrequencyEntropy aggregates carrying the query's accuracy target, so Pass 1 offers exact and summary candidates for each: - L2: SQRT(SUM(c * c)) over COUNT(*) per key, only when the product is already Float64 (CAST(c AS DOUBLE) factors or CAST(COUNT(*) AS DOUBLE)). The Int64 product stays relational: SQL integer overflow is observable. - Entropy: -SUM(p * LN(p)) with p = COUNT(*)[*1.0] / SUM(COUNT(*)) OVER () over the complete unordered frame; the IR's bits are converted to nats and SQL's -0.0 is kept. Both require a single non-nullable Bool/Int64/Utf8 key, no HAVING and no measure filter, follow projections only, keep the original output schema, and decide SQL's empty-input NULL by an exact population count joined beside the statistic, never by an estimate. Scaled probabilities, LOG2, partitioned or ordered windows and nullable keys stay relational. Ports the frontend part of #598 (originally #562 and #564). --- crates/frontend-sql/src/frequency.rs | 403 ++++++++++++++++++ crates/frontend-sql/src/lib.rs | 5 +- .../frontend-sql/tests/frequency_entropy.rs | 95 +++++ crates/frontend-sql/tests/frequency_l2.rs | 225 ++++++++++ 4 files changed, 727 insertions(+), 1 deletion(-) create mode 100644 crates/frontend-sql/src/frequency.rs create mode 100644 crates/frontend-sql/tests/frequency_entropy.rs create mode 100644 crates/frontend-sql/tests/frequency_l2.rs diff --git a/crates/frontend-sql/src/frequency.rs b/crates/frontend-sql/src/frequency.rs new file mode 100644 index 000000000..fe6334178 --- /dev/null +++ b/crates/frontend-sql/src/frequency.rs @@ -0,0 +1,403 @@ +//! Recognition of the SQL frequency idioms at the query root. +//! +//! #509 Example 2 writes `L2(x)` and `Entropy(x)` as SQL idioms over grouped +//! unit counts. [`recognize_frequency_idioms`] names them as +//! `FrequencyL2` / `FrequencyEntropy` intents, so Pass 1 can offer exact and +//! summary candidates (UnivMon among them) for each. It preserves the +//! output schema, the predicates below the grouping, the query's accuracy +//! target, SQL's NULL result on empty input (decided by an exact population +//! count, never by an estimate) and the entropy unit: the IR's entropy is in +//! bits, the SQL idiom's `LN` gives nats. Integer products and nullable +//! keys are refused, because SQL integer overflow and NULL groups are +//! observable; such queries stay relational. +use asap_types::ir::non_asap::any_measure_filtered; +use asap_types::ir::{ + ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, ProjectItem, ScalarExpr, +}; +use asap_types::pre_asap::{ + AggIntent, ArithmeticOpKind, CompareOpKind, DataType, GroupKeys, JoinKind, Reduction, + ScalarValue, WindowFrameBound, WindowFrameOffset, WindowFuncKind, +}; +use asap_types::types::{AccuracyTarget, ErrorMetric}; +use std::rc::Rc; + +/// The recognized form of a query whose root is the L2 or natural-log entropy +/// idiom over grouped unit counts, or `None` when neither rule applies. +pub fn recognize_frequency_idioms(root: &Rc) -> Option> { + frequency_entropy_rewrite(root).or_else(|| frequency_l2_rewrite(root)) +} + +/// `expr` over `node`, followed down through projections only: crossing a +/// filter or a limit would change the population. +fn expand( + mut expr: ScalarExpr, + mut node: Rc, +) -> Option<(ScalarExpr, Rc)> { + while let Some(NonASAPOp::Project { cols, child, .. }) = node.non_asap() { + expr = substitute(&expr, cols)?; + node = Rc::clone(child); + } + Some((expr, node)) +} +fn substitute(expr: &ScalarExpr, cols: &[ProjectItem]) -> Option { + Some(match expr { + ScalarExpr::Column(id) => cols.get(*id)?.expr.clone(), + ScalarExpr::Literal(_) => expr.clone(), + ScalarExpr::Negative { expr, semantics } => ScalarExpr::Negative { + expr: Box::new(substitute(expr, cols)?), + semantics: *semantics, + }, + ScalarExpr::Cast { + expr, + to, + try_cast: false, + } if *to == DataType::Float64 => ScalarExpr::Cast { + expr: Box::new(substitute(expr, cols)?), + to: to.clone(), + try_cast: false, + }, + ScalarExpr::Arithmetic { + op, + left, + right, + semantics, + } => ScalarExpr::Arithmetic { + op: op.clone(), + left: Box::new(substitute(left, cols)?), + right: Box::new(substitute(right, cols)?), + semantics: *semantics, + }, + ScalarExpr::FunctionCall { name, args } => ScalarExpr::FunctionCall { + name: name.clone(), + args: args + .iter() + .map(|arg| substitute(arg, cols)) + .collect::>()?, + }, + _ => return None, + }) +} +fn uncast(expr: &ScalarExpr) -> &ScalarExpr { + match expr { + ScalarExpr::Cast { + expr, + to: DataType::Float64, + try_cast: false, + } => uncast(expr), + _ => expr, + } +} + +fn frequency_l2_rewrite(root: &Rc) -> Option> { + let NonASAPOp::Project { cols, child, .. } = root.non_asap()? else { + return None; + }; + let [item] = cols.as_slice() else { + return None; + }; + let (expr, outer) = expand(item.expr.clone(), Rc::clone(child))?; + let ScalarExpr::FunctionCall { name, args } = &expr else { + return None; + }; + if !name.eq_ignore_ascii_case("sqrt") { + return None; + } + let [arg] = args.as_slice() else { + return None; + }; + if !matches!(uncast(arg), ScalarExpr::Column(0)) { + return None; + } + let NonASAPOp::Aggregate { + reduction: Reduction::Reduce(keys), + measures, + filters, + having: None, + child, + .. + } = outer.non_asap()? + else { + return None; + }; + if keys.is_without() || !keys.keys().is_empty() || any_measure_filtered(filters) { + return None; + } + let [AggIntent::Sum { col: Some(col) }] = measures.as_slice() else { + return None; + }; + let (product, inner) = expand(ScalarExpr::Column(*col), Rc::clone(child))?; + let ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Mul, + left, + right, + semantics: ExprSemantics::Sql, + } = &product + else { + return None; + }; + // SQL Int64 multiplication can overflow. Admit only products already typed Float64. + if product.scalar_type(&inner.schema).ok()?.0 != DataType::Float64 { + return None; + } + if !matches!( + (uncast(left), uncast(right)), + (ScalarExpr::Column(1), ScalarExpr::Column(1)) + ) { + return None; + } + let (key, accuracy, input) = grouped_unit_count(&inner)?; + sql_frequency_result( + root, + input, + AggIntent::FrequencyL2 { + col: Some(key), + // The count's ε and δ, stated as the norm's relative error. + accuracy: accuracy.with_metric(ErrorMetric::Relative), + }, + "frequency_l2", + ScalarExpr::Column(0), + ) +} + +// Both frequency rules require a complete, unfiltered count per non-NULL identity. +fn grouped_unit_count( + node: &Rc, +) -> Option<(usize, AccuracyTarget, Rc)> { + let NonASAPOp::Aggregate { + reduction: Reduction::Reduce(keys), + measures, + filters, + having: None, + child, + .. + } = node.non_asap()? + else { + return None; + }; + let [key] = keys.keys() else { + return None; + }; + let [AggIntent::Count { accuracy }] = measures.as_slice() else { + return None; + }; + let field = child.schema.fields.get(*key)?; + if keys.is_without() + || any_measure_filtered(filters) + || field.nullable + || !matches!( + field.plain_dtype()?, + DataType::Bool | DataType::Int64 | DataType::Utf8 + ) + { + return None; + } + Some((*key, accuracy.clone(), Rc::clone(child))) +} + +fn probability_term(product: &ScalarExpr) -> Option<&ScalarExpr> { + let ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Mul, + left, + right, + semantics: ExprSemantics::Sql, + } = product + else { + return None; + }; + for (probability, logarithm) in [(left, right), (right, left)] { + let ScalarExpr::FunctionCall { name, args } = logarithm.as_ref() else { + continue; + }; + if name.eq_ignore_ascii_case("ln") && args.as_slice() == [probability.as_ref().clone()] { + return Some(probability); + } + } + None +} + +fn unit_count_term(expr: &ScalarExpr) -> bool { + match uncast(expr) { + ScalarExpr::Column(1) => true, + ScalarExpr::Arithmetic { op: ArithmeticOpKind::Mul, left, right, semantics: ExprSemantics::Sql } => { + [(left, right), (right, left)].into_iter().any(|(count, scale)| matches!(uncast(count), ScalarExpr::Column(1)) && matches!(scale.as_ref(), ScalarExpr::Literal(ScalarValue::Float64(value)) if *value == 1.0)) + }, + _ => false, + } +} + +fn frequency_entropy_rewrite(root: &Rc) -> Option> { + let NonASAPOp::Project { cols, child, .. } = root.non_asap()? else { + return None; + }; + let [item] = cols.as_slice() else { + return None; + }; + let (expr, outer) = expand(item.expr.clone(), Rc::clone(child))?; + let ScalarExpr::Negative { + expr, + semantics: ExprSemantics::Sql, + } = expr + else { + return None; + }; + if !matches!(uncast(&expr), ScalarExpr::Column(0)) { + return None; + } + let NonASAPOp::Aggregate { + reduction: Reduction::Reduce(keys), + measures, + filters, + having: None, + child, + .. + } = outer.non_asap()? + else { + return None; + }; + if keys.is_without() || !keys.keys().is_empty() || any_measure_filtered(filters) { + return None; + } + let [AggIntent::Sum { col: Some(col) }] = measures.as_slice() else { + return None; + }; + let (product, window) = expand(ScalarExpr::Column(*col), Rc::clone(child))?; + let probability = probability_term(&product)?; + let ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Div, + left, + right, + semantics: ExprSemantics::Sql, + } = probability + else { + return None; + }; + if !unit_count_term(left) + || !matches!(uncast(right), ScalarExpr::Column(2)) + || probability.scalar_type(&window.schema).ok()?.0 != DataType::Float64 + { + return None; + } + let NonASAPOp::SQLWindowFunc { + func: WindowFuncKind::Sum, + args, + partition_by, + order_by, + frame: Some(frame), + child, + .. + } = window.non_asap()? + else { + return None; + }; + if partition_by.is_without() + || !partition_by.keys().is_empty() + || !order_by.is_empty() + || args.as_slice() != [ScalarExpr::Column(1)] + { + return None; + } + if !matches!( + frame.start_bound, + WindowFrameBound::Preceding(WindowFrameOffset::Scalar(ScalarValue::Null)) + ) || !matches!( + frame.end_bound, + WindowFrameBound::Following(WindowFrameOffset::Scalar(ScalarValue::Null)) + ) { + return None; + } + let (key, accuracy, input) = grouped_unit_count(child)?; + let nats = ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Mul, + left: Box::new(ScalarExpr::Column(0)), + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64( + std::f64::consts::LN_2, + ))), + semantics: ExprSemantics::Sql, + }; + // -SUM(p*LN(p)) is negative zero for a single-identity population. + let nats = ScalarExpr::Negative { + expr: Box::new(ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Sub, + left: Box::new(ScalarExpr::Literal(ScalarValue::Float64(0.0))), + right: Box::new(nats), + semantics: ExprSemantics::Sql, + }), + semantics: ExprSemantics::Sql, + }; + sql_frequency_result( + root, + input, + AggIntent::FrequencyEntropy { + col: Some(key), + // The count's ε and δ, stated as an absolute error in bits. + accuracy: accuracy.with_metric(ErrorMetric::ABSOLUTE), + }, + "frequency_entropy", + nats, + ) +} + +// Both rules use an exact population guard: an approximate statistic may be +// zero even for nonempty input, and must not control SQL's NULL result. +fn sql_frequency_result( + root: &Rc, + input: Rc, + measure: AggIntent, + name: &str, + value: ScalarExpr, +) -> Option> { + let NonASAPOp::Project { qualifier, .. } = root.non_asap()? else { + return None; + }; + let aggregate = |measure, name: &str| { + OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::Reduce(GroupKeys::none()), + measures: vec![measure], + output_names: vec![name.into()], + filters: vec![], + having: None, + child: Rc::clone(&input), + })) + .ok() + }; + let statistic = aggregate(measure, name)?; + let count = aggregate( + AggIntent::Count { + accuracy: AccuracyTarget::exact(), + }, + "population_count", + )?; + let child = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Join { + kind: JoinKind::Cross, + pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), + left: statistic, + right: count, + })) + .ok()?; + let rewritten = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { + cols: vec![ProjectItem { + alias: Some(root.schema.fields.first()?.name.clone()), + expr: ScalarExpr::Case { + operand: None, + branches: vec![( + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(1)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(0))), + semantics: ExprSemantics::Sql, + }, + ScalarExpr::Cast { + expr: Box::new(ScalarExpr::Literal(ScalarValue::Null)), + to: DataType::Float64, + try_cast: false, + }, + )], + else_expr: Some(Box::new(value)), + }, + }], + qualifier: qualifier.clone(), + child, + })) + .ok()?; + (root.schema == rewritten.schema).then_some(rewritten) +} diff --git a/crates/frontend-sql/src/lib.rs b/crates/frontend-sql/src/lib.rs index e65a25f0a..d2c40dade 100644 --- a/crates/frontend-sql/src/lib.rs +++ b/crates/frontend-sql/src/lib.rs @@ -9,6 +9,7 @@ //! Depends on DataFusion only — never on the PromQL parser. pub mod error; +pub mod frequency; pub mod sql; use std::rc::Rc; @@ -55,7 +56,9 @@ pub async fn lower_sql_dialect( // Binding resolves names and derives every node's schema; result-type // checks (such as temporal subtraction, whose duration unit the IR cannot // represent) surface here as `ResolveDAGError::Schema`. - Ok(resolve_root(&unresolved)?) + let root = resolve_root(&unresolved)?; + // #509 Example 2: the L2 and entropy idioms become frequency intents. + Ok(frequency::recognize_frequency_idioms(&root).unwrap_or(root)) } /// Lower every SQL batch entry in `workload` to an operator DAG. diff --git a/crates/frontend-sql/tests/frequency_entropy.rs b/crates/frontend-sql/tests/frequency_entropy.rs new file mode 100644 index 000000000..4fa911e4d --- /dev/null +++ b/crates/frontend-sql/tests/frequency_entropy.rs @@ -0,0 +1,95 @@ +//! The SQL frontend names the natural-log entropy idiom as a +//! `FrequencyEntropy` intent (#509 Example 2's Q2). +use asap_frontend_common::resolve_root; +use asap_frontend_sql::{lower_sql, SqlCatalog, SqlLowerer}; +use asap_types::{ + ir::{NonASAPOp, OperatorNode}, + pre_asap::{AggIntent, DataType, Field, Schema}, + types::{ErrorMetric, RequestedAccuracy}, +}; + +fn catalog(nullable: bool) -> SqlCatalog { + SqlCatalog::new().with_table( + "flows", + Schema::new(vec![Field::plain("src_ip", DataType::Utf8, nullable)]), + ) +} + +fn has_entropy(node: &OperatorNode) -> bool { + if let Some(NonASAPOp::Aggregate { measures, .. }) = node.non_asap() { + if measures + .iter() + .any(|m| matches!(m, AggIntent::FrequencyEntropy { .. })) + { + return true; + } + } + node.children().iter().any(|child| has_entropy(child)) +} + +/// The natural-log idiom lowers to an entropy intent with the original +/// output schema; the relational form before recognition has none. +#[tokio::test] +async fn recognizes_sql_entropy_in_nats() { + let sql = "SELECT -SUM(p * LN(p)) AS entropy FROM (SELECT COUNT(*) * 1.0 / SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip) f"; + let rewritten = lower_sql(sql, &catalog(false), RequestedAccuracy::exact()) + .await + .unwrap(); + let original = resolve_root( + &SqlLowerer::new(&catalog(false)) + .lower(sql, &RequestedAccuracy::exact()) + .await + .unwrap(), + ) + .unwrap(); + assert!(has_entropy(&rewritten)); + assert_eq!(original.schema, rewritten.schema); + assert!(!has_entropy(&original)); +} + +/// Other units, normalizations, window frames or counted populations are +/// not entropy: scaled probabilities, `LOG2`, partitioned or ordered +/// windows, `HAVING` and a nullable key stay relational. +#[tokio::test] +async fn declines_non_equivalent_entropy_shapes() { + for (sql, nullable) in [ + ("SELECT -SUM(p*LN(p)) FROM (SELECT COUNT(*)*2.0/SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip) f", false), + ("SELECT -SUM(p*LOG2(p)) FROM (SELECT COUNT(*)*1.0/SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip) f", false), + ("SELECT -SUM(p*LN(p)) FROM (SELECT COUNT(*)*1.0/SUM(COUNT(*)) OVER (PARTITION BY src_ip) AS p FROM flows GROUP BY src_ip) f", false), + ("SELECT -SUM(p*LN(p)) FROM (SELECT COUNT(*)*1.0/SUM(COUNT(*)) OVER (ORDER BY src_ip) AS p FROM flows GROUP BY src_ip) f", false), + ("SELECT -SUM(p*LN(p)) FROM (SELECT COUNT(*)*1.0/SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip HAVING COUNT(*) > 1) f", false), + ("SELECT -SUM(p*LN(p)) FROM (SELECT COUNT(*)*1.0/SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip) f", true), + ] { + let root = lower_sql(sql, &catalog(nullable), RequestedAccuracy::exact()) + .await + .unwrap(); + assert!(!has_entropy(&root), "{sql}"); + } +} + +/// Entropy carries the query's accuracy target; its empty-population guard +/// is an exact count. +#[tokio::test] +async fn entropy_accuracy_and_population_guard_are_separate() { + let target = RequestedAccuracy::epsilon_delta(0.05, 0.01); + let node = lower_sql("SELECT -SUM(p*LN(p)) FROM (SELECT COUNT(*)*1.0/SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip) f", &catalog(false), target.clone()).await.unwrap(); + let NonASAPOp::Project { child, .. } = node.expect_non_asap() else { + panic!("project"); + }; + let NonASAPOp::Join { left, right, .. } = child.expect_non_asap() else { + panic!("guarded statistic"); + }; + let NonASAPOp::Aggregate { measures, .. } = left.expect_non_asap() else { + panic!("entropy"); + }; + assert!( + matches!(&measures[0], AggIntent::FrequencyEntropy { accuracy, .. } if *accuracy == target.target_in(ErrorMetric::ABSOLUTE)) + ); + let NonASAPOp::Aggregate { measures, .. } = right.expect_non_asap() else { + panic!("count"); + }; + assert!(matches!( + measures.as_slice(), + [AggIntent::Count { accuracy }] if accuracy.is_exact() + )); +} diff --git a/crates/frontend-sql/tests/frequency_l2.rs b/crates/frontend-sql/tests/frequency_l2.rs new file mode 100644 index 000000000..153dab4ba --- /dev/null +++ b/crates/frontend-sql/tests/frequency_l2.rs @@ -0,0 +1,225 @@ +//! The SQL frontend names the frequency L2 idiom as a `FrequencyL2` intent +//! (#509 Example 2's Q3, in its Float64 form). +use asap_frontend_common::resolve_root; +use asap_frontend_sql::{lower_sql, lower_sql_batch, SqlCatalog, SqlLowerer}; +use asap_types::{ + ir::{NonASAPOp, OperatorNode}, + pre_asap::{AggIntent, DataType, Field, Schema}, + types::{ErrorMetric, RequestedAccuracy}, +}; +use std::rc::Rc; + +fn catalog(nullable: bool) -> SqlCatalog { + SqlCatalog::new().with_table( + "flows", + Schema::new(vec![ + Field::plain("src_ip", DataType::Utf8, nullable), + Field::plain("keep", DataType::Bool, false), + ]), + ) +} + +fn has_l2(node: &OperatorNode) -> bool { + if let Some(NonASAPOp::Aggregate { measures, .. }) = node.non_asap() { + if measures + .iter() + .any(|m| matches!(m, AggIntent::FrequencyL2 { .. })) + { + return true; + } + } + node.children().iter().any(|child| has_l2(child)) +} + +/// The relational DAG the frontend lowers before frequency recognition. +async fn relational(sql: &str, catalog: &SqlCatalog) -> Rc { + resolve_root( + &SqlLowerer::new(catalog) + .lower(sql, &RequestedAccuracy::exact()) + .await + .unwrap(), + ) + .unwrap() +} + +const CAST_PRODUCT: &str = "SELECT SQRT(SUM(CAST(c AS DOUBLE) * CAST(c AS DOUBLE))) AS norm FROM (SELECT src_ip, COUNT(*) AS c FROM flows WHERE keep GROUP BY src_ip) f"; +const CAST_COUNT: &str = "SELECT SQRT(SUM(c * c)) AS norm FROM (SELECT src_ip, CAST(COUNT(*) AS DOUBLE) AS c FROM flows GROUP BY src_ip) f"; + +/// Float64 count products lower to an L2 intent that keeps the alias, the +/// filter, the type and the empty-input nullability of the original. +#[tokio::test] +async fn recognizes_float_frequency_l2() { + for sql in [CAST_PRODUCT, CAST_COUNT] { + let rewritten = lower_sql(sql, &catalog(false), RequestedAccuracy::exact()) + .await + .unwrap(); + let original = relational(sql, &catalog(false)).await; + assert!(has_l2(&rewritten), "{sql}"); + assert_eq!(original.schema, rewritten.schema); + assert!(!has_l2(&original)); + } +} + +/// Nearby forms with other overflow, NULL or frequency semantics stay +/// relational: the Int64 product (SQL integer overflow is observable), a +/// nullable key, `HAVING`, and a sum that is not a unit count. +#[tokio::test] +async fn declines_non_equivalent_frequency_shapes() { + for (sql, nullable) in [ + ("SELECT SQRT(SUM(c*c)) FROM (SELECT src_ip, COUNT(*) AS c FROM flows GROUP BY src_ip) f", false), + ("SELECT SQRT(SUM(CAST(c AS DOUBLE)*CAST(c AS DOUBLE))) FROM (SELECT src_ip, COUNT(*) AS c FROM flows GROUP BY src_ip) f", true), + ("SELECT SQRT(SUM(c*c)) FROM (SELECT src_ip, CAST(COUNT(*) AS DOUBLE) AS c FROM flows GROUP BY src_ip HAVING COUNT(*) > 1) f", false), + ("SELECT SQRT(SUM(c*c)) FROM (SELECT src_ip, CAST(SUM(CAST(keep AS BIGINT)) AS DOUBLE) AS c FROM flows GROUP BY src_ip) f", false), + ] { + let root = lower_sql(sql, &catalog(nullable), RequestedAccuracy::exact()) + .await + .unwrap(); + assert!(!has_l2(&root), "{sql}"); + } +} + +/// The L2 intent carries the query's accuracy target, not a default. +#[tokio::test] +async fn frequency_l2_preserves_accuracy_target() { + let target = RequestedAccuracy::epsilon_delta(0.01, 0.01); + let node = lower_sql(CAST_COUNT, &catalog(false), target.clone()) + .await + .unwrap(); + let NonASAPOp::Project { child, .. } = node.expect_non_asap() else { + panic!("project"); + }; + let NonASAPOp::Join { left, .. } = child.expect_non_asap() else { + panic!("guarded statistic"); + }; + let NonASAPOp::Aggregate { measures, .. } = left.expect_non_asap() else { + panic!("aggregate"); + }; + assert!( + matches!(&measures[0], AggIntent::FrequencyL2 { accuracy, .. } if *accuracy == target.target_in(ErrorMetric::Relative)) + ); +} + +/// An approximate L2 estimate never decides whether SQL returns NULL: the +/// empty-input guard is an exact count. +#[tokio::test] +async fn frequency_empty_input_guard_uses_an_exact_count() { + let target = RequestedAccuracy::epsilon_delta(0.01, 0.01); + let node = lower_sql(CAST_COUNT, &catalog(false), target) + .await + .unwrap(); + let NonASAPOp::Project { child, .. } = node.expect_non_asap() else { + panic!("project"); + }; + let NonASAPOp::Join { right, .. } = child.expect_non_asap() else { + panic!("exact population guard"); + }; + let NonASAPOp::Aggregate { measures, .. } = right.expect_non_asap() else { + panic!("count"); + }; + assert!(matches!( + measures.as_slice(), + [AggIntent::Count { accuracy }] if accuracy.is_exact() + )); +} + +/// The batch path recognizes the idiom as the single-query path does. +#[tokio::test] +async fn batch_lowering_recognizes_frequency_l2() { + use asap_types::workload::{ + AccuracyRequirement, BatchEntry, LatencyRequirement, Predictability, Query, QueryLanguage, + QueryRequirements, QueryWorkload, SqlDialect, TimeSelection, + }; + let workload = QueryWorkload { + language: QueryLanguage::SQL(SqlDialect::DataFusionSQL), + query_batch: Some(vec![BatchEntry { + query: Query(CAST_COUNT.into()), + requirements: QueryRequirements { + accuracy: AccuracyRequirement::Explicit(RequestedAccuracy::exact()), + response_latency: LatencyRequirement::Unspecified, + }, + predictability: Predictability::Unknown, + invocations: 1, + execute_at: None, + time_selection: TimeSelection::default(), + }]), + repeating_queries: None, + }; + let roots = lower_sql_batch(&workload, &catalog(false)).await; + assert!(has_l2(roots[0].as_ref().unwrap())); +} + +/// Stage 1 offers the exact option and summary candidates for the +/// recognized L2 intent. +#[tokio::test] +async fn stage1_offers_exact_and_summary_l2_candidates() { + use asap_aware_mapping::per_computation_candidates::{ + generate_per_computation_candidates, ComputationCandidate, + }; + use asap_types::ir::QueryRoot; + let target = RequestedAccuracy::epsilon_delta(0.01, 0.01); + let root = lower_sql(CAST_COUNT, &catalog(false), target) + .await + .unwrap(); + let inventory = + generate_per_computation_candidates(vec![(0, QueryRoot::Operator(root))]).unwrap(); + let l2 = inventory + .targets + .iter() + .find(|t| has_l2(&t.target)) + .expect("L2 target"); + let candidates = &l2.choices.candidates; + assert!( + candidates.iter().any(|c| matches!( + c, + ComputationCandidate::Exact | ComputationCandidate::ExactAccumulator { .. } + )), + "{candidates:?}" + ); + assert!( + candidates + .iter() + .any(|c| matches!(c, ComputationCandidate::Summary(_))), + "{candidates:?}" + ); +} + +/// #509 Example 2 over `flows(ts, src_ip)` with its one-minute window: Q2 +/// lowers to entropy and the Float64 Q3 to L2; the design's literal Int64 +/// Q3 stays relational. +#[tokio::test] +async fn example2_queries_lower_to_frequency_intents() { + let catalog = SqlCatalog::new().with_table( + "flows", + Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("src_ip", DataType::Utf8, false), + ], + 0, + vec![], + ), + ); + const WINDOW: &str = "WHERE ts >= now() - INTERVAL '1 minute'"; + let q2 = format!("SELECT -SUM(p * LN(p)) FROM (SELECT COUNT(*) * 1.0 / SUM(COUNT(*)) OVER () AS p FROM flows {WINDOW} GROUP BY src_ip)"); + let q3_float = format!("SELECT SQRT(SUM(CAST(c AS DOUBLE) * CAST(c AS DOUBLE))) FROM (SELECT src_ip, COUNT(*) AS c FROM flows {WINDOW} GROUP BY src_ip)"); + let q3_int = format!("SELECT SQRT(SUM(c * c)) FROM (SELECT src_ip, COUNT(*) AS c FROM flows {WINDOW} GROUP BY src_ip)"); + let lower = |sql: String| { + let catalog = &catalog; + async move { + lower_sql(&sql, catalog, RequestedAccuracy::exact()) + .await + .unwrap() + } + }; + let q2 = lower(q2).await; + assert!( + q2.children().iter().any(|join| join.children().iter().any( + |aggregate| matches!(aggregate.non_asap(), Some(NonASAPOp::Aggregate { measures, .. }) + if matches!(measures[..], [AggIntent::FrequencyEntropy { col: Some(1), .. }])) + )), + "{q2:#?}" + ); + let q3_float = lower(q3_float).await; + assert!(has_l2(&q3_float)); + assert!(!has_l2(&*lower(q3_int).await)); +} From 0d08cc25f6b3233a9c3079d963e4fc475f919e06 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 15 Oct 2026 00:04:24 +0000 Subject: [PATCH 04/14] feat(physical-operators): exact distinct, L2 and entropy reducers Exact Cardinality, FrequencyL2 and FrequencyEntropy aggregates had no native implementation, so the exact option of #509 Example 2's three statistics could not execute. - Reduction::Cardinality(cols) counts distinct tuples; a tuple with any NULL is skipped, and grouped key encoding keeps tuple boundaries and folds signed zero and NaN payloads. - Reduction::FrequencyL2(col) and FrequencyEntropy(col) count the unit frequencies of one Bool/Int64/Float64/Utf8 identity column: NULL is skipped, a non-finite float is an error, an empty population gives 0.0, and entropy is in bits, as AggIntent::FrequencyEntropy and UnivMon's readout define it. - The dictionaries are memory-accounted and cooperative. - Logical aggregates bind them: grouped ones (SQL COUNT(DISTINCT)) and per-series range reductions (distinct_over_time, l2_over_time, entropy_over_time), which reduce each series' window samples, restore the run's timestamp and drop the metric name as range functions do. - The analytical cost lowering prices them as hash aggregates. Ports the exact-reducer part of #597 (from #559 and #563). --- .../src/query_physical_lowering.rs | 75 ++++++ .../src/operators/aggregate/mod.rs | 93 ++++++- .../src/physical_planner/mod.rs | 97 +++++++ .../tests/blocking_resources.rs | 46 ++++ .../tests/physical_semantics.rs | 239 ++++++++++++++++++ .../tests/physical_common/mod.rs | 57 +++++ .../tests/sql_cardinality.rs | 48 ++++ 7 files changed, 654 insertions(+), 1 deletion(-) create mode 100644 crates/integration-tests/tests/sql_cardinality.rs diff --git a/crates/asap-aware-mapping/src/query_physical_lowering.rs b/crates/asap-aware-mapping/src/query_physical_lowering.rs index bb7f914e5..49bdc6e91 100644 --- a/crates/asap-aware-mapping/src/query_physical_lowering.rs +++ b/crates/asap-aware-mapping/src/query_physical_lowering.rs @@ -1252,6 +1252,9 @@ fn supports_hash_aggregate( | AggIntent::PearsonCorr { .. } | AggIntent::Group | AggIntent::CountValues { .. } + | AggIntent::Cardinality { .. } + | AggIntent::FrequencyL2 { .. } + | AggIntent::FrequencyEntropy { .. } ) }) } @@ -1555,6 +1558,78 @@ mod tests { assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).is_ok()); } + // The exact distinct count, L2 norm and entropy are hash aggregates: the + // executor runs them as native reducers over one identity column. + #[test] + fn exact_frequency_intents_lower_to_physical_hash_aggregates() { + use asap_types::pre_asap::{AggIntent, DataType, Field, Reduction, Schema, Source}; + use asap_types::types::AccuracyTarget; + let source = Source::Table { + table_ref: "flows".into(), + }; + for measure in [ + AggIntent::Cardinality { + cols: vec![0], + accuracy: AccuracyTarget::exact(), + }, + AggIntent::FrequencyL2 { + col: Some(0), + accuracy: AccuracyTarget::exact(), + }, + AggIntent::FrequencyEntropy { + col: Some(0), + accuracy: AccuracyTarget::exact(), + }, + ] { + let root = + OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![]), + measures: vec![measure.clone()], + output_names: vec!["v".into()], + filters: vec![], + having: None, + child: OperatorNode::new_shared(asap_types::ir::Operator::NonASAP( + NonASAPOp::Scan { + source: source.clone(), + predicates: vec![], + schema: Schema::new(vec![Field::plain( + "src_ip", + DataType::Utf8, + false, + )]), + }, + )) + .unwrap(), + })) + .unwrap(); + let scope = scope(vec![coverage(source.clone(), vec![])]); + let provided = HashMap::from([ + ( + "query-1".into(), + evidence(scan_stats(edge(100, 1600), 1600)), + ), + ( + "query-0".into(), + evidence(OperatorStatistics::HashAggregate { + edges: unary_edges(edge(100, 1600), edge(1, 8)), + group_count: 1, + key_bytes: 0, + accumulator_bytes_per_group: 48, + }), + ), + ]); + let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)) + .unwrap_or_else(|e| panic!("{measure:?}: {e:?}")); + assert!(matches!( + dag.nodes.last().unwrap().operator, + PhysicalOperator::HashAggregate { + grouping_key_count: 0, + accumulator_count: 1, + } + )); + } + } + #[test] fn query_lowering_recurses_and_fuses_global_sort_limit() { use asap_types::pre_asap::{AggIntent, GroupKeys, Reduction, Source}; diff --git a/crates/asap-physical-operators/src/operators/aggregate/mod.rs b/crates/asap-physical-operators/src/operators/aggregate/mod.rs index 0ef4c6f7d..8f69dba57 100644 --- a/crates/asap-physical-operators/src/operators/aggregate/mod.rs +++ b/crates/asap-physical-operators/src/operators/aggregate/mod.rs @@ -13,6 +13,28 @@ impl Operator { for (name, reduction) in &measures { let (t, n) = match reduction { Reduction::Count => (DataType::Int64, false), + Reduction::Cardinality(columns) => { + if columns.is_empty() { + return Err(invalid( + "distinct aggregate requires at least one identity column", + )); + } + for column in columns { + plain(&input, *column)?; + } + (DataType::Int64, false) + } + Reduction::FrequencyL2(i) | Reduction::FrequencyEntropy(i) => { + if !matches!( + plain(&input, *i)?.0, + DataType::Bool | DataType::Int64 | DataType::Float64 | DataType::Utf8 + ) { + return Err(invalid( + "frequency aggregate requires a Boolean, Int64, Float64 or Utf8 identity", + )); + } + (DataType::Float64, false) + } Reduction::Sum(i) | Reduction::Avg(i) => { let (t, _) = plain(&input, *i)?; if !matches!(t, DataType::Int64 | DataType::Float64) { @@ -170,6 +192,14 @@ impl Operator { #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub enum Reduction { Count, + /// Exact distinct tuple count; a tuple with any NULL component is skipped. + Cardinality(Vec), + /// L2 norm of the unit-update frequencies of one identity column; NULL + /// identities are skipped. + FrequencyL2(usize), + /// Shannon entropy in bits of the frequencies of one identity column; + /// NULL identities are skipped. + FrequencyEntropy(usize), Sum(usize), Avg(usize), Min(usize), @@ -276,7 +306,7 @@ async fn reduce( selected.push(row); } } - result.push(reduce_one(&selected, measure, input, &mut work).await?); + result.push(reduce_one(&selected, measure, input, &mut work, context).await?); } workspace.grow(row_bytes(&result))?; output.push(result); @@ -314,6 +344,7 @@ async fn reduce_one( measure: &Reduction, input: &SchemaRef, work: &mut Cooperative, + context: &RunContext, ) -> Result { let column = match measure { Reduction::Count => { @@ -322,6 +353,66 @@ async fn reduce_one( )) } Reduction::Sum(i) | Reduction::Avg(i) | Reduction::Min(i) | Reduction::Max(i) => *i, + Reduction::Cardinality(columns) => { + let mut workspace = Workspace::new(context)?; + let mut identities = std::collections::BTreeSet::new(); + for row in rows { + work.checkpoint().await?; + if columns + .iter() + .any(|column| matches!(row[*column], Value::Null)) + { + continue; + } + let key = group_key(row, columns)?; + if !identities.contains(&key) { + workspace.grow(key_bytes(&key))?; + identities.insert(key); + } + } + return Ok(Value::Int64( + i64::try_from(identities.len()).map_err(|_| invalid("distinct count overflow"))?, + )); + } + Reduction::FrequencyL2(column) | Reduction::FrequencyEntropy(column) => { + let mut workspace = Workspace::new(context)?; + let mut counts = BTreeMap::, u64>::new(); + let mut total = 0_u64; + for row in rows { + work.checkpoint().await?; + let value = &row[*column]; + if matches!(value, Value::Null) { + continue; + } + if matches!(value, Value::Float64(v) if !v.is_finite()) { + return Err(invalid( + "frequency aggregate requires a finite floating identity", + )); + } + let key = value.key()?; + if !counts.contains_key(&key) { + workspace.grow( + 64 + std::mem::size_of::>() + + key.len() + + std::mem::size_of::(), + )?; + } + *counts.entry(key).or_default() += 1; + total += 1; + } + let mut result = 0.0_f64; + for count in counts.into_values() { + work.checkpoint().await?; + let count = count as f64; + if matches!(measure, Reduction::FrequencyL2(_)) { + result = result.hypot(count); + } else { + let probability = count / total as f64; + result -= probability * probability.log2(); + } + } + return Ok(Value::Float64(result)); + } Reduction::Quantile { column, q } => { let mut values = Vec::with_capacity(rows.len()); for row in rows { diff --git a/crates/asap-physical-operators/src/physical_planner/mod.rs b/crates/asap-physical-operators/src/physical_planner/mod.rs index 36fc2a247..eac2332f4 100644 --- a/crates/asap-physical-operators/src/physical_planner/mod.rs +++ b/crates/asap-physical-operators/src/physical_planner/mod.rs @@ -315,6 +315,90 @@ fn compile_internal( )?; continue; } + // An exact per-series range reduction with no exact accumulator + // form (`distinct_over_time`, `l2_over_time`, `entropy_over_time`) + // reduces each series' raw samples in the window natively. + if let Payload::NonASAP(NonASAPOp::Aggregate { + reduction: PlannerReduction::PerEntity, + measures, + filters, + having: None, + .. + }) = &node.payload + { + let [measure] = measures.as_slice() else { + return Err(invalid("per-entity aggregate has one measure")); + }; + if filters.iter().any(Option::is_some) { + return Err(invalid("filtered aggregate has no native implementation")); + } + let [input] = schemas.as_slice() else { + return Err(invalid("per-entity aggregate requires one input")); + }; + let value = match measure.input_cols().first() { + Some(column) => *column, + None => named_column(input, &ColumnRef::SampleValue)?, + }; + let reduction = match measure { + AggIntent::Cardinality { .. } => Reduction::Cardinality(vec![value]), + AggIntent::FrequencyL2 { .. } => Reduction::FrequencyL2(value), + AggIntent::FrequencyEntropy { .. } => Reduction::FrequencyEntropy(value), + _ => { + return Err(invalid(format!( + "node {id}: per-entity aggregate requires an explicit entity binding" + ))) + } + }; + let coordinate = input + .time_index + .ok_or_else(|| invalid("temporal input lacks time"))?; + let groups = (0..input.fields.len()) + .filter(|&column| column != value && column != coordinate) + .collect(); + let aggregate = + Operator::aggregate(input.clone(), groups, vec![("value".into(), reduction)])?; + // A PromQL sample is Float64; the distinct count is Int64. + let reduced = aggregate.schema(); + let project = Operator::project( + reduced.clone(), + reduced + .fields + .iter() + .enumerate() + .map(|(i, field)| { + let expression = if field.dtype == FieldDataType::Plain(DataType::Int64) + && i + 1 == reduced.fields.len() + { + Expression::ExactFloat64(i) + } else { + Expression::Column(i) + }; + (field.name.clone(), expression) + }) + .collect(), + )?; + let projected = project.schema(); + physical_dag.add(auxiliary, inputs, aggregate)?; + physical_dag.add(helper_id(id, 1)?, vec![auxiliary], project)?; + let stamped = Operator::scope_timestamp(projected, output.clone())?; + if output + .fields + .iter() + .any(|field| field.name == promql_rows::SERIES_IDENTITY_COLUMN) + { + // Range functions drop the metric name, as their + // summary evaluations do. + physical_dag.add(helper_id(id, 2)?, vec![helper_id(id, 1)?], stamped)?; + physical_dag.add( + id, + vec![helper_id(id, 2)?], + Operator::series_without_name(output)?, + )?; + } else { + physical_dag.add(id, vec![helper_id(id, 1)?], stamped)?; + } + continue; + } // A closed row must include either all source labels or the explicit // complete-label identity. Projected labels alone are insufficient. if let Payload::ASAP(ASAPOp::SummaryAgg { @@ -766,6 +850,19 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result Reduction::Count, + AggIntent::Cardinality { cols, .. } => { + Reduction::Cardinality(if cols.is_empty() { + vec![column(None)?] + } else { + cols.clone() + }) + } + AggIntent::FrequencyL2 { col, .. } => { + Reduction::FrequencyL2(column(*col)?) + } + AggIntent::FrequencyEntropy { col, .. } => { + Reduction::FrequencyEntropy(column(*col)?) + } AggIntent::Sum { col } => Reduction::Sum(column(*col)?), AggIntent::Avg { col } => Reduction::Avg(column(*col)?), AggIntent::Min { col } => Reduction::Min(column(*col)?), diff --git a/crates/asap-physical-operators/tests/blocking_resources.rs b/crates/asap-physical-operators/tests/blocking_resources.rs index 09cee7a4e..24bc1244b 100644 --- a/crates/asap-physical-operators/tests/blocking_resources.rs +++ b/crates/asap-physical-operators/tests/blocking_resources.rs @@ -248,3 +248,49 @@ fn weighted_summary_build_yields_within_a_batch() { drop(output); assert_eq!(run.retained_bytes(), 0); } + +// Frequency and distinct dictionaries count against the budget and release +// their reservations on failure. +#[test] +fn frequency_dictionary_enforces_memory_budget() { + use asap_physical_operators::operators::Reduction; + let input = schema(1); + let mut sources = PhysicalDAG::default(); + sources + .add( + 0, + vec![], + Operator::source( + input.clone(), + vec![Batch::try_new( + input.clone(), + (0..64).map(|i| vec![Value::Int64(i)]).collect(), + ) + .unwrap()], + ) + .unwrap(), + ) + .unwrap(); + for (reduction, succeeds) in [ + (Reduction::Count, true), + (Reduction::FrequencyL2(0), false), + (Reduction::FrequencyEntropy(0), false), + (Reduction::Cardinality(vec![0]), false), + ] { + // Count needs about 13.4 kB here, the dictionaries at least 14.5 kB. + let run = context(14_000); + let inputs = sources.execute(&[0], run.clone()).unwrap(); + let operator = + Operator::aggregate(input.clone(), vec![], vec![("result".into(), reduction)]).unwrap(); + let mut output = operator.start(inputs, run.clone()).unwrap(); + let result = block_on(output.next()).unwrap(); + if succeeds { + assert!(result.is_ok(), "{result:?}"); + } else { + assert!(matches!(result, Err(Error::MemoryLimit)), "{result:?}"); + } + drop(result); + drop(output); + assert_eq!(run.retained_bytes(), 0); + } +} diff --git a/crates/asap-physical-operators/tests/physical_semantics.rs b/crates/asap-physical-operators/tests/physical_semantics.rs index a934b99ce..627a14421 100644 --- a/crates/asap-physical-operators/tests/physical_semantics.rs +++ b/crates/asap-physical-operators/tests/physical_semantics.rs @@ -1155,3 +1155,242 @@ fn measure_filters_are_rejected_over_promql_series_identity() { .expect("filtered measure over PromQL series identity"); assert!(error.to_string().contains("PromQL"), "{error}"); } + +/// An exact aggregate node over `input` with one `measure` named `name`, +/// whose output is one non-null `dtype` column. +fn exact_aggregate_node( + measure: planner_types::pre_asap::AggIntent, + name: &str, + dtype: DataType, +) -> PhysicalASAPDAGNode { + use planner_types::{ + post_asap::ExecutionDataState, + pre_asap::{GroupKeys, Reduction as PlanReduction}, + }; + PhysicalASAPDAGNode { + id: 1, + payload: PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Aggregate { + reduction: PlanReduction::Reduce(GroupKeys::by(vec![])), + measures: vec![measure], + output_names: vec![name.into()], + filters: vec![], + having: None, + child: 0, + }), + output_state: ExecutionDataState::QUERY_ROWS, + materialization: Default::default(), + output_schema: (*schema(&[(name, dtype, false)])).clone(), + guarantee: None, + } +} + +// Exact frequency intents bind to native reducers over typed keys, without a +// sketch or numeric key conversion; empty and all-NULL input give 0. +#[test] +fn exact_frequency_intents_execute_typed_keys_and_empty_input() { + use asap_physical_operators::physical_planner::compile_node; + use planner_types::pre_asap::AggIntent; + use planner_types::types::AccuracyTarget; + for (dtype, values) in [ + ( + DataType::Utf8, + vec![Value::Utf8("a".into()), Value::Utf8("b".into())], + ), + ( + DataType::Int64, + vec![ + Value::Int64(9_007_199_254_740_992), + Value::Int64(9_007_199_254_740_993), + ], + ), + (DataType::Bool, vec![Value::Bool(false), Value::Bool(true)]), + ( + DataType::Float64, + vec![Value::Float64(-0.0), Value::Float64(1.0)], + ), + ] { + let input = schema(&[("key", dtype, true)]); + for (measure, name, expected) in [ + ( + AggIntent::FrequencyL2 { + col: Some(0), + accuracy: AccuracyTarget::exact(), + }, + "frequency_l2", + 8.0_f64.sqrt(), + ), + ( + AggIntent::FrequencyEntropy { + col: Some(0), + accuracy: AccuracyTarget::exact(), + }, + "frequency_entropy", + 1.0, + ), + ] { + let node = exact_aggregate_node(measure, name, DataType::Float64); + let operator = compile_node(&node, std::slice::from_ref(&input)) + .expect("exact frequency intent binds"); + let rows = values + .iter() + .flat_map(|v| [vec![v.clone()], vec![v.clone()]]) + .chain([vec![Value::Null]]) + .collect(); + let result = unary(input.clone(), vec![rows], operator.clone()); + assert!( + matches!(result[0][0], Value::Float64(v) if (v - expected).abs() < 1e-12), + "{name}: {result:?}" + ); + for batches in [vec![], vec![vec![vec![Value::Null]]]] { + let result = unary(input.clone(), batches, operator.clone()); + assert!(matches!(result[0][0], Value::Float64(v) if v == 0.0)); + } + } + } +} + +// Each group gets its own frequency population, with one canonical +// signed-zero identity; entropy is in bits. +#[test] +fn exact_frequency_grouping_and_entropy_bits() { + let input = schema(&[ + ("group", DataType::Int64, false), + ("key", DataType::Float64, true), + ]); + let operator = Operator::aggregate( + input.clone(), + vec![0], + vec![ + ("l2".into(), Reduction::FrequencyL2(1)), + ("entropy".into(), Reduction::FrequencyEntropy(1)), + ], + ) + .unwrap(); + let rows = vec![ + vec![Value::Int64(1), Value::Float64(-0.0)], + vec![Value::Int64(1), Value::Float64(0.0)], + vec![Value::Int64(1), Value::Float64(0.0)], + vec![Value::Int64(1), Value::Float64(1.0)], + vec![Value::Int64(2), Value::Float64(2.0)], + vec![Value::Int64(2), Value::Null], + vec![Value::Int64(3), Value::Null], + ]; + let result = unary(input.clone(), vec![rows], operator.clone()); + assert_eq!(result.len(), 3); + let expected_entropy = -0.75_f64 * 0.75_f64.log2() - 0.25_f64 * 0.25_f64.log2(); + for (row, l2, entropy) in [ + (&result[0], 10.0_f64.sqrt(), expected_entropy), + (&result[1], 1.0, 0.0), + (&result[2], 0.0, 0.0), + ] { + assert!(matches!(row[1], Value::Float64(v) if (v - l2).abs() < 1e-12)); + assert!(matches!(row[2], Value::Float64(v) if (v - entropy).abs() < 1e-12)); + } + assert!(unary(input, vec![], operator).is_empty()); +} + +// A non-finite floating identity is an error, not a frequency key. +#[test] +fn exact_frequency_rejects_non_finite_identities() { + let input = schema(&[("key", DataType::Float64, false)]); + let operator = Operator::aggregate( + input.clone(), + vec![], + vec![("l2".into(), Reduction::FrequencyL2(0))], + ) + .unwrap(); + let mut dag = PhysicalDAG::default(); + dag.add( + 0, + vec![], + Operator::source( + input.clone(), + vec![Batch::try_new(input, vec![vec![Value::Float64(f64::INFINITY)]]).unwrap()], + ) + .unwrap(), + ) + .unwrap(); + dag.add(1, vec![0], operator).unwrap(); + let run = context(); + let result = block_on(async { + let mut stream = dag.execute(&[1], run.clone()).unwrap().remove(0); + stream.next().await.unwrap() + }); + assert!(result.is_err()); +} + +// Exact distinct binding keeps typed tuples, skips any tuple with a NULL +// and returns zero on no input. +#[test] +fn exact_cardinality_binds_and_executes_typed_tuples() { + use asap_physical_operators::physical_planner::compile_node; + use planner_types::pre_asap::AggIntent; + use planner_types::types::AccuracyTarget; + let input = schema(&[ + ("key", DataType::Int64, true), + ("tag", DataType::Utf8, true), + ]); + for (cols, expected) in [(vec![0], 2), (vec![0, 1], 3)] { + let node = exact_aggregate_node( + AggIntent::Cardinality { + cols, + accuracy: AccuracyTarget::exact(), + }, + "distinct", + DataType::Int64, + ); + let operator = + compile_node(&node, std::slice::from_ref(&input)).expect("exact distinct intent binds"); + let rows = vec![ + vec![Value::Int64(9_007_199_254_740_992), Value::Utf8("a".into())], + vec![Value::Int64(9_007_199_254_740_992), Value::Utf8("a".into())], + vec![Value::Int64(9_007_199_254_740_992), Value::Utf8("b".into())], + vec![Value::Int64(9_007_199_254_740_993), Value::Utf8("a".into())], + vec![Value::Null, Value::Utf8("c".into())], + ]; + let result = unary(input.clone(), vec![rows], operator.clone()); + assert!(matches!(result[0][0], Value::Int64(v) if v == expected)); + for rows in [vec![], vec![vec![Value::Null, Value::Null]]] { + let result = unary(input.clone(), vec![rows], operator.clone()); + assert!(matches!(result[0][0], Value::Int64(0))); + } + } +} + +// Distinct uses grouped equality: signed zero and NaN payloads each form one +// identity. +#[test] +fn exact_cardinality_grouping_normalizes_float_identities() { + let input = schema(&[ + ("group", DataType::Int64, false), + ("key", DataType::Float64, true), + ]); + let operator = Operator::aggregate( + input.clone(), + vec![0], + vec![("distinct".into(), Reduction::Cardinality(vec![1]))], + ) + .unwrap(); + let result = unary( + input, + vec![vec![ + vec![Value::Int64(1), Value::Float64(0.0)], + vec![Value::Int64(1), Value::Float64(-0.0)], + vec![Value::Int64(1), Value::Float64(f64::NAN)], + vec![ + Value::Int64(1), + Value::Float64(f64::from_bits(f64::NAN.to_bits() + 1)), + ], + vec![Value::Int64(2), Value::Null], + ]], + operator, + ); + assert!(matches!( + result[0].as_slice(), + [Value::Int64(1), Value::Int64(2)] + )); + assert!(matches!( + result[1].as_slice(), + [Value::Int64(2), Value::Int64(0)] + )); +} diff --git a/crates/integration-tests/tests/physical_common/mod.rs b/crates/integration-tests/tests/physical_common/mod.rs index 7bd2b1157..37eb6edfe 100644 --- a/crates/integration-tests/tests/physical_common/mod.rs +++ b/crates/integration-tests/tests/physical_common/mod.rs @@ -73,3 +73,60 @@ fn compile_with( &root, )?) } + +/// Execute a relational DAG at query time against an in-memory connector +/// for its one scan, scan predicates included; checks that no memory stays +/// reserved afterwards. +#[allow(dead_code)] +pub fn execute_raw_rows( + root: &std::rc::Rc, + rows: Vec>, +) -> Vec> { + use asap_physical_operators::dag::{ + planner::bind_with_data_sources, + scan::{DataSources, MemorySource}, + }; + use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; + use asap_types::ir::NonASAPOp; + use std::sync::Arc; + let wire = compile_physical_asap_dag(root).unwrap(); + let (source, schema) = wire + .nodes + .iter() + .find_map(|node| match &node.payload { + PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { source, .. }) => { + Some((source.clone(), node.output_schema.clone())) + } + _ => None, + }) + .expect("one scan"); + let input = Arc::new(schema); + let batch = Batch::try_new(input.clone(), rows).unwrap(); + let mut sources = DataSources::default(); + sources + .register( + source, + Arc::new(MemorySource::new(input, vec![batch]).unwrap()), + ) + .unwrap(); + let root_id = wire.roots[0] as u64; + let plan = bind_with_data_sources(&wire, BTreeMap::new(), &[root_id], &sources).unwrap(); + let context = RunContext::new( + Scope::Query { + evaluation_time_ms: 0, + revision: 1, + }, + Limits::default(), + ) + .unwrap(); + let result = block_on(async { + let mut output = plan.execute(&[root_id], context.clone()).unwrap().remove(0); + let mut rows = vec![]; + while let Some(batch) = output.next().await { + rows.extend_from_slice(batch.unwrap().rows()); + } + rows + }); + assert_eq!(context.retained_bytes(), 0); + result +} diff --git a/crates/integration-tests/tests/sql_cardinality.rs b/crates/integration-tests/tests/sql_cardinality.rs new file mode 100644 index 000000000..df6e76d24 --- /dev/null +++ b/crates/integration-tests/tests/sql_cardinality.rs @@ -0,0 +1,48 @@ +//! The design's SQL distinct query executes natively and exactly. +mod physical_common; +use asap_frontend_sql::{lower_sql, SqlCatalog}; +use asap_physical_operators::values::Value; +use asap_types::{ + pre_asap::{DataType, Field, Schema}, + types::RequestedAccuracy, +}; + +// COUNT(DISTINCT src_ip) skips NULL and keeps the WHERE, Utf8 identities and +// zero on no input. +#[tokio::test] +async fn sql_distinct_executes_through_raw_scan_and_native_binding() { + let catalog = SqlCatalog::new().with_table( + "flows", + Schema::new(vec![ + Field::plain("src_ip", DataType::Utf8, true), + Field::plain("keep", DataType::Bool, false), + ]), + ); + let root = lower_sql( + "SELECT COUNT(DISTINCT src_ip) AS sources FROM flows WHERE keep", + &catalog, + RequestedAccuracy::exact(), + ) + .await + .unwrap(); + for (rows, expected) in [ + (vec![], 0), + (vec![vec![Value::Null, Value::Bool(true)]], 0), + ( + vec![ + vec![Value::Utf8("a".into()), Value::Bool(true)], + vec![Value::Utf8("a".into()), Value::Bool(true)], + vec![Value::Utf8("b".into()), Value::Bool(true)], + vec![Value::Utf8("discard".into()), Value::Bool(false)], + vec![Value::Null, Value::Bool(true)], + ], + 2, + ), + ] { + let result = physical_common::execute_raw_rows(&root, rows); + assert!( + matches!(result.as_slice(), [row] if matches!(row.as_slice(), [Value::Int64(v)] if *v == expected)), + "{result:?}" + ); + } +} From aab7a84c93387809623ca89573fd478fc7cf721c Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 15 Oct 2026 00:05:02 +0000 Subject: [PATCH 05/14] feat(physical-operators): native UnivMon state over typed identities UnivMon had a kernel but no native DAG state: a build was rejected ("summary family has no native DAG state implementation"), a summary build accepted only Float64 samples, and Pass 1's UnivMon update (each value of the item column one unit occurrence) did not bind, so no UnivMon candidate of #509 Example 2 could execute. - UnivMon is a native family; its states validate against their parameters, and one state answers its total, distinct count, L2 norm and entropy. - A UnivMon build takes Bool, Int64 and Utf8 identities as type-tagged key bytes (Int64 above 2^53 stays distinct) besides Float64 samples; NULL is skipped. Retained heap key bytes count toward state memory. - A unit-frequency update over one item column binds, for UnivMon and HLL, as a value-frequency build over that column: grouped (SQL src_ip) and per series (PromQL sample values). Ports the UnivMon part of #597 (from #557). --- .../asap-physical-operators/src/capability.rs | 35 ++- .../src/operators/summary/mod.rs | 22 +- .../src/physical_planner/mod.rs | 48 +++- .../src/summary_kernels/factory.rs | 22 +- .../src/summary_kernels/univmon.rs | 112 +++++++++- crates/asap-physical-operators/src/values.rs | 20 +- .../tests/univmon_execution.rs | 208 ++++++++++++++++++ .../tests/frequency_execution.rs | 152 +++++++++++++ .../tests/precompute_raw_samples.rs | 19 +- 9 files changed, 607 insertions(+), 31 deletions(-) create mode 100644 crates/asap-physical-operators/tests/univmon_execution.rs create mode 100644 crates/integration-tests/tests/frequency_execution.rs diff --git a/crates/asap-physical-operators/src/capability.rs b/crates/asap-physical-operators/src/capability.rs index c8441313e..6e0441ec2 100644 --- a/crates/asap-physical-operators/src/capability.rs +++ b/crates/asap-physical-operators/src/capability.rs @@ -112,7 +112,7 @@ pub fn validate_summary_kernel( } _ => return Err(format!("unsupported summary kernel {family:?}")), }; - if keyed != input.item.is_some() && !is_unit_sample_frequency(input) { + if keyed != input.item.is_some() && unit_frequency_column(input).is_none() { return Err("Planner item expression does not match kernel layout".into()); } Ok(()) @@ -217,6 +217,25 @@ fn valid_matrix(width: u32, depth: u32) -> bool { .is_some() } +/// The one item column of a unit-count frequency update (each value is one +/// occurrence), the update an unkeyed frequency family (HLL, UnivMon) builds +/// from that column's values. +pub(crate) fn unit_frequency_column( + update: &planner_types::post_asap::SummaryUpdate, +) -> Option<&planner_types::pre_asap::ColumnRef> { + use planner_types::post_asap::{NonNegativeWeightProof, SummaryInputExpr, WeightDomain}; + match (&update.item, &update.weight, &update.weight_domain) { + ( + Some(SummaryInputExpr::Column(column)), + SummaryInputExpr::Constant(weight), + WeightDomain::NonNegative { + proof: NonNegativeWeightProof::UnitCount, + }, + ) if *weight == 1.0 => Some(column), + _ => None, + } +} + pub(crate) fn is_unit_sample_frequency(update: &planner_types::post_asap::SummaryUpdate) -> bool { use planner_types::post_asap::{NonNegativeWeightProof, SummaryInputExpr, WeightDomain}; matches!( @@ -266,7 +285,10 @@ pub fn validate_native_family(family: &SummaryFamilyType) -> Result<(), Error> { match family { SummaryFamilyType::ExactAggregate(..) => {} SummaryFamilyType::Sketch(kind, _) - if matches!(kind.algorithm(), A::Kll | A::DDSketch | A::Hll | A::Kmv) => {} + if matches!( + kind.algorithm(), + A::Kll | A::DDSketch | A::Hll | A::Kmv | A::UnivMon + ) => {} _ => { return Err(Error::Invalid( "summary family has no native DAG state implementation".into(), @@ -323,6 +345,15 @@ pub fn validate_sketch_evaluation( } (A::Kmv, SketchStatistic::Cardinality) => true, (A::Kmv, _) => bare_count, + // One frequency state answers its total, distinct count, L2 norm + // and entropy. + ( + A::UnivMon, + SketchStatistic::Cardinality + | SketchStatistic::FrequencyL2 + | SketchStatistic::FrequencyEntropy, + ) => true, + (A::UnivMon, _) => bare_count, // Only count intents read a Count-Min bare count, and their // updates have unit weight; the evaluation is typed Int64 on that basis. (A::Cms, _) => bare_count, diff --git a/crates/asap-physical-operators/src/operators/summary/mod.rs b/crates/asap-physical-operators/src/operators/summary/mod.rs index a180948ac..7e8edcfb7 100644 --- a/crates/asap-physical-operators/src/operators/summary/mod.rs +++ b/crates/asap-physical-operators/src/operators/summary/mod.rs @@ -167,8 +167,14 @@ impl Operator { ) -> Result { crate::values::validate_family(&family)?; validate_groups(&input, &groups)?; - if plain(&input, value)?.0 != &DataType::Float64 { - return Err(invalid("summary numeric update requires Float64")); + let dtype = plain(&input, value)?.0; + let typed_frequency = matches!(&family, SummaryFamilyType::Sketch(kind, _) + if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::UnivMon); + if *dtype != DataType::Float64 + && !(typed_frequency + && matches!(dtype, DataType::Utf8 | DataType::Int64 | DataType::Bool)) + { + return Err(invalid("summary update type is unsupported by its family")); } if let Some(time) = time { if plain(&input, time)? != (&DataType::Timestamp, false) { @@ -532,11 +538,10 @@ async fn build_summary( states.get_mut(&key).expect("inserted group"); // SQL aggregates ignore NULL samples while retaining the group. // A missing counter sample also contributes no observation. - let value = match row[value] { - Value::Float64(value) => value, - Value::Null => continue, - _ => return Err(invalid("summary update type")), - }; + let value = &row[value]; + if matches!(value, Value::Null) { + continue; + } let timestamp = if let Some(time) = time { let Value::Timestamp(time) = row[time] else { return Err(invalid("summary time type")); @@ -552,9 +557,8 @@ async fn build_summary( )); } updater - .validate_single_input(value) + .update_value(value, timestamp) .map_err(Error::Operator)?; - updater.update_single(value, timestamp); *previous = Some(timestamp); memory.resize(updater.memory_usage_bytes() + *overhead)?; } diff --git a/crates/asap-physical-operators/src/physical_planner/mod.rs b/crates/asap-physical-operators/src/physical_planner/mod.rs index eac2332f4..3b0612bbf 100644 --- a/crates/asap-physical-operators/src/physical_planner/mod.rs +++ b/crates/asap-physical-operators/src/physical_planner/mod.rs @@ -427,17 +427,25 @@ fn compile_internal( let Some(NonASAPOp::Scan { schema, .. }) = source.non_asap() else { return Err(invalid("per-entity summary requires a resolved source")); }; - if !schema.closed || update.item.is_some() { + // An unkeyed frequency family (UnivMon) counts each value of + // its one item column once, as a value-frequency build. + let frequency_item = crate::capability::unit_frequency_column(update) + .filter(|_| unkeyed_frequency_family(family)); + if !schema.closed || (update.item.is_some() && frequency_item.is_none()) { return Err(invalid( "per-entity summary requires complete source identity", )); } crate::capability::validate_summary_kernel(family, update, grouping) .map_err(Error::Invalid)?; - let SummaryInputExpr::Column(value) = &update.weight else { - return Err(invalid( - "per-entity update requires a projected value column", - )); + let value = match (frequency_item, &update.weight) { + (Some(item), _) => item, + (None, SummaryInputExpr::Column(value)) => value, + (None, _) => { + return Err(invalid( + "per-entity update requires a projected value column", + )) + } }; let input = schemas[0].clone(); let value = named_column(&input, value)?; @@ -966,6 +974,27 @@ fn summary_build( groups(input, keys)?, ); } + // An unkeyed frequency family (UnivMon) counts each value of its + // one item column once: a value-frequency build over that column. + if let (Some(item), true) = ( + crate::capability::unit_frequency_column(update), + unkeyed_frequency_family(family), + ) { + crate::capability::validate_summary_kernel(family, update, grouping) + .map_err(Error::Invalid)?; + let PlannerReduction::Reduce(keys) = reduction else { + return Err(invalid( + "summary construction requires explicit grouping columns", + )); + }; + return Operator::summary_build( + input.clone(), + family.clone(), + named_column(input, item)?, + input.time_index, + groups(input, keys)?, + ); + } if let Some(item) = &update.item { let PlannerReduction::Reduce(keys) = reduction else { return Err(invalid("keyed summary requires explicit partitions")); @@ -1028,6 +1057,15 @@ fn summary_build( groups(input, keys)?, ) } +/// Families whose state counts values of one column without keyed items: +/// their unit-frequency update builds over that column. +fn unkeyed_frequency_family(family: &FieldDataType) -> bool { + matches!(family, FieldDataType::Sketch(kind, _) if matches!( + kind.algorithm(), + planner_types::post_asap::SketchAlgorithm::UnivMon + | planner_types::post_asap::SketchAlgorithm::Hll + )) +} fn summary_column(input: &SchemaRef) -> Result { let columns = input .fields diff --git a/crates/asap-physical-operators/src/summary_kernels/factory.rs b/crates/asap-physical-operators/src/summary_kernels/factory.rs index 1d9da341e..caa1e6b17 100644 --- a/crates/asap-physical-operators/src/summary_kernels/factory.rs +++ b/crates/asap-physical-operators/src/summary_kernels/factory.rs @@ -46,6 +46,21 @@ pub trait AccumulatorUpdater: Send { } } + /// Update from one typed row value. Numeric kernels keep their Float64 + /// contract; a frequency kernel may also take non-numeric identities. + fn update_value( + &mut self, + value: &crate::values::Value, + timestamp_ms: i64, + ) -> Result<(), String> { + let crate::values::Value::Float64(value) = value else { + return Err("summary update requires Float64".into()); + }; + self.validate_single_input(*value)?; + self.update_single(*value, timestamp_ms); + Ok(()) + } + /// Feed a single (value, timestamp_ms) pair — for SingleSubpopulation types. fn update_single(&mut self, value: f64, timestamp_ms: i64); @@ -645,7 +660,7 @@ pub fn create_planner_accumulator( } }; if updater.is_keyed() != input.item.is_some() - && !crate::capability::is_unit_sample_frequency(input) + && crate::capability::unit_frequency_column(input).is_none() { return Err("Planner item expression does not match the selected kernel layout".into()); } @@ -718,6 +733,11 @@ impl AccumulatorUpdater for HllUpdater { } impl AccumulatorUpdater for UnivMonUpdater { + fn update_value(&mut self, value: &crate::values::Value, _: i64) -> Result<(), String> { + self.acc + .insert_value(value) + .map_err(|error| error.to_string()) + } fn is_keyed(&self) -> bool { false } diff --git a/crates/asap-physical-operators/src/summary_kernels/univmon.rs b/crates/asap-physical-operators/src/summary_kernels/univmon.rs index 63104a22b..128e7ab6a 100644 --- a/crates/asap-physical-operators/src/summary_kernels/univmon.rs +++ b/crates/asap-physical-operators/src/summary_kernels/univmon.rs @@ -90,6 +90,27 @@ impl UnivMonAccumulator { Ok(()) } + /// One occurrence of a typed identity. A finite Float64 is a sample + /// ([`Self::insert_sample`]); Bool, Int64 and Utf8 keep their type as + /// type-tagged key bytes, since converting Int64 to Float64 would collapse + /// neighbouring keys above 2^53. NULL is skipped. + pub fn insert_value(&mut self, value: &crate::values::Value) -> Result<(), Error> { + use crate::values::Value; + match value { + Value::Null => return Ok(()), + Value::Float64(number) if number.is_finite() => return self.insert_sample(*number), + Value::Bool(_) | Value::Int64(_) | Value::Utf8(_) => {} + _ => return Err("unsupported UnivMon identity type or non-finite sample".into()), + } + let key = value.key()?; + self.inner + .bucket_size + .checked_add(1) + .ok_or("UnivMon count overflow")?; + self.inner.insert(&DataInput::Bytes(&key), 1); + Ok(()) + } + fn compatible(&self, other: &Self) -> bool { ( self.inner.heap_size, @@ -127,16 +148,28 @@ impl UnivMonAccumulator { } impl AggregateCore for UnivMonAccumulator { + /// Counters and heap slots of every layer, plus the bytes of the + /// variable-length keys the heaps retain. fn approx_memory_bytes(&self) -> usize { - std::mem::size_of::().saturating_add( - self.inner.layer_size.saturating_mul( - self.inner - .sketch_row - .saturating_mul(self.inner.sketch_col) - .saturating_mul(16) - .saturating_add(self.inner.heap_size.saturating_mul(256)), - ), - ) + let key_bytes = (0..self.inner.layer_size) + .flat_map(|layer| self.inner.hh_layers[layer].heap()) + .map(|item| match &item.key { + asap_sketchlib::HeapItem::String(key) => key.capacity(), + asap_sketchlib::HeapItem::Bytes(key) => key.capacity(), + _ => 0, + }) + .fold(0usize, usize::saturating_add); + key_bytes + .saturating_add(std::mem::size_of::()) + .saturating_add( + self.inner.layer_size.saturating_mul( + self.inner + .sketch_row + .saturating_mul(self.inner.sketch_col) + .saturating_mul(16) + .saturating_add(self.inner.heap_size.saturating_mul(256)), + ), + ) } fn clone_boxed_core(&self) -> Box { Box::new(self.clone()) @@ -248,4 +281,65 @@ mod tests { assert!(UnivMonAccumulator::new(4, 11, 2047, 1).is_err()); assert!(UnivMonAccumulator::new(4, 10, 2047, 1).is_ok()); } + + // Typed keys stay distinct, and merge, when entered as type-tagged bytes: + // Int64 identities above 2^53 do not collapse. + #[test] + fn typed_keys_merge_and_stay_distinct() { + use crate::values::Value; + for keys in [ + [ + Value::Utf8("192.0.2.1".into()), + Value::Utf8("192.0.2.2".into()), + ], + [ + Value::Int64(9_007_199_254_740_992), + Value::Int64(9_007_199_254_740_993), + ], + [Value::Bool(false), Value::Bool(true)], + ] { + let mut left = UnivMonAccumulator::new(32, 5, 1024, 4).unwrap(); + let mut right = UnivMonAccumulator::new(32, 5, 1024, 4).unwrap(); + for _ in 0..2 { + left.insert_value(&keys[0]).unwrap(); + right.insert_value(&keys[1]).unwrap(); + } + left.insert_value(&Value::Null).unwrap(); + left.merge_in_place(&right).unwrap(); + for (statistic, expected) in [ + (count(), 4.0), + (SketchStatistic::Cardinality, 2.0), + (SketchStatistic::FrequencyL2, 8.0_f64.sqrt()), + (SketchStatistic::FrequencyEntropy, 1.0), + ] { + let estimate = left.estimate(&statistic).unwrap(); + assert!( + (estimate - expected).abs() < 0.01, + "{keys:?} {statistic:?}: {estimate}" + ); + } + } + } + + // A non-finite sample or an unsupported identity type is an error. + #[test] + fn rejects_non_finite_and_untyped_identities() { + use crate::values::Value; + let mut state = UnivMonAccumulator::new(32, 5, 1024, 4).unwrap(); + assert!(state.insert_value(&Value::Float64(f64::INFINITY)).is_err()); + assert!(state.insert_value(&Value::Timestamp(1)).is_err()); + } + + // Retained variable-length identities count toward the state's memory. + #[test] + fn memory_accounts_for_string_identities() { + let mut state = UnivMonAccumulator::new(32, 5, 1024, 4).unwrap(); + let empty = state.approx_memory_bytes(); + state + .insert_value(&crate::values::Value::Utf8("x".repeat(4096).into())) + .unwrap(); + assert!(state.approx_memory_bytes() >= empty + 4096); + state.clear(); + assert_eq!(state.approx_memory_bytes(), empty); + } } diff --git a/crates/asap-physical-operators/src/values.rs b/crates/asap-physical-operators/src/values.rs index 33f8ff2b4..830a624a8 100644 --- a/crates/asap-physical-operators/src/values.rs +++ b/crates/asap-physical-operators/src/values.rs @@ -277,7 +277,7 @@ fn validate_state(family: &SummaryFamilyType, state: &dyn AggregateCore) -> Resu use crate::summary_kernels::{ count_min_sketch::CountMinSketchAccumulator, datasketches_kll::DatasketchesKLLAccumulator, dd_sketch::DDSketchAccumulator, exact::ExactAccumulator, hll_sketch::HllSketchAccumulator, - kmv::KmvAccumulator, + kmv::KmvAccumulator, univmon::UnivMonAccumulator, }; use planner_types::post_asap::SketchParams; validate_family(family)?; @@ -328,6 +328,24 @@ fn validate_state(family: &SummaryFamilyType, state: &dyn AggregateCore) -> Resu .as_any() .downcast_ref::() .is_some_and(|s| s.inner.k == *k as usize), + SketchParams::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, + .. + } => state + .as_any() + .downcast_ref::() + .is_some_and(|s| { + s.dimensions() + == ( + *heap_size as usize, + *sketch_rows as usize, + *sketch_cols as usize, + usize::from(*layers), + ) + }), SketchParams::Cms { width, depth } => state .as_any() .downcast_ref::() diff --git a/crates/asap-physical-operators/tests/univmon_execution.rs b/crates/asap-physical-operators/tests/univmon_execution.rs new file mode 100644 index 000000000..fc8725117 --- /dev/null +++ b/crates/asap-physical-operators/tests/univmon_execution.rs @@ -0,0 +1,208 @@ +//! One native UnivMon build supplies the three statistics of #509 Example 2: +//! a distinct count, an entropy and an L2 norm of one identity column. +use asap_physical_operators::{ + operators::{Operator, SummaryEvaluation}, + physical_planner::compile_node, + plan::{PhysicalDAG, PhysicalOperator}, + runtime::{Limits, RunContext, Scope}, + values::{Batch, SchemaRef, Value}, + Error, +}; +use futures::{executor::block_on, StreamExt}; +use planner_types::ir::physical_export::{PhysicalASAPDAGNode, PhysicalASAPOperatorPayload}; +use planner_types::ir::ASAPOp; +use planner_types::post_asap::{ + ExecutionDataState, Field, FieldDataType, GroupingStrategy, NonNegativeWeightProof, + SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, SummaryInputExpr, SummaryUpdate, + WeightDomain, +}; +use planner_types::pre_asap::{ColumnRef, DataType, GroupKeys, Reduction, Schema}; +use std::sync::Arc; + +fn family() -> FieldDataType { + FieldDataType::Sketch( + SketchKind::new( + SketchAlgorithm::UnivMon, + SketchParams::UnivMon { + heap_size: 64, + sketch_rows: 5, + sketch_cols: 128, + layers: 4, + distinct_failure_probability: None, + }, + ), + GroupingStrategy::default(), + ) +} + +fn context() -> RunContext { + RunContext::new( + Scope::Query { + evaluation_time_ms: 0, + revision: 1, + }, + Limits::default(), + ) + .unwrap() +} + +fn value(row: &[Value]) -> f64 { + match row { + [Value::Float64(value)] => *value, + other => panic!("expected one floating point result, got {other:?}"), + } +} + +/// Build `build` over `batch` and read the three statistics from its state. +fn three_statistics(batch: Batch, build: Operator) -> Result, Error> { + let state = build.output_schema(); + let mut dag = PhysicalDAG::default(); + dag.add( + 0, + vec![], + Operator::source(batch.schema().clone(), vec![batch])?, + )?; + dag.add(1, vec![0], build)?; + for (id, statistic) in [ + (2, SketchStatistic::Cardinality), + (3, SketchStatistic::FrequencyL2), + (4, SketchStatistic::FrequencyEntropy), + ] { + dag.add( + id, + vec![1], + Operator::evaluation(state.clone(), 0, SummaryEvaluation::Sketch(statistic))?, + )?; + } + let outputs = block_on(futures::future::join_all( + dag.execute(&[2, 3, 4], context())? + .into_iter() + .map(|stream| stream.collect::>()), + )); + outputs + .into_iter() + .map(|stream| { + let batches = stream.into_iter().collect::, _>>()?; + Ok(value(&batches[0].rows()[0])) + }) + .collect() +} + +// One UnivMon state over Float64 samples answers distinct, L2 and entropy. +#[test] +fn one_univmon_state_answers_distinct_l2_and_entropy() -> Result<(), Error> { + let input: SchemaRef = Arc::new(Schema::new(vec![Field::plain( + "value", + DataType::Float64, + false, + )])); + let batch = Batch::try_new( + input.clone(), + [1.0, 1.0, 2.0, 2.0, 2.0] + .into_iter() + .map(|value| vec![Value::Float64(value)]) + .collect(), + )?; + let build = Operator::summary_build(input, family(), 0, None, vec![])?; + let [distinct, l2, entropy] = three_statistics(batch, build)?[..] else { + unreachable!() + }; + assert!((distinct - 2.0).abs() < 0.01); + assert!((l2 - 13.0_f64.sqrt()).abs() < 0.01); + let expected = -0.4_f64 * 0.4_f64.log2() - 0.6 * 0.6_f64.log2(); + assert!((entropy - expected).abs() < 0.01, "{entropy}"); + Ok(()) +} + +// SQL source IPs and integer identifiers enter frequency state without +// numeric coercion; NULL identities are skipped. +#[test] +fn typed_frequency_keys_preserve_identity() -> Result<(), Error> { + for (dtype, keys) in [ + ( + DataType::Utf8, + [ + Value::Utf8("192.0.2.1".into()), + Value::Utf8("192.0.2.2".into()), + ], + ), + ( + DataType::Int64, + [ + Value::Int64(9_007_199_254_740_992), + Value::Int64(9_007_199_254_740_993), + ], + ), + (DataType::Bool, [Value::Bool(false), Value::Bool(true)]), + ] { + let input: SchemaRef = Arc::new(Schema::new(vec![Field::plain("src_ip", dtype, true)])); + let batch = Batch::try_new( + input.clone(), + vec![ + vec![keys[0].clone()], + vec![keys[0].clone()], + vec![keys[1].clone()], + vec![keys[1].clone()], + vec![Value::Null], + ], + )?; + let build = Operator::summary_build(input, family(), 0, None, vec![])?; + for (found, expected) in + three_statistics(batch, build)? + .into_iter() + .zip([2.0, 8.0_f64.sqrt(), 1.0]) + { + assert!((found - expected).abs() < 0.01, "{found} vs {expected}"); + } + } + Ok(()) +} + +// Pass 1's UnivMon update (each item column value is one unit occurrence) +// binds as a value-frequency build over that column, a Utf8 one included. +#[test] +fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<(), Error> { + let input: SchemaRef = Arc::new(Schema::new(vec![Field::plain( + "src_ip", + DataType::Utf8, + false, + )])); + let state = Arc::new(Schema::new(vec![Field { + table: None, + name: "state".into(), + dtype: family(), + nullable: false, + }])); + let node = PhysicalASAPDAGNode { + id: 1, + payload: PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { + child: 0, + family: family(), + input: SummaryUpdate { + item: Some(SummaryInputExpr::Column(ColumnRef::Named("src_ip".into()))), + weight: SummaryInputExpr::Constant(1.0), + weight_domain: WeightDomain::NonNegative { + proof: NonNegativeWeightProof::UnitCount, + }, + }, + reduction: Reduction::Reduce(GroupKeys::by(vec![])), + grouping: GroupingStrategy::default(), + filter: None, + }), + output_state: ExecutionDataState::QUERY_ROWS, + materialization: Default::default(), + output_schema: (*state).clone(), + guarantee: None, + }; + let build = compile_node(&node, std::slice::from_ref(&input))?; + let batch = Batch::try_new( + input, + ["a", "a", "b", "b"] + .into_iter() + .map(|key| vec![Value::Utf8(key.into())]) + .collect(), + )?; + let found = three_statistics(batch, build)?; + assert!((found[0] - 2.0).abs() < 0.01, "{found:?}"); + Ok(()) +} diff --git a/crates/integration-tests/tests/frequency_execution.rs b/crates/integration-tests/tests/frequency_execution.rs new file mode 100644 index 000000000..a1a89cb5c --- /dev/null +++ b/crates/integration-tests/tests/frequency_execution.rs @@ -0,0 +1,152 @@ +//! #509 Example 2's PromQL stand-in executes natively: the distinct count, +//! entropy and L2 norm of one series' sample values over 1 min, through the +//! exact candidate and through the one that builds a UnivMon, at query time. + +use std::collections::BTreeMap; +use std::rc::Rc; +use std::sync::Arc; + +use asap_integration_tests::fixtures::lower_promql; +use asap_integration_tests::planning::logical_candidates; +use asap_physical_operators::physical_planner::{ + compile_candidate, frontier_from_timing, promql_rows, InputContract, +}; +use asap_physical_operators::runtime::Scope; +use asap_physical_operators::values::{Batch, Value}; +use asap_types::ir::physical_export::{compile_physical_asap_dag, PhysicalASAPOperatorPayload}; +use asap_types::ir::{ + apply_materialization_timings, ASAPOp, MaterializationAssignment, NonASAPOp, OperatorNode, + TimingMemo, +}; +use asap_types::post_asap::{FieldDataType, SketchAlgorithm}; +use asap_types::types::RequestedAccuracy; + +mod physical_common; + +const EVALUATION_MS: i64 = 300_000; +/// `flows_src_ip` samples: source 1 twice, 2 three times, 3 once. +const SAMPLES: [f64; 6] = [1.0, 2.0, 1.0, 2.0, 3.0, 2.0]; + +/// The summary family a candidate builds: `None` for the exact plan. +fn built_family(candidate: &Rc) -> Option { + OperatorNode::reachable(candidate) + .iter() + .find_map(|node| match node.asap() { + Some(ASAPOp::SummaryAgg { family, .. }) => Some(family.clone()), + _ => None, + }) +} + +/// Execute `candidate` at query time over `SAMPLES`, all inside the window. +fn execute(candidate: &Rc) -> Vec { + let timed = apply_materialization_timings( + candidate, + &MaterializationAssignment::default(), + &mut TimingMemo::new(), + ) + .unwrap(); + let dag = compile_physical_asap_dag(&timed).unwrap(); + let raw: BTreeMap = dag + .nodes + .iter() + .filter(|node| { + matches!( + node.payload, + PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::TimeRange { .. }) + ) + }) + .map(|node| (node.id as u64, Arc::new(node.output_schema.clone()))) + .collect(); + assert_eq!(raw.len(), 1, "one range selector"); + let frontier = frontier_from_timing(&dag).unwrap(); + let compiled = compile_candidate( + &dag, + raw.iter() + .map(|(id, schema)| (*id, InputContract::bounded(schema.clone()))) + .collect(), + &[dag.roots[0] as u64], + &frontier, + ) + .unwrap(); + assert!(compiled.precompute.is_none()); + let labels = BTreeMap::from([("__name__".to_string(), "flows_src_ip".to_string())]); + let inputs = raw + .iter() + .map(|(id, schema)| { + let rows = SAMPLES + .iter() + .enumerate() + .map(|(i, value)| { + promql_rows::series_row(schema, &labels, 290_000 + i as i64, *value).unwrap() + }) + .collect(); + (*id, Batch::try_new(schema.clone(), rows).unwrap()) + }) + .collect(); + let result = physical_common::execute( + &compiled.query, + inputs, + Scope::Query { + evaluation_time_ms: EVALUATION_MS, + revision: 1, + }, + ); + result[0] + .iter() + .flat_map(|batch| batch.rows()) + .flat_map(|row| row.iter()) + .filter_map(|value| match value { + Value::Float64(value) => Some(*value), + Value::Int64(value) => Some(*value as f64), + _ => None, + }) + .collect() +} + +// The exact candidate and the UnivMon candidate of each statistic both +// execute and give the statistic of the window's sample values (entropy in +// bits). +#[test] +fn standin_exact_and_univmon_candidates_execute() { + let entropy = -[2.0_f64, 3.0, 1.0] + .iter() + .map(|count| count / 6.0 * (count / 6.0).log2()) + .sum::(); + // A UnivMon sized for a distinct count at ε = 0.01 holds a heap of + // millions of keys on each of its 19 layers, past the default 64 MiB. + for (query, expected, epsilon, delta) in [ + ("distinct_over_time(flows_src_ip[1m])", 3.0, 0.2, 0.1), + ("entropy_over_time(flows_src_ip[1m])", entropy, 0.01, 0.01), + ( + "l2_over_time(flows_src_ip[1m])", + 14.0_f64.sqrt(), + 0.01, + 0.01, + ), + ] { + let accuracy = RequestedAccuracy::epsilon_delta(epsilon, delta); + let root = lower_promql(query, accuracy.clone()).unwrap(); + let root = promql_rows::with_series_identity(&root).unwrap(); + let mut executed = Vec::new(); + for candidate in logical_candidates(root, accuracy) { + let family = built_family(&candidate); + let name = match &family { + None => "exact".to_string(), + Some(FieldDataType::Sketch(kind, _)) + if kind.algorithm() == &SketchAlgorithm::UnivMon => + { + "UnivMon".to_string() + } + _ => continue, + }; + let values = execute(&candidate); + assert!( + matches!(values.as_slice(), [value] if (value - expected).abs() < 1e-9), + "{query} {name}: {values:?}, expected {expected}" + ); + executed.push(name); + } + executed.sort(); + assert_eq!(executed, ["UnivMon", "exact"], "{query}"); + } +} diff --git a/crates/integration-tests/tests/precompute_raw_samples.rs b/crates/integration-tests/tests/precompute_raw_samples.rs index da1fdaea9..71fe0cb37 100644 --- a/crates/integration-tests/tests/precompute_raw_samples.rs +++ b/crates/integration-tests/tests/precompute_raw_samples.rs @@ -277,6 +277,14 @@ fn evaluations(state: &dyn AggregateCore, family: &FieldDataType) -> Vec { SketchAlgorithm::Hll | SketchAlgorithm::Kmv => { vec![state.estimate(&SketchStatistic::Cardinality).unwrap()] } + SketchAlgorithm::UnivMon => [ + SketchStatistic::Cardinality, + SketchStatistic::FrequencyL2, + SketchStatistic::FrequencyEntropy, + ] + .iter() + .map(|statistic| state.estimate(statistic).unwrap()) + .collect(), other => panic!("unexpected unkeyed sketch {other:?}"), } } @@ -314,8 +322,7 @@ fn check( let stored_only = matches!(family, FieldDataType::Sketch(kind, _) if kind.algorithm() == &asap_types::post_asap::SketchAlgorithm::Cms); if stored_only || asap_physical_operators::capability::validate_native_family(family).is_err() { - // Families without a native state (e.g. UnivMon), or with native - // stored state only (plain CMS), are outside precompute execution; + // Families without a native state, or with native stored state only (plain CMS), are outside precompute execution; // their compile must fail. assert!(precompute::compile(dag, &[source], &[root]).is_err()); return Err(format!("{family:?}")); @@ -331,7 +338,7 @@ fn check( &input.item, matches!( kind.algorithm(), - SketchAlgorithm::Hll | SketchAlgorithm::Kmv + SketchAlgorithm::Hll | SketchAlgorithm::Kmv | SketchAlgorithm::UnivMon ), ) { // Keyed heaps: every item's estimated weight is its exact @@ -391,6 +398,9 @@ fn check( fn raw_sample_summaries_compile_and_match_their_kernels() { let exact = RequestedAccuracy::exact(); let sketch = RequestedAccuracy::epsilon(0.02); + // A UnivMon sized for a distinct count at ε = 0.02 holds a heap of + // millions of keys on each of its 19 layers, past the default 64 MiB. + let distinct = RequestedAccuracy::epsilon_delta(0.2, 0.1); let queries = [ ("sum_over_time(m[5m])", &exact), ("count_over_time(m[5m])", &exact), @@ -404,7 +414,7 @@ fn raw_sample_summaries_compile_and_match_their_kernels() { ("quantile_over_time(0.9, m[5m])", &sketch), ("sum by (service) (quantile_over_time(0.9, m[5m]))", &sketch), ("quantile by (service) (0.9, m)", &sketch), - ("distinct_over_time(m[5m])", &sketch), + ("distinct_over_time(m[5m])", &distinct), ("count(m)", &sketch), ("topk(2, m)", &sketch), ( @@ -456,6 +466,7 @@ fn raw_sample_summaries_compile_and_match_their_kernels() { "DDSketch", "Hll", "Kmv", + "UnivMon", // The whole-expression heaps over raw samples. "CmsWithHeap", "CountSketchWithHeap", From 835bb2d8c38acc208e2f5472659f99c2b3aefe30 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 15 Oct 2026 03:03:11 +0000 Subject: [PATCH 06/14] test(integration-tests): Example 2 in SQL; the recognized idioms execute Example 2's SQL Q3 casts its counts to DOUBLE (the design's Int64 SUM(c * c) can overflow and stays relational), and the SQL Q2 and Q3 now lower to the entropy and L2 intents, so the SQL workload has the PromQL stand-in's Pass 1 options. The recognized entropy executes in nats with SQL's -0.0, L2 executes exactly, and both give NULL on empty input, decided by the exact population count. --- .../tests/planner_layering_example2.rs | 15 +++-- .../integration-tests/tests/sql_frequency.rs | 65 +++++++++++++++++++ 2 files changed, 73 insertions(+), 7 deletions(-) create mode 100644 crates/integration-tests/tests/sql_frequency.rs diff --git a/crates/integration-tests/tests/planner_layering_example2.rs b/crates/integration-tests/tests/planner_layering_example2.rs index 50b7c5498..c18f5a6a6 100644 --- a/crates/integration-tests/tests/planner_layering_example2.rs +++ b/crates/integration-tests/tests/planner_layering_example2.rs @@ -6,10 +6,11 @@ //! need an unimplemented feature are `#[ignore]`d, naming it and the PR that //! delivers it. //! -//! The doc's workload is SQL. The SQL frontend lowers Q1 to a distinct count -//! but not Q2 and Q3 to `Entropy` and `L2` (the doc's own TODO, #598), so -//! the planning stages run on a PromQL stand-in with the same structure: -//! three statistics of one input over the same 1-min window, through the +//! The doc's workload is SQL; the SQL frontend lowers Q1 to a distinct +//! count and recognizes Q2 and Q3 as `Entropy` and `L2` (Q3 with a Float64 +//! product). Its time filter stays a scan predicate, so the planning +//! stages also run on a PromQL stand-in with the same structure: three +//! statistics of one input over the same 1-min window, through the //! frontend's `distinct_over_time`, `entropy_over_time` and `l2_over_time`. mod physical_common; @@ -39,7 +40,9 @@ const SQL_Q1: &str = const SQL_Q2: &str = "SELECT -SUM(p * LN(p)) FROM (\ SELECT COUNT(*) * 1.0 / SUM(COUNT(*)) OVER () AS p FROM flows \ WHERE ts >= now() - INTERVAL '1 minute' GROUP BY src_ip)"; -const SQL_Q3: &str = "SELECT SQRT(SUM(c * c)) FROM (\ +/// Q3 casts the counts to DOUBLE: the design's Int64 `SUM(c * c)` can +/// overflow, so the SQL frontend recognizes only the Float64 product. +const SQL_Q3: &str = "SELECT SQRT(SUM(CAST(c AS DOUBLE) * CAST(c AS DOUBLE))) FROM (\ SELECT src_ip, COUNT(*) AS c FROM flows \ WHERE ts >= now() - INTERVAL '1 minute' GROUP BY src_ip)"; @@ -271,7 +274,6 @@ async fn stage0_query_language_frontends_sql_q1_lowers_to_a_distinct_count() { /// The SQL frontend recognizes Q2 as `Entropy(src_ip)` and Q3 as `L2(src_ip)`. #[tokio::test] -#[ignore = "needs SQL frontend recognition of the Entropy and L2 idioms (#598)"] async fn stage0_query_language_frontends_sql_q2_q3_lower_to_entropy_and_l2() { let roots = lower_sql(&sql_workload()).await; assert!(frontend_operations(&roots[1]).contains(&"aggregate:frequency_entropy".to_string())); @@ -280,7 +282,6 @@ async fn stage0_query_language_frontends_sql_q2_q3_lower_to_entropy_and_l2() { /// Once the SQL forms are recognized, the SQL workload has the stand-in's Pass 1 options. #[tokio::test] -#[ignore = "needs SQL frontend recognition of the Entropy and L2 idioms (#598)"] async fn stage1_logical_asap_optimization_sql_has_the_standin_options() { let workload = sql_workload(); let sql = run_stages(&workload, lower_sql(&workload).await); diff --git a/crates/integration-tests/tests/sql_frequency.rs b/crates/integration-tests/tests/sql_frequency.rs new file mode 100644 index 000000000..7cb40514b --- /dev/null +++ b/crates/integration-tests/tests/sql_frequency.rs @@ -0,0 +1,65 @@ +//! The SQL entropy and L2 idioms, recognized by the SQL frontend, execute +//! natively and exactly: entropy in nats (SQL's `LN`), L2 of the +//! per-source counts, and NULL on empty input, decided by an exact count. +mod physical_common; +use asap_frontend_sql::{lower_sql, SqlCatalog}; +use asap_physical_operators::values::Value; +use asap_types::{ + pre_asap::{DataType, Field, Schema}, + types::RequestedAccuracy, +}; + +const ENTROPY: &str = "SELECT -SUM(p * LN(p)) FROM (\ + SELECT COUNT(*) * 1.0 / SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip)"; +const L2: &str = "SELECT SQRT(SUM(CAST(c AS DOUBLE) * CAST(c AS DOUBLE))) FROM (\ + SELECT src_ip, COUNT(*) AS c FROM flows GROUP BY src_ip)"; + +fn catalog() -> SqlCatalog { + SqlCatalog::new().with_table( + "flows", + Schema::new(vec![Field::plain("src_ip", DataType::Utf8, false)]), + ) +} + +fn rows(sources: &[&str]) -> Vec> { + sources + .iter() + .map(|s| vec![Value::Utf8((*s).into())]) + .collect() +} + +async fn run(sql: &str, sources: &[&str]) -> Option { + let root = lower_sql(sql, &catalog(), RequestedAccuracy::exact()) + .await + .unwrap(); + let result = physical_common::execute_raw_rows(&root, rows(sources)); + match result.as_slice() { + [row] => match row.as_slice() { + [Value::Float64(v)] => Some(*v), + [Value::Null] => None, + other => panic!("{other:?}"), + }, + other => panic!("{other:?}"), + } +} + +/// Counts 3 and 1 over four rows: entropy −0.75 ln 0.75 − 0.25 ln 0.25 +/// nats; one source gives −0.0; no rows give NULL. +#[tokio::test] +async fn recognized_sql_entropy_executes_in_nats() { + let expected = -0.75f64 * 0.75f64.ln() - 0.25 * 0.25f64.ln(); + let got = run(ENTROPY, &["a", "a", "a", "b"]).await.unwrap(); + assert!((got - expected).abs() < 1e-12, "{got} vs {expected}"); + let single = run(ENTROPY, &["a", "a"]).await.unwrap(); + assert_eq!(single, 0.0); + assert!(single.is_sign_negative(), "SQL's -SUM(1 * LN 1) is -0.0"); + assert_eq!(run(ENTROPY, &[]).await, None); +} + +/// Counts 2 and 1: L2 √5; no rows give NULL. +#[tokio::test] +async fn recognized_sql_l2_executes_exactly() { + let got = run(L2, &["a", "a", "b"]).await.unwrap(); + assert!((got - 5f64.sqrt()).abs() < 1e-12, "{got}"); + assert_eq!(run(L2, &[]).await, None); +} From bba6aff8ecdca928452263a25715f583a28ee789 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 15 Oct 2026 03:03:11 +0000 Subject: [PATCH 07/14] feat(devtools): stage_pipeline example planner-layering-2 stage_pipeline plans #509 Example 2: three SQL dashboard panels over the last minute of flows (distinct sources at eps 0.02, entropy at 0.05, L2 at 0.01, delta 0.01), lowered over its flows(ts, src_ip) table. SQL workloads lower every entry, batch and repeating, each with its own accuracy target (lower_sql_batch read only batch entries). The stage-viewer tab 2 row is regenerated from it: 82 plans, 16 from Pass 1, the rest from the identical-expression and summary-capability rules (one UnivMon for each pair of queries and for all three); 6 valid, P1 (all exact) selected at 6.08 per second. The row previously said 8 plans were valid. --- crates/devtools/src/bin/stage_pipeline.rs | 21 +++-- .../devtools/src/planner_layering_examples.rs | 76 ++++++++++++++++++- crates/devtools/src/planning_stages.rs | 30 +++++--- crates/devtools/tests/stage_pipeline.rs | 38 +++++++++- docs/user_guide_docs/stage-viewer.md | 2 +- 5 files changed, 146 insertions(+), 21 deletions(-) diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index 323df79ab..f965c8a7e 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -1,7 +1,8 @@ // cargo run -p asap-devtools --bin stage_pipeline -- \ // --example planner-layering-1 --out planner-layering-example1.json -// (also planner-layering-3a and planner-layering-3b: #509 Example 3, -// Patterns A and B; 3a has 972 candidates, all priced, 64 shown by default) +// (also planner-layering-2: #509 Example 2, three SQL flow statistics; +// planner-layering-3a and planner-layering-3b: #509 Example 3, Patterns A +// and B; 3a has 972 candidates, all priced, 64 shown by default) // cargo run -p asap-devtools --bin stage_pipeline -- \ // --promql "topk by (job) (10, rate(x[1m]))" --epsilon 0.01 --delta 0.001 --out run.json // cargo run -p asap-devtools --bin stage_pipeline -- \ @@ -14,8 +15,10 @@ // - stage0_query_language_frontends: the frontends' workload DAG, one root per query; // - stage1_logical_asap_optimization: Pass 1 whole-workload candidates, one per // choice of a per-computation candidate for every target (the Cartesian -// product), each followed by its Pass 2 identical-expression candidates -// (one per subset of its sharing groups), in enumeration order. Every candidate is optimized and +// product), each followed by its Pass 2 candidates (the +// identical-expression rule's, one per subset of its sharing groups, +// then the summary-capability rule's, one per choice of shared summary +// builds), in enumeration order. Every candidate is optimized and // priced; the document shows the first `--max-displayed` (default 64) // and the selected one, with `combinations` (how many exist) and // `capped` (whether some are not shown). A candidate that cannot be @@ -59,7 +62,7 @@ use std::collections::HashSet; use asap_aware_mapping::stage3_plan_selection::{Rejection, Selection, SelectionError}; use asap_devtools::planner_layering_examples::{ - declared, example1, pattern_a, pattern_b, PatternARecurrence, + declared, example1, example2, example2_catalog, pattern_a, pattern_b, PatternARecurrence, }; use asap_devtools::planning_stages::{ lower_promql_roots, lower_sql_roots, run_planning_stages, run_planning_stages_over, @@ -78,7 +81,7 @@ use asap_types::workload::{ use serde_json::{json, Value}; const USAGE: &str = - "usage: stage_pipeline (--example planner-layering-{1,3a,3b} | ((--promql | --sql) \ + "usage: stage_pipeline (--example planner-layering-{1,2,3a,3b} | ((--promql | --sql) \ [--name ]... [--table ...] [--epsilon --delta --metric ] [--interval-ms ])) \ [--max-combinations ] [--max-displayed ] --out "; @@ -211,13 +214,17 @@ fn run(args: Vec) -> Result<(), Failure> { (Some(example), true) => { let workload = match example { "planner-layering-1" => example1(), + "planner-layering-2" => example2(), "planner-layering-3a" => { pattern_a(PatternARecurrence::OnceAdHoc, DataArrival::Mixed) } "planner-layering-3b" => pattern_b(), other => return Err(format!("unknown example {other}").into()), }; - let roots = lower_promql_roots(&workload)?; + let roots = match workload.query_workload.language { + QueryLanguage::PromQL => lower_promql_roots(&workload)?, + _ => lower_sql_roots(&workload, &example2_catalog())?, + }; let result = run_planning_stages(&workload, roots.clone(), Some(max_combinations)); document_of(workload_queries(&workload), &roots, result, max_displayed)? } diff --git a/crates/devtools/src/planner_layering_examples.rs b/crates/devtools/src/planner_layering_examples.rs index 6789bd995..a964b956a 100644 --- a/crates/devtools/src/planner_layering_examples.rs +++ b/crates/devtools/src/planner_layering_examples.rs @@ -1,4 +1,4 @@ -//! The workloads of #509 Examples 1, 3 and 4 +//! The workloads of #509 Examples 1, 2, 3 and 4 //! (`docs/design_docs/proposals/planner-layering.md`), shared by //! `stage_pipeline` and the planner-layering acceptance tests. @@ -7,7 +7,7 @@ use asap_types::workload::{ AccuracyRequirement, BatchEntry, DataArrival, DataDistribution, DataWorkload, DurationMs, Evidence, EvidenceSource, LatencyRequirement, MetricType, PlanningWorkload, Predictability, Query, QueryLanguage, QueryRequirements, QueryTimeScope, QueryWorkload, Rate, RepeatedDemand, - RepeatingEntry, RepetitionInterval, TimeSelection, TimestampMs, + RepeatingEntry, RepetitionInterval, SqlDialect, TimeSelection, TimestampMs, }; pub fn declared(value: T) -> Evidence { @@ -88,6 +88,78 @@ pub fn example1() -> PlanningWorkload { } } +/// Example 2's three SQL queries over the last minute of `flows`: the +/// distinct sources, the entropy of the per-source counts and their L2 +/// norm. Q3 casts the counts to DOUBLE: the design's Int64 `c * c` can +/// overflow, so the SQL frontend recognizes only the Float64 product. +pub const EXAMPLE2_SQL: [&str; 3] = [ + "SELECT COUNT(DISTINCT src_ip) FROM flows WHERE ts >= now() - INTERVAL '1 minute'", + "SELECT -SUM(p * LN(p)) FROM (\ + SELECT COUNT(*) * 1.0 / SUM(COUNT(*)) OVER () AS p FROM flows \ + WHERE ts >= now() - INTERVAL '1 minute' GROUP BY src_ip)", + "SELECT SQRT(SUM(CAST(c AS DOUBLE) * CAST(c AS DOUBLE))) FROM (\ + SELECT src_ip, COUNT(*) AS c FROM flows \ + WHERE ts >= now() - INTERVAL '1 minute' GROUP BY src_ip)", +]; + +/// (ε, δ) of Example 2's Q1, Q2 and Q3; Q3 is the strictest. +pub const EXAMPLE2_ACCURACY: [(f64, f64); 3] = [(0.02, 0.01), (0.05, 0.01), (0.01, 0.01)]; + +/// Example 2's `flows(ts, src_ip)` table: the design names only these +/// two columns. +pub fn example2_catalog() -> asap_frontend_sql::SqlCatalog { + use asap_types::pre_asap::schema::{DataType, Field, Schema}; + asap_frontend_sql::SqlCatalog::new().with_table( + "flows", + Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("src_ip", DataType::Utf8, false), + ], + 0, + vec![], + ), + ) +} + +/// Example 2: three dashboard panels over one flow stream, every 10 s +/// over the last minute, with 10,000,000 distinct sources. +pub fn example2() -> PlanningWorkload { + PlanningWorkload { + query_workload: QueryWorkload { + language: QueryLanguage::SQL(SqlDialect::DataFusionSQL), + query_batch: None, + repeating_queries: Some( + EXAMPLE2_SQL + .iter() + .zip(EXAMPLE2_ACCURACY) + .map(|(query, (epsilon, delta))| RepeatingEntry { + query: Query((*query).into()), + demand: RepeatedDemand::FixedInterval(RepetitionInterval(10_000)), + requirements: QueryRequirements { + accuracy: AccuracyRequirement::Explicit( + RequestedAccuracy::epsilon_delta(epsilon, delta), + ), + response_latency: LatencyRequirement::Unspecified, + }, + predictability: Predictability::Predictable { known_at: None }, + time_selection: TimeSelection { + scope: QueryTimeScope::RealTime, + lookback: Some(DurationMs(MINUTE_MS)), + as_of: None, + }, + }) + .collect(), + ), + }, + data_workload: Some(DataWorkload { + data_ingestion_interval: Evidence::default(), + input_cardinality: declared(10_000_000), + ..shared_data_workload(DataArrival::ContinuouslyIngesting) + }), + } +} + /// Pattern A's five queries: (PromQL, `lookback`, T − `as_of`). pub const PATTERN_A: [(&str, u64, u64); 5] = [ ("quantile_over_time(0.99, latency_ms[5y])", 5 * YEAR_MS, 0), diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index 17fc08ef0..869915abc 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -39,21 +39,31 @@ pub fn lower_promql_roots(workload: &PlanningWorkload) -> Result, .collect() } -/// Lower a SQL workload over the tables of `catalog`. +/// Lower a SQL workload over the tables of `catalog`: every entry, batch +/// and repeating, in workload order, each with its own accuracy target. pub fn lower_sql_roots( workload: &PlanningWorkload, catalog: &asap_frontend_sql::SqlCatalog, ) -> Result, String> { - tokio::runtime::Builder::new_current_thread() + let dialect = match &workload.query_workload.language { + asap_types::workload::QueryLanguage::SQL(dialect) => dialect.clone(), + _ => asap_types::workload::SqlDialect::DataFusionSQL, + }; + let runtime = tokio::runtime::Builder::new_current_thread() .build() - .map_err(|e| e.to_string())? - .block_on(asap_frontend_sql::lower_sql_batch( - &workload.query_workload, - catalog, - )) - .into_iter() - .map(|root| { - root.map(QueryRoot::Operator) + .map_err(|e| e.to_string())?; + workload + .query_workload + .entries() + .map(|entry| { + runtime + .block_on(asap_frontend_sql::lower_sql_dialect( + &entry.query.0, + catalog, + dialect.clone(), + entry.requirements.accuracy.requested(), + )) + .map(QueryRoot::Operator) .map_err(|e| format!("lowering: {e}")) }) .collect() diff --git a/crates/devtools/tests/stage_pipeline.rs b/crates/devtools/tests/stage_pipeline.rs index 27184d8bb..c8cdda2e7 100644 --- a/crates/devtools/tests/stage_pipeline.rs +++ b/crates/devtools/tests/stage_pipeline.rs @@ -1,4 +1,4 @@ -//! `stage_pipeline` writes a valid four-stage viewer document for #509 Example 1. +//! `stage_pipeline` writes valid four-stage viewer documents for #509's examples. use std::collections::HashSet; use std::path::PathBuf; use std::process::Command; @@ -210,6 +210,42 @@ fn example3a_selects_the_cheapest_of_all_candidates() { assert!(physical.iter().any(|p| p["id"] == expected.as_str())); } +/// Example 2 plans its three SQL queries: Stage 1 lists a candidate in +/// which one UnivMon build serves all three, and plan selection selects a +/// plan. +#[test] +fn example2_plans_the_sql_workload_with_one_univmon_for_all_three() { + let document = generate_example("planner-layering-2", &["--max-displayed", "100000"]); + let queries = document["workload"]["queries"].as_array().unwrap(); + assert_eq!(queries.len(), 3); + assert!(queries.iter().all(|q| q["language"] == "sql")); + validated_dag(&document["stage0_query_language_frontends"]["dag"], 3); + let univmon_builds = |candidate: &Value| { + candidate["dag"]["nodes"] + .as_array() + .unwrap() + .iter() + .filter(|n| { + n["operator"]["ASAP"]["SummaryAgg"]["family"]["Sketch"][0]["algorithm"] == "UnivMon" + }) + .count() + }; + let candidates = document["stage1_logical_asap_optimization"]["candidates"] + .as_array() + .unwrap(); + assert!( + candidates.iter().any(|c| { + c["display_name"] + .as_str() + .unwrap() + .starts_with("Q1 UnivMon · Q2 UnivMon (Count acc) · Q3 UnivMon (Count acc)") + && univmon_builds(c) == 1 + }), + "no candidate reads one UnivMon for all three" + ); + assert!(document["stage3_plan_selection"]["selected"].is_string()); +} + /// SQL queries over tables declared with `--table` are planned through the /// same stages, and the document records each query's language. #[test] diff --git a/docs/user_guide_docs/stage-viewer.md b/docs/user_guide_docs/stage-viewer.md index 14024c8f0..34d668796 100644 --- a/docs/user_guide_docs/stage-viewer.md +++ b/docs/user_guide_docs/stage-viewer.md @@ -69,7 +69,7 @@ plan wins. | Tab | Workload | What to look for | | --- | --- | --- | | 1 · dashboard panels | Two PromQL panels over 1M counter series every 10 s: an exact per-job rate, and the approximate top-10 series per job | The exact query stays exact; the CMS+heap and CountSketch+heap top-k plans are rejected with their reasons, so the all-exact plan wins; the plans that share the two queries' scan and range selector tie with it, and the independent plan is kept on enumeration order | -| 2 · SQL flow statistics | Distinct sources, entropy and L2 of per-source counts over the last minute of flows | All 64 plans compose and are priced; none is pruned. The SQL frontend does not yet recognize entropy and L2, so Q2 and Q3 are per-source counts with exact, Count-Min, CountSketch or UnivMon options. 56 plans are invalid: HLL for Q1 because its bound has "failure probability None", and UnivMon for Q1 or Count-Min or CountSketch for Q2 and Q3 because "the output has no guarantee, and the deployment has no evidence that it meets target". 8 are valid: Q1 exact or KMV with Q2 and Q3 each exact or UnivMon. Theta is not offered: `asap_sketchlib` has no Theta sketch. P1 (all exact) is selected at 8.64 per second, tied with its shared-input plans; KMV on Q1 costs more | +| 2 · SQL flow statistics | Distinct sources, entropy and L2 of per-source counts over the last minute of flows | All 82 plans compose and are priced; none is pruned. The SQL frontend recognizes Q2 as entropy and Q3 (its counts cast to DOUBLE) as L2, each with an exact or UnivMon option and an exact population count for the empty-input NULL. 16 plans come from Pass 1. Pass 2 adds the plans that share identical nodes and, by the summary-capability rule, the plans in which one UnivMon serves two of the queries or all three, sized with each parameter's largest value among them. 76 plans are invalid: HLL or UnivMon for Q1 (50) because its bound has "failure probability None" (both need the source's distinct bound, which this example does not declare); UnivMon for Q2 (20) because entropy has no certified bound; and UnivMon for Q3 (6) because "the output has no guarantee" (the L2 estimate is certified, but no guarantee rule covers the join with the exact population count). 6 are valid: Q1 exact or KMV, Q2 and Q3 exact, each alone or sharing identical nodes. Theta is not offered: `asap_sketchlib` has no Theta sketch. P1 (all exact) is selected at 6.08 per second; KMV on Q1 costs the same to three decimals; the shared plans cost 7.44, because a shared node that is not materialized is charged once per query that reads it | | 3a · historical p99 batch | Five p99 reports over 1–5 years, run once | All 972 plans are priced: 243 Pass 1 plans, each with 3 shared-input plans. Each query builds its own sketch at query time. P485 is selected: KLL for all five queries. DDSketch in place of KLL ties, and the tie goes to enumeration order. The all-exact plan costs about 16% more | | 3b · live p99 panel | p99 over the last 5 min every minute, 1M series | P2, KLL, is selected at 2.100 per second, tied with DDSketch and chosen on enumeration order; exact over the window's raw samples costs 2.400. Memory a query holds only while it runs is not charged. Everything runs at query time and nothing is materialized. KLL panes kept from ingestion time and the 200 ms latency check come with the window-summary PRs | | 4a · monthly p99 reports | Example 3a repeated monthly | The same plan, with its cost amortized over a month | From a8922afb7ba26affcd8d8b2db58a6579e470fb1f Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:12:48 +0000 Subject: [PATCH 08/14] feat(logical-asap-optimization): guarantees through a join beside exact inputs and an exactly decided CASE SQL Example 2's L2 decides the empty-input NULL from an exact population count cross-joined beside the estimate and a CASE on that count. No guarantee rule covered the join or the CASE, so every SQL plan with UnivMon for Q3 was rejected ("the output has no guarantee"), although the L2 estimate is certified. - A join (inner, outer or cross) whose inputs are exact but one keeps that input's guarantee when its predicate reads only exact columns: which rows match, and how often, then does not depend on an estimate. A key on the approximate input's columns, a semi- or anti-join, or two approximate inputs have none. - A CASE whose operand and conditions are exact keeps its weakest branch's guarantee; a literal or NULL branch, or a missing ELSE, is exact. A condition over an approximate value has none. - A projection over such a join composes each output column this way. Guarantees are per node, so every column of the approximate input counts as approximate; column-level tracking (#689) stays the general solution. A root's default metric (#690) looks through the same shape: a projection whose CASE value branches read columns (or literals, cast or not) takes those columns' metric, following them through joins and row-preserving operators to an aggregate's measure, so SQL's L2 root resolves to a relative target as the PromQL one does. Any other expression stays absolute. --- .../src/per_computation_candidates.rs | 307 +++++++++++++++++- .../tests/planner_layering_example2.rs | 13 + crates/types/src/types.rs | 161 +++++++++ 3 files changed, 480 insertions(+), 1 deletion(-) diff --git a/crates/asap-aware-mapping/src/per_computation_candidates.rs b/crates/asap-aware-mapping/src/per_computation_candidates.rs index 63d405771..fa7920559 100644 --- a/crates/asap-aware-mapping/src/per_computation_candidates.rs +++ b/crates/asap-aware-mapping/src/per_computation_candidates.rs @@ -978,7 +978,10 @@ fn composed_guarantee(node: &OperatorNode) -> Option { Operator::NonASAP(_) if node.children().into_iter().all(|child| exact(child)) => { Some(AccuracyGuarantee::exact("exact over exact inputs")) } - Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) => project_guarantee(cols, child), + Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) => { + project_guarantee(cols, child).or_else(|| relational_project_guarantee(cols, child)) + } + Operator::NonASAP(NonASAPOp::Join { .. }) => join_guarantee(node), // A measure filter or `HAVING` over approximate values changes // which rows or groups count: no rule. Operator::NonASAP(NonASAPOp::Aggregate { @@ -1221,6 +1224,150 @@ fn project_guarantee(cols: &[ProjectItem], child: &Rc) -> Option Option> { + let width = node.schema.fields.len(); + if let Some(guarantee) = node.guarantee.as_ref().filter(|g| g.is_exact()) { + return Some(vec![guarantee.clone(); width]); + } + match &node.operator { + Operator::NonASAP(NonASAPOp::Join { + kind: + JoinKind::Inner | JoinKind::Left | JoinKind::Right | JoinKind::Full | JoinKind::Cross, + left, + right, + .. + }) => { + let input = join_input_guarantees(left, right)?; + (input.len() == width).then_some(input) + } + Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) => { + let child = column_guarantees(child)?; + cols.iter() + .map(|item| match item.expr { + ScalarExpr::Column(id) => child.get(id).cloned(), + _ => None, + }) + .collect() + } + _ => None, + } +} + +/// The guarantees of a join's input columns (`left ++ right`) when at most +/// one input is approximate and the others are exact: the exact input's +/// columns exact, the approximate input's columns its guarantee. +fn join_input_guarantees( + left: &OperatorNode, + right: &OperatorNode, +) -> Option> { + let exact = |node: &OperatorNode| node.guarantee.as_ref().is_some_and(|g| g.is_exact()); + if !exact(left) && !exact(right) { + return None; + } + let side = |node: &OperatorNode| { + let guarantee = node.guarantee.clone()?; + Some(vec![guarantee; node.schema.fields.len()]) + }; + let mut columns = side(left)?; + columns.extend(side(right)?); + Some(columns) +} + +/// A join whose inputs are exact but one keeps that input's guarantee when +/// its predicate reads only exact columns (a cross join, or a key equality +/// on the exact inputs' columns): which rows match, and how often, then +/// does not depend on an estimate, and each output row carries one row of +/// the approximate input unchanged. Outer joins pad on the same exact +/// decision. A semi- or anti-join, a key on an approximate input's column +/// (per-node guarantees count all its columns approximate; #689), or two +/// approximate inputs have none. An exact join is composed before this. +fn join_guarantee(node: &OperatorNode) -> Option { + let Operator::NonASAP(NonASAPOp::Join { + kind: JoinKind::Inner | JoinKind::Left | JoinKind::Right | JoinKind::Full | JoinKind::Cross, + pred, + left, + right, + }) = &node.operator + else { + return None; + }; + let columns = join_input_guarantees(left, right)?; + let pred_exact = + scalar_guarantee(&pred.0, &|id| columns.get(id).cloned()).is_some_and(|g| g.is_exact()); + if !pred_exact { + return None; + } + [left, right] + .into_iter() + .find_map(|input| input.guarantee.clone().filter(|g| !g.is_exact())) +} + +/// The guarantee of a projection over relational columns +/// ([`column_guarantees`]): each output column's as [`scalar_guarantee`] +/// (a `CASE` as [`case_guarantee`]), and the node's the weakest of them. +/// `None` when the child's column guarantees are unknown or any output +/// column has none. +fn relational_project_guarantee( + cols: &[ProjectItem], + child: &Rc, +) -> Option { + let columns = column_guarantees(child)?; + let column = |id: usize| columns.get(id).cloned(); + weakest( + cols.iter() + .map(|item| scalar_guarantee(&item.expr, &column)) + .collect::>>()?, + ) +} + +/// The weakest of `guarantees`: exact when all are, the approximate one +/// when there is one (or all approximate ones are equal), and otherwise +/// as an extremum of them ([`lipschitz_guarantee`]). +fn weakest(guarantees: Vec) -> Option { + let mut approximate: Vec = Vec::new(); + for guarantee in guarantees.into_iter().filter(|g| !g.is_exact()) { + if !approximate.contains(&guarantee) { + approximate.push(guarantee); + } + } + match approximate.len() { + 0 => Some(AccuracyGuarantee::exact("exact over exact operands")), + 1 => approximate.pop(), + _ => lipschitz_guarantee(approximate), + } +} + +/// A `CASE` whose operand and conditions are exact picks its branch by an +/// exact decision, so it keeps the weakest branch's guarantee (a literal or +/// NULL branch, or a missing `ELSE`, is exact). A condition over an +/// approximate value has none. +fn case_guarantee( + operand: Option<&ScalarExpr>, + branches: &[(ScalarExpr, ScalarExpr)], + else_expr: Option<&ScalarExpr>, + column: &dyn Fn(usize) -> Option, +) -> Option { + let exact = |expr: &ScalarExpr| scalar_guarantee(expr, column).is_some_and(|g| g.is_exact()); + if !operand.is_none_or(exact) || !branches.iter().all(|(when, _)| exact(when)) { + return None; + } + weakest( + branches + .iter() + .map(|(_, then)| then) + .chain(else_expr) + .map(|expr| scalar_guarantee(expr, column)) + .collect::>>()?, + ) +} + /// The guarantee of `column` of a projection's `child`: the child's sample /// value has the child's as an operand ([`operand_guarantee`]); other /// columns (labels, timestamps) are exact. `None` when the child has no @@ -1267,6 +1414,11 @@ pub(crate) fn scalar_guarantee( ScalarExpr::Negative { expr, .. } => { lipschitz_guarantee(vec![scalar_guarantee(expr, column)?]) } + ScalarExpr::Case { + operand, + branches, + else_expr, + } => case_guarantee(operand.as_deref(), branches, else_expr.as_deref(), column), ScalarExpr::Arithmetic { op, left, right, .. } => arithmetic_guarantee( @@ -2760,6 +2912,159 @@ mod tests { assert!(close(delta(&vector(&shared, &shared)), 0.001)); assert!(close(delta(&vector(&shared, &other)), 0.002)); } + + /// `left JOIN right ON pred`, composed. + fn join( + kind: JoinKind, + left: &Rc, + right: &Rc, + pred: ScalarExpr, + ) -> OperatorNode { + let mut node = OperatorNode::new(Operator::NonASAP(NonASAPOp::Join { + kind, + pred: Predicate(pred), + left: left.clone(), + right: right.clone(), + })) + .unwrap(); + node.guarantee = composed_guarantee(&node); + node + } + fn column(id: usize) -> ScalarExpr { + ScalarExpr::Column(id) + } + fn equals(left: ScalarExpr, right: ScalarExpr) -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(left), + op: CompareOpKind::Eq, + right: Box::new(right), + semantics: ExprSemantics::Sql, + } + } + /// `CASE WHEN condition THEN NULL ELSE otherwise END` over `child`, + /// composed. + fn case_over( + child: &OperatorNode, + condition: ScalarExpr, + otherwise: ScalarExpr, + ) -> Option { + let node = OperatorNode::new(Operator::NonASAP(NonASAPOp::Project { + cols: vec![ProjectItem { + alias: Some("v".into()), + expr: ScalarExpr::Case { + operand: None, + branches: vec![(condition, ScalarExpr::Literal(ScalarValue::Null))], + else_expr: Some(Box::new(otherwise)), + }, + }], + qualifier: None, + child: Rc::new(child.clone()), + })) + .unwrap(); + composed_guarantee(&node) + } + /// An approximate node and an exact one, and the approximate one's + /// column count (its columns come first in a join). + fn approximate_and_exact() -> (Rc, Rc, usize) { + let approximate = node(rel(0.01)); + let exact = node_over("count(m)", AccuracyGuarantee::exact("test")); + let width = approximate.schema.fields.len(); + (approximate, exact, width) + } + + /// A join whose other input is exact and whose predicate reads only + /// exact columns keeps the approximate input's guarantee: a cross + /// join, or a key equality on the exact input's columns. + #[test] + fn join_beside_an_exact_input_on_exact_columns_keeps_the_guarantee() { + let (approximate, exact, width) = approximate_and_exact(); + for (kind, pred) in [ + ( + JoinKind::Cross, + ScalarExpr::Literal(ScalarValue::Boolean(true)), + ), + (JoinKind::Inner, equals(column(width), lit(0.0))), + (JoinKind::Left, equals(column(width), lit(0.0))), + ] { + assert_eq!( + join(kind.clone(), &approximate, &exact, pred).guarantee, + Some(rel(0.01)), + "{kind:?}" + ); + } + // Either side. + assert_eq!( + join( + JoinKind::Cross, + &exact, + &approximate, + ScalarExpr::Literal(ScalarValue::Boolean(true)) + ) + .guarantee, + Some(rel(0.01)) + ); + } + + /// A join keyed on an approximate input's column, a semi-join, or + /// two approximate inputs have no guarantee (column-level tracking + /// is #689). + #[test] + fn join_on_an_approximate_key_has_no_guarantee() { + let (approximate, exact, width) = approximate_and_exact(); + assert_eq!( + join( + JoinKind::Inner, + &approximate, + &exact, + equals(column(0), column(width)) + ) + .guarantee, + None + ); + assert_eq!( + join( + JoinKind::Semi, + &exact, + &approximate, + ScalarExpr::Literal(ScalarValue::Boolean(true)) + ) + .guarantee, + None + ); + assert_eq!( + join( + JoinKind::Cross, + &approximate, + &approximate, + ScalarExpr::Literal(ScalarValue::Boolean(true)) + ) + .guarantee, + None + ); + } + + /// A `CASE` whose condition reads only exact columns keeps each + /// branch's guarantee (a NULL branch is exact); one testing an + /// approximate column has none. + #[test] + fn case_on_exact_columns_keeps_the_branch_guarantee() { + let (approximate, exact, width) = approximate_and_exact(); + let joined = join( + JoinKind::Cross, + &approximate, + &exact, + ScalarExpr::Literal(ScalarValue::Boolean(true)), + ); + let value = approximate.schema.fields.len() - 1; + assert_eq!( + case_over(&joined, equals(column(width), lit(0.0)), column(value)), + Some(rel(0.01)) + ); + assert_eq!( + case_over(&joined, equals(column(value), lit(0.0)), column(value)), + None + ); + } } /// Which stream norm bounds a Count-Min or Count Sketch estimate (#669). diff --git a/crates/integration-tests/tests/planner_layering_example2.rs b/crates/integration-tests/tests/planner_layering_example2.rs index c18f5a6a6..c6440f1a9 100644 --- a/crates/integration-tests/tests/planner_layering_example2.rs +++ b/crates/integration-tests/tests/planner_layering_example2.rs @@ -758,6 +758,19 @@ fn stage3_plan_selection_accepts_univmon_l2() { } } +/// SQL Q3's L2 through UnivMon keeps its guarantee: the exact population +/// count joined beside it (a cross join, no key) and the `CASE` deciding the +/// empty-input NULL on that exact count keep the estimate's guarantee, so +/// plan selection accepts it. +#[tokio::test] +async fn stage3_plan_selection_accepts_sql_univmon_l2_beside_its_exact_population_count() { + let workload = sql_workload(); + let run = run_stages(&workload, lower_sql(&workload).await); + for id in univmon_only_candidates(&run, 2) { + assert!(run.cost(id).is_some(), "{id}: {:?}", run.invalid().get(id)); + } +} + /// One materialized UnivMon for all three costs no more than three separate UnivMons. #[test] #[ignore = "needs a UnivMon entropy accuracy model and materializing the shared UnivMon (#604): not materialized, it is charged once per consuming query"] diff --git a/crates/types/src/types.rs b/crates/types/src/types.rs index 4de62c478..09fa74f89 100644 --- a/crates/types/src/types.rs +++ b/crates/types/src/types.rs @@ -394,6 +394,16 @@ fn node_metric(node: &OperatorNode) -> Result { node_metric(child) } + // A `CASE` choosing between columns and literals, e.g. SQL's + // empty-input NULL guard over a statistic cross-joined with an exact + // count: the metric of the columns its value branches read. + NonASAPOp::Project { .. } => (0..node.schema.fields.len()) + .map(|column| column_metric(node, column)) + .collect::, _>>() + .map(|metrics| match metrics.split_first() { + Some((first, rest)) if rest.iter().all(|metric| metric == first) => *first, + _ => ErrorMetric::ABSOLUTE, + }), NonASAPOp::Aggregate { measures, .. } => { let mut metrics = Vec::new(); for intent in measures { @@ -422,6 +432,83 @@ fn node_metric(node: &OperatorNode) -> Result } } +/// The metric of `node`'s output `column`, following columns through +/// projections, `CASE` value branches, joins and operators that only +/// select, order or rename rows, down to an aggregate's measure. A group +/// key or any other expression is absolute. +fn column_metric(node: &OperatorNode, column: usize) -> Result { + use crate::ir::operator_properties::JoinKind; + let Operator::NonASAP(op) = &node.operator else { + return Ok(ErrorMetric::ABSOLUTE); + }; + match op { + NonASAPOp::Filter { child, .. } + | NonASAPOp::Sort { child, .. } + | NonASAPOp::Limit { child, .. } + | NonASAPOp::Dedup { child, .. } => column_metric(child, column), + NonASAPOp::Project { cols, child, .. } => match cols.get(column).map(|item| &item.expr) { + Some(ScalarExpr::Column(id)) => column_metric(child, *id), + Some(ScalarExpr::Case { + branches, + else_expr, + .. + }) => { + let mut metrics = Vec::new(); + for value in branches + .iter() + .map(|(_, then)| then) + .chain(else_expr.as_deref()) + { + match value { + // A literal, possibly cast (SQL's typed NULL). + ScalarExpr::Literal(_) => {} + ScalarExpr::Cast { expr, .. } + if matches!(**expr, ScalarExpr::Literal(_)) => {} + ScalarExpr::Column(id) => metrics.push(column_metric(child, *id)?), + _ => return Ok(ErrorMetric::ABSOLUTE), + } + } + Ok(match metrics.split_first() { + Some((first, rest)) if rest.iter().all(|metric| metric == first) => *first, + _ => ErrorMetric::ABSOLUTE, + }) + } + _ => Ok(ErrorMetric::ABSOLUTE), + }, + NonASAPOp::Join { + kind: + JoinKind::Inner | JoinKind::Left | JoinKind::Right | JoinKind::Full | JoinKind::Cross, + left, + right, + .. + } => { + let width = left.schema.fields.len(); + if column < width { + column_metric(left, column) + } else { + column_metric(right, column - width) + } + } + NonASAPOp::Aggregate { measures, .. } => { + // Group keys first, then one column per measure. + let keys = node.schema.fields.len().saturating_sub(measures.len()); + match column.checked_sub(keys).and_then(|m| measures.get(m)) { + Some(AggIntent::Quantile { .. }) => { + Err(AccuracyRequestError::MissingQuantileMetric { intent: "quantile" }) + } + Some(AggIntent::HistogramQuantile { .. }) => { + Err(AccuracyRequestError::MissingQuantileMetric { + intent: "histogram_quantile", + }) + } + Some(intent) => Ok(intent_metric(intent).unwrap_or(ErrorMetric::ABSOLUTE)), + None => Ok(ErrorMetric::ABSOLUTE), + } + } + _ => Ok(ErrorMetric::ABSOLUTE), + } +} + #[cfg(test)] mod tests { use super::*; @@ -572,6 +659,80 @@ mod tests { ); } + /// SQL's empty-input NULL guard (a statistic cross-joined with an exact + /// row count, and `CASE WHEN count = 0 THEN NULL ELSE statistic END`) + /// keeps the statistic's metric: the root's value is the statistic's + /// column. A `CASE` whose value branches read columns of different + /// metrics, or any other expression, is absolute. + #[test] + fn root_metric_looks_through_a_null_guard_to_its_statistic() { + use crate::ir::operator_properties::JoinKind; + use crate::ir::{Predicate, ProjectItem}; + use crate::pre_asap::{CompareOpKind, ScalarValue}; + let exact = AccuracyTarget::exact(); + let QueryRoot::Operator(l2) = aggregate(vec![AggIntent::FrequencyL2 { + col: None, + accuracy: exact.clone(), + }]) else { + unreachable!() + }; + let QueryRoot::Operator(count) = aggregate(vec![AggIntent::Count { accuracy: exact }]) + else { + unreachable!() + }; + let joined = node(NonASAPOp::Join { + kind: JoinKind::Cross, + pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), + left: l2, + right: count, + }); + let guard = |otherwise: ScalarExpr| { + QueryRoot::Operator(node(NonASAPOp::Project { + cols: vec![ProjectItem { + alias: Some("v".into()), + expr: ScalarExpr::Case { + operand: None, + branches: vec![( + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(1)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(0))), + semantics: crate::ir::ExprSemantics::Sql, + }, + ScalarExpr::Cast { + expr: Box::new(ScalarExpr::Literal(ScalarValue::Null)), + to: DataType::Float64, + try_cast: false, + }, + )], + else_expr: Some(Box::new(otherwise)), + }, + }], + qualifier: None, + child: joined.clone(), + })) + }; + let metric = |root: QueryRoot| { + RequestedAccuracy::epsilon(0.01) + .for_root(&root) + .unwrap() + .metric + }; + assert_eq!(metric(guard(ScalarExpr::Column(0))), ErrorMetric::Relative); + // The count's column, or arithmetic over the statistic. + assert_eq!( + metric(guard(ScalarExpr::Column(1))), + ErrorMetric::ABSOLUTE_STREAM_L1 + ); + assert_eq!( + metric(guard(ScalarExpr::Negative { + expr: Box::new(ScalarExpr::Column(0)), + semantics: crate::ir::ExprSemantics::Sql, + })), + ErrorMetric::ABSOLUTE + ); + } + /// A quantile root's request must name its metric; a named metric or an /// exact request needs no default. #[test] From 7320486a3ea7cc275305a07bccb29b26cace0cbf Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:12:48 +0000 Subject: [PATCH 09/14] docs(stage-viewer): regenerate the Example 2 row with SQL L2 through UnivMon With the join and CASE guarantee rules, the SQL Q3 UnivMon plans are valid: 12 of the 82 plans are valid (Q3 exact or UnivMon), 70 invalid (Q1 HLL or UnivMon without a distinct bound, Q2 UnivMon without an entropy bound). P1 (all exact) is still selected at 6.08 per second; UnivMon on Q3 costs 8.08. --- docs/user_guide_docs/stage-viewer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user_guide_docs/stage-viewer.md b/docs/user_guide_docs/stage-viewer.md index 34d668796..3043c00b2 100644 --- a/docs/user_guide_docs/stage-viewer.md +++ b/docs/user_guide_docs/stage-viewer.md @@ -69,7 +69,7 @@ plan wins. | Tab | Workload | What to look for | | --- | --- | --- | | 1 · dashboard panels | Two PromQL panels over 1M counter series every 10 s: an exact per-job rate, and the approximate top-10 series per job | The exact query stays exact; the CMS+heap and CountSketch+heap top-k plans are rejected with their reasons, so the all-exact plan wins; the plans that share the two queries' scan and range selector tie with it, and the independent plan is kept on enumeration order | -| 2 · SQL flow statistics | Distinct sources, entropy and L2 of per-source counts over the last minute of flows | All 82 plans compose and are priced; none is pruned. The SQL frontend recognizes Q2 as entropy and Q3 (its counts cast to DOUBLE) as L2, each with an exact or UnivMon option and an exact population count for the empty-input NULL. 16 plans come from Pass 1. Pass 2 adds the plans that share identical nodes and, by the summary-capability rule, the plans in which one UnivMon serves two of the queries or all three, sized with each parameter's largest value among them. 76 plans are invalid: HLL or UnivMon for Q1 (50) because its bound has "failure probability None" (both need the source's distinct bound, which this example does not declare); UnivMon for Q2 (20) because entropy has no certified bound; and UnivMon for Q3 (6) because "the output has no guarantee" (the L2 estimate is certified, but no guarantee rule covers the join with the exact population count). 6 are valid: Q1 exact or KMV, Q2 and Q3 exact, each alone or sharing identical nodes. Theta is not offered: `asap_sketchlib` has no Theta sketch. P1 (all exact) is selected at 6.08 per second; KMV on Q1 costs the same to three decimals; the shared plans cost 7.44, because a shared node that is not materialized is charged once per query that reads it | +| 2 · SQL flow statistics | Distinct sources, entropy and L2 of per-source counts over the last minute of flows | All 82 plans compose and are priced; none is pruned. The SQL frontend recognizes Q2 as entropy and Q3 (its counts cast to DOUBLE) as L2, each with an exact or UnivMon option and an exact population count for the empty-input NULL. 16 plans come from Pass 1. Pass 2 adds the plans that share identical nodes and, by the summary-capability rule, the plans in which one UnivMon serves two of the queries or all three, sized with each parameter's largest value among them. 70 plans are invalid: HLL or UnivMon for Q1 (50) because its bound has "failure probability None" (both need the source's distinct bound, which this example does not declare), and UnivMon for Q2 (20) because entropy has no certified bound. 12 are valid: Q1 exact or KMV, Q2 exact, and Q3 exact or UnivMon, each alone or sharing identical nodes. Q3's UnivMon keeps its L2 guarantee through the exact population count joined beside it and the `CASE` that decides the empty-input NULL on that count. Theta is not offered: `asap_sketchlib` has no Theta sketch. P1 (all exact) is selected at 6.08 per second; KMV on Q1 costs the same to three decimals, UnivMon on Q3 costs 8.08, and the shared plans cost 7.44 or more, because a shared node that is not materialized is charged once per query that reads it | | 3a · historical p99 batch | Five p99 reports over 1–5 years, run once | All 972 plans are priced: 243 Pass 1 plans, each with 3 shared-input plans. Each query builds its own sketch at query time. P485 is selected: KLL for all five queries. DDSketch in place of KLL ties, and the tie goes to enumeration order. The all-exact plan costs about 16% more | | 3b · live p99 panel | p99 over the last 5 min every minute, 1M series | P2, KLL, is selected at 2.100 per second, tied with DDSketch and chosen on enumeration order; exact over the window's raw samples costs 2.400. Memory a query holds only while it runs is not charged. Everything runs at query time and nothing is materialized. KLL panes kept from ingestion time and the 200 ms latency check come with the window-summary PRs | | 4a · monthly p99 reports | Example 3a repeated monthly | The same plan, with its cost amortized over a month | From b429e6e3da3dde096329c5f7fa1247314c9c7303 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:17:37 +0000 Subject: [PATCH 10/14] fix(physical-operators): charge UnivMon key memory without walking its heaps A build charges a state's memory after every input row, and UnivMon's charge summed the bytes of every key on every layer's heap, so building was quadratic in the heap: a 20,000-key heap on 19 layers over 40,000 keys took 3.3 s in release. The charge now bounds the retained keys by every heap slot holding the longest key seen (tracked on insert, merge and adoption of a decoded sketch), in constant time. A test checks that the charge does not grow with heap occupancy; a timing test (ignored, release) checks the large build. --- .../src/summary_kernels/univmon.rs | 79 ++++++++++++++++--- 1 file changed, 68 insertions(+), 11 deletions(-) diff --git a/crates/asap-physical-operators/src/summary_kernels/univmon.rs b/crates/asap-physical-operators/src/summary_kernels/univmon.rs index 128e7ab6a..60b6c2376 100644 --- a/crates/asap-physical-operators/src/summary_kernels/univmon.rs +++ b/crates/asap-physical-operators/src/summary_kernels/univmon.rs @@ -40,18 +40,25 @@ pub(crate) fn check_dimensions( #[derive(Debug, Clone)] pub struct UnivMonAccumulator { inner: UnivMon, + /// The longest variable-length key the heaps may retain, in bytes: the + /// memory charge bounds the retained keys by every heap slot holding + /// one this long, so it never walks the heaps (a build charges memory + /// after every row). + max_key_bytes: usize, } impl UnivMonAccumulator { /// Empty the sketch in place, keeping its shape. pub(crate) fn clear(&mut self) { self.inner.free(); + self.max_key_bytes = 0; } pub fn new(heap_size: usize, rows: usize, cols: usize, layers: usize) -> Result { check_dimensions(heap_size, rows, cols, layers)?; Ok(Self { inner: UnivMon::init_univmon(heap_size, rows, cols, layers), + max_key_bytes: 0, }) } @@ -68,7 +75,20 @@ impl UnivMonAccumulator { if !sketch.accepts_standard_updates() { return Err("terminal-mode UnivMon state cannot accept standard updates".into()); } - Ok(Self { inner: sketch }) + // One walk of the decoded heaps, not one per charge. + let max_key_bytes = (0..sketch.layer_size) + .flat_map(|layer| sketch.hh_layers[layer].heap()) + .map(|item| match &item.key { + asap_sketchlib::HeapItem::String(key) => key.capacity(), + asap_sketchlib::HeapItem::Bytes(key) => key.capacity(), + _ => 0, + }) + .max() + .unwrap_or(0); + Ok(Self { + inner: sketch, + max_key_bytes, + }) } /// The underlying sketch, so a deployment can encode it with sketchlib's codec. @@ -108,6 +128,7 @@ impl UnivMonAccumulator { .checked_add(1) .ok_or("UnivMon count overflow")?; self.inner.insert(&DataInput::Bytes(&key), 1); + self.max_key_bytes = self.max_key_bytes.max(key.len()); Ok(()) } @@ -143,22 +164,22 @@ impl UnivMonAccumulator { .checked_add(other.inner.bucket_size) .ok_or("UnivMon count overflow")?; self.inner.merge(&other.inner); + self.max_key_bytes = self.max_key_bytes.max(other.max_key_bytes); Ok(()) } } impl AggregateCore for UnivMonAccumulator { - /// Counters and heap slots of every layer, plus the bytes of the - /// variable-length keys the heaps retain. + /// Counters and heap slots of every layer, plus an upper bound on the + /// variable-length keys the heaps retain: every slot holding the + /// longest key seen. Constant time, since a build charges memory after + /// every row. fn approx_memory_bytes(&self) -> usize { - let key_bytes = (0..self.inner.layer_size) - .flat_map(|layer| self.inner.hh_layers[layer].heap()) - .map(|item| match &item.key { - asap_sketchlib::HeapItem::String(key) => key.capacity(), - asap_sketchlib::HeapItem::Bytes(key) => key.capacity(), - _ => 0, - }) - .fold(0usize, usize::saturating_add); + let key_bytes = self + .inner + .layer_size + .saturating_mul(self.inner.heap_size) + .saturating_mul(self.max_key_bytes); key_bytes .saturating_add(std::mem::size_of::()) .saturating_add( @@ -342,4 +363,40 @@ mod tests { state.clear(); assert_eq!(state.approx_memory_bytes(), empty); } + + // A build charges memory after every row, so the charge must not walk + // the heaps: with keys of one length it stays the same however many + // keys the heaps hold. + #[test] + fn memory_charge_does_not_grow_with_heap_occupancy() { + let mut state = UnivMonAccumulator::new(512, 3, 64, 4).unwrap(); + let key = |i: i64| crate::values::Value::Utf8(format!("{i:08}").into()); + state.insert_value(&key(0)).unwrap(); + let one = state.approx_memory_bytes(); + for i in 1..2000 { + state.insert_value(&key(i)).unwrap(); + } + assert_eq!(state.approx_memory_bytes(), one); + } + + // Charging memory after every row of a large build stays linear in the + // rows (it was quadratic: 3 s for this build). Timing-based, so ignored + // by default; run with `--ignored --release`. + #[test] + #[ignore = "timing; run in release"] + fn per_row_memory_charge_of_a_large_build_is_fast() { + let mut state = UnivMonAccumulator::new(20_000, 3, 1024, 19).unwrap(); + let start = std::time::Instant::now(); + let mut charged = 0usize; + for i in 0..40_000i64 { + state.insert_value(&crate::values::Value::Int64(i)).unwrap(); + charged = charged.max(state.approx_memory_bytes()); + } + assert!(charged > 0); + assert!( + start.elapsed() < std::time::Duration::from_millis(500), + "{:?}", + start.elapsed() + ); + } } From a53489361ba9c4744123d624b5e69ef86a13a6e1 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:24:26 +0000 Subject: [PATCH 11/14] fix(logical-asap-optimization): summary-capability review fixes - A capability group of more than 63 builds is an error: one 64-bit mask enumerates the subsets of at most 63, and the shift was clamped, so a larger group was enumerated wrongly (and took 2^63 steps) instead of refused. - A summary-capability variant whose guarantee composition fails carries its own error, so Stage 1 lists it as a candidate that cannot be composed and keeps the remaining variants; one failure used to drop them all. - shared_size_params no longer claims every sizing bound is monotone (a Count Sketch's collision bound is tighter at a power-of-two width than one column wider); the shared state is checked by composing every guarantee above it again. --- .../src/accuracy/estimators/mod.rs | 8 ++- .../asap-aware-mapping/src/asap_aware_cse.rs | 60 +++++++++++++++---- .../src/pass/stage1_to_stage3.rs | 11 +++- 3 files changed, 65 insertions(+), 14 deletions(-) diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/mod.rs b/crates/asap-aware-mapping/src/accuracy/estimators/mod.rs index 516512f71..3f6dfcec5 100644 --- a/crates/asap-aware-mapping/src/accuracy/estimators/mod.rs +++ b/crates/asap-aware-mapping/src/accuracy/estimators/mod.rs @@ -170,8 +170,12 @@ pub(crate) fn column_collision(cols: u32) -> f64 { /// each sized alone with one of `params` (Pass 2's summary-capability /// rule): each parameter's largest value, except where a smaller value is /// stricter, where it takes the smallest (a KLL's failure probability, -/// DDSketch's α). Every sizing formula is monotonic in its parameters, so -/// the shared state meets each consumer's target. `None` for no parameters, +/// DDSketch's α). Not every bound is monotone in its parameters (a Count +/// Sketch's collision bound is tighter at a power-of-two width than one +/// column wider), so the shared state is not assumed to meet each +/// consumer's target: the summary-capability rule composes every guarantee +/// above it again, and plan selection checks each consumer against the +/// shared state. `None` for no parameters, /// parameters of different algorithms, or a shared UnivMon the library /// would not build ([`univmon::hash_independent`]). pub(crate) fn shared_size_params(params: &[&SketchParams]) -> Option { diff --git a/crates/asap-aware-mapping/src/asap_aware_cse.rs b/crates/asap-aware-mapping/src/asap_aware_cse.rs index 35e23b3d8..0a223e4a6 100644 --- a/crates/asap-aware-mapping/src/asap_aware_cse.rs +++ b/crates/asap-aware-mapping/src/asap_aware_cse.rs @@ -304,11 +304,12 @@ pub struct SharedSummary { /// A Pass 2 candidate of the summary-capability rule: the candidate's /// roots with the builds in `shared` made one build each, sized for all -/// their consumers, and every guarantee composed again. -#[derive(Debug, Clone)] +/// their consumers, and every guarantee composed again; the error when +/// that composition fails, which leaves the other variants unaffected. +#[derive(Debug)] pub struct SummaryCapabilityVariant { pub shared: Vec, - pub roots: Vec, + pub roots: Result, PerComputationCandidateError>, } /// The summary-capability groups of the candidate with query roots @@ -430,19 +431,27 @@ fn all_equal<'a>(mut nodes: impl Iterator>) -> bool /// with query roots `roots` and groups `groups` ([`capability_groups`]): /// one per choice, for each group, of no sharing or one subset of two or /// more members whose sizes are not all equal, other than sharing nothing. +/// An error only for a group of more than 63 members; a variant whose +/// composition fails carries its own error. /// The first group varies slowest; within a group, subsets come in /// binary-counter order (the first member the lowest bit). pub fn summary_capability_variants( roots: &[QueryRoot], groups: &[CapabilityGroup], ) -> Result, PerComputationCandidateError> { + // One 64-bit mask enumerates the subsets of at most 63 members. + if groups.iter().any(|group| group.members.len() > 63) { + return Err(PerComputationCandidateError::Unsupported( + "a summary-capability group of more than 63 summary builds", + )); + } // Each group's options: `None` (no sharing), then its subsets. let options: Vec>>> = groups .iter() .map(|group| { let n = group.members.len(); let mut options = vec![None]; - for mask in 1u64..(1u64 << n.min(63)) { + for mask in 1u64..(1u64 << n) { let members: Vec = (0..n).filter(|&m| mask >> m & 1 == 1).collect(); if members.len() >= 2 && !all_equal(members.iter().map(|&m| &group.members[m])) { options.push(Some(members)); @@ -476,8 +485,16 @@ pub fn summary_capability_variants( .map(|members| SharedSummary { group, members }) }) .collect(); - if let Some(roots) = share_summaries(roots, groups, &shared)? { - variants.push(SummaryCapabilityVariant { shared, roots }); + match share_summaries(roots, groups, &shared) { + Ok(None) => {} + Ok(Some(roots)) => variants.push(SummaryCapabilityVariant { + shared, + roots: Ok(roots), + }), + Err(error) => variants.push(SummaryCapabilityVariant { + shared, + roots: Err(error), + }), } } Ok(variants) @@ -908,11 +925,14 @@ mod tests { members: vec![0, 1] }] ); - let shared = kll_builds(&variants[0].roots); + let shared = kll_builds(variants[0].roots.as_ref().unwrap()); assert_eq!(shared.len(), 1, "one KLL serves both"); assert_eq!(shared[0].1, own[1].1); - assert!(root_bound(&variants[0].roots[0]) < root_bound(&roots[0])); - assert_eq!(root_bound(&variants[0].roots[1]), root_bound(&roots[1])); + assert!(root_bound(&variants[0].roots.as_ref().unwrap()[0]) < root_bound(&roots[0])); + assert_eq!( + root_bound(&variants[0].roots.as_ref().unwrap()[1]), + root_bound(&roots[1]) + ); } /// Three consumers at three targets: every subset of two or more of @@ -938,11 +958,31 @@ mod tests { [vec![0, 1], vec![0, 2], vec![1, 2], vec![0, 1, 2]] ); for variant in &variants { - let builds = kll_builds(&variant.roots); + let builds = kll_builds(variant.roots.as_ref().unwrap()); assert_eq!(builds.len(), 4 - variant.shared[0].members.len()); } } + /// A group of more than 63 members has more subsets than one 64-bit + /// mask enumerates: an error, not a silent skip. + #[test] + fn a_group_past_63_members_is_an_error() { + let roots = chose( + &[ + ("quantile_over_time(0.5, lat[5m])", 0.01), + ("quantile_over_time(0.99, lat[5m])", 0.001), + ], + SketchAlgorithm::Kll, + ); + let mut groups = capability_groups(&roots); + let members = groups[0].members.clone(); + groups[0].members = members.iter().cycle().take(64).cloned().collect(); + assert!(matches!( + summary_capability_variants(&roots, &groups), + Err(PerComputationCandidateError::Unsupported(_)) + )); + } + /// Equal targets give identical builds, which are the /// identical-expression rule's; a different window or family is a /// different input or capability, never shared. diff --git a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs index 4c6a817b2..bb1c2fa20 100644 --- a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs +++ b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs @@ -290,8 +290,15 @@ fn pass2_candidates(roots: &[QueryRoot]) -> Vec { format!("{:?} for {}", group.family, queries.join(", ")) }) .collect(); - found.push(&summaries, Vec::new(), &variant.roots); - found.identical_variants(&summaries, &variant.roots); + match variant.roots { + Ok(roots) => { + found.push(&summaries, Vec::new(), &roots); + found.identical_variants(&summaries, &roots); + } + // Listed as a candidate that cannot be composed; the other + // variants go on. + Err(error) => found.out.push((summaries, Vec::new(), Err(error))), + } } found.out } From 250e0687cb6f6716931de35baf707cb70a5427e7 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:22:52 +0000 Subject: [PATCH 12/14] fix(logical-asap-optimization): read each column's guarantee in a projection over a join A projection's guarantee assumed the error lies only in its child's sample-value column (the column named "value", or the only numeric one). Over a join that column can be an exact input's, so the approximate input's columns were read as exact: projecting sum(m) * 1000 beside the exact side's value composed to the approximate side's absolute error 1, while the truth is 1000. A projection over a join (directly, or through projections, filters, sorts, limits or deduplication) now reads each column's own guarantee (column_guarantees), and a projection over a computed projection over a join, whose columns' guarantees are unknown, has none. Plan selection's operand-domain lookup reads columns the same way. column_guarantees and join_guarantee document their assumption that every approximate node of Stage 1 has an exact row set. Per-column tracking in general is #689. --- .../src/per_computation_candidates.rs | 103 ++++++++++++++++-- 1 file changed, 95 insertions(+), 8 deletions(-) diff --git a/crates/asap-aware-mapping/src/per_computation_candidates.rs b/crates/asap-aware-mapping/src/per_computation_candidates.rs index fa7920559..440eb56f8 100644 --- a/crates/asap-aware-mapping/src/per_computation_candidates.rs +++ b/crates/asap-aware-mapping/src/per_computation_candidates.rs @@ -978,9 +978,12 @@ fn composed_guarantee(node: &OperatorNode) -> Option { Operator::NonASAP(_) if node.children().into_iter().all(|child| exact(child)) => { Some(AccuracyGuarantee::exact("exact over exact inputs")) } - Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) => { - project_guarantee(cols, child).or_else(|| relational_project_guarantee(cols, child)) + // Over a join, the child's sample value may be an exact input's + // column, so each column's own guarantee is read instead. + Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) if derives_from_join(child) => { + relational_project_guarantee(cols, child) } + Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) => project_guarantee(cols, child), Operator::NonASAP(NonASAPOp::Join { .. }) => join_guarantee(node), // A measure filter or `HAVING` over approximate values changes // which rows or groups count: no rule. @@ -1224,13 +1227,37 @@ fn project_guarantee(cols: &[ProjectItem], child: &Rc) -> Option bool { + match node.non_asap() { + Some(NonASAPOp::Join { .. }) => true, + Some( + NonASAPOp::Project { child, .. } + | NonASAPOp::Filter { child, .. } + | NonASAPOp::Sort { child, .. } + | NonASAPOp::Limit { child, .. } + | NonASAPOp::Dedup { child, .. }, + ) => derives_from_join(child), + _ => false, + } +} + /// The guarantee of each output column of `node`, where it is known: every /// column of an exact node is exact; a join beside exact inputs /// ([`join_guarantee`]) gives the exact inputs' columns exact and the /// approximate input's columns its guarantee; a projection of columns maps -/// its child's. `None` anywhere else. Guarantees are per node, so every -/// column of an approximate input counts as approximate; column-level -/// tracking is #689. +/// its child's. `None` anywhere else, a computed projection included. +/// Guarantees are per node, so every column of an approximate input counts +/// as approximate; column-level tracking is #689. +/// +/// This assumes every approximate node of Stage 1 has an exact row set: +/// its guarantee bounds each row's values, never which rows exist (a +/// top-k estimate, whose rows are approximate, has no bound; +/// `estimate_has_bound`). A guarantee on row membership would need these +/// rules revisited. fn column_guarantees(node: &OperatorNode) -> Option> { let width = node.schema.fields.len(); if let Some(guarantee) = node.guarantee.as_ref().filter(|g| g.is_exact()) { @@ -1288,6 +1315,8 @@ fn join_input_guarantees( /// decision. A semi- or anti-join, a key on an approximate input's column /// (per-node guarantees count all its columns approximate; #689), or two /// approximate inputs have none. An exact join is composed before this. +/// Like [`column_guarantees`], this assumes the approximate input's row set +/// is exact: only its values carry error. fn join_guarantee(node: &OperatorNode) -> Option { let Operator::NonASAP(NonASAPOp::Join { kind: JoinKind::Inner | JoinKind::Left | JoinKind::Right | JoinKind::Full | JoinKind::Cross, @@ -1368,14 +1397,19 @@ fn case_guarantee( ) } -/// The guarantee of `column` of a projection's `child`: the child's sample -/// value has the child's as an operand ([`operand_guarantee`]); other +/// The guarantee of `column` of a projection's `child`: over a join, the +/// column's own ([`column_guarantees`]); otherwise the child's sample +/// value has the child's as an operand ([`operand_guarantee`]) and other /// columns (labels, timestamps) are exact. `None` when the child has no -/// sample value. +/// sample value or the column's guarantee is unknown. pub(crate) fn project_column_guarantee( child: &Rc, column: usize, ) -> Option { + // Over a join, each column's own guarantee ([`column_guarantees`]). + if derives_from_join(child) { + return column_guarantees(child)?.get(column).cloned(); + } let value = resolve_column_ref(&ColumnRef::SampleValue, &child.schema).ok()?; if column == value { operand_guarantee(child) @@ -3065,6 +3099,59 @@ mod tests { None ); } + + /// `node` projecting `cols` over `child`, composed. + fn project(child: &OperatorNode, cols: Vec) -> OperatorNode { + let mut node = OperatorNode::new(Operator::NonASAP(NonASAPOp::Project { + cols: cols + .into_iter() + .enumerate() + .map(|(i, expr)| ProjectItem { + alias: Some(format!("c{i}")), + expr, + }) + .collect(), + qualifier: None, + child: Rc::new(child.clone()), + })) + .unwrap(); + node.guarantee = composed_guarantee(&node); + node + } + + /// Over a join, a projection reads each column's own guarantee, + /// not the child's sample value (which may be an exact input's + /// column): `sum(m) * 1000` beside the exact side's `value` has + /// the scaled error 1000, not the approximate side's 1. A + /// projection over that computed projection has no per-column + /// guarantees, so it has none (#689). + #[test] + fn projection_over_a_join_reads_each_columns_guarantee() { + let approximate = node(abs(1.0, 0.01)); + let exact = node_over("m", AccuracyGuarantee::exact("test")); + let joined = join( + JoinKind::Cross, + &approximate, + &exact, + ScalarExpr::Literal(ScalarValue::Boolean(true)), + ); + assert_eq!(joined.guarantee, Some(abs(1.0, 0.01))); + let sum = approximate.schema.fields.len() - 1; + let value = approximate.schema.fields.len() + exact.schema.fields.len() - 1; + let scaled = project( + &joined, + vec![ + arith(ArithmeticOpKind::Mul, column(sum), lit(1000.0)), + column(value), + ], + ); + assert_eq!(scaled.guarantee.as_ref().map(bound), Some(1000.0)); + let stacked = project( + &scaled, + vec![arith(ArithmeticOpKind::Add, column(1), lit(1.0)), column(0)], + ); + assert_eq!(stacked.guarantee, None); + } } /// Which stream norm bounds a Count-Min or Count Sketch estimate (#669). From ac6d02d3cbc39ce4421ddf59d7e426accbd3920b Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:22:52 +0000 Subject: [PATCH 13/14] fix(logical-asap-optimization): refuse summary-capability groups of more than 20 builds Every subset of two or more builds of a group is a candidate, 2^n of them; past 20 builds (about a million) the rule returns an error naming the limit instead of enumerating. --- .../asap-aware-mapping/src/asap_aware_cse.rs | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/crates/asap-aware-mapping/src/asap_aware_cse.rs b/crates/asap-aware-mapping/src/asap_aware_cse.rs index 0a223e4a6..2187df33f 100644 --- a/crates/asap-aware-mapping/src/asap_aware_cse.rs +++ b/crates/asap-aware-mapping/src/asap_aware_cse.rs @@ -427,11 +427,15 @@ fn all_equal<'a>(mut nodes: impl Iterator>) -> bool nodes.all(|node| Rc::ptr_eq(node, first) || **node == **first) } +/// The most summary builds one summary-capability group may have: every +/// subset of two or more is a candidate, about a million at 20. +pub const MAX_GROUP_MEMBERS: usize = 20; + /// Every Pass 2 candidate of the summary-capability rule for the candidate /// with query roots `roots` and groups `groups` ([`capability_groups`]): /// one per choice, for each group, of no sharing or one subset of two or /// more members whose sizes are not all equal, other than sharing nothing. -/// An error only for a group of more than 63 members; a variant whose +/// An error only for a group of more than [`MAX_GROUP_MEMBERS`] members; a variant whose /// composition fails carries its own error. /// The first group varies slowest; within a group, subsets come in /// binary-counter order (the first member the lowest bit). @@ -439,10 +443,14 @@ pub fn summary_capability_variants( roots: &[QueryRoot], groups: &[CapabilityGroup], ) -> Result, PerComputationCandidateError> { - // One 64-bit mask enumerates the subsets of at most 63 members. - if groups.iter().any(|group| group.members.len() > 63) { + // Every subset of a group is a candidate: 2^n of them. Past + // MAX_GROUP_MEMBERS that is no longer a list plan selection can price. + if groups + .iter() + .any(|group| group.members.len() > MAX_GROUP_MEMBERS) + { return Err(PerComputationCandidateError::Unsupported( - "a summary-capability group of more than 63 summary builds", + "a summary-capability group of more than 20 summary builds (2^n sharing subsets)", )); } // Each group's options: `None` (no sharing), then its subsets. @@ -963,10 +971,10 @@ mod tests { } } - /// A group of more than 63 members has more subsets than one 64-bit - /// mask enumerates: an error, not a silent skip. + /// A group of more than 20 members (over a million sharing subsets) + /// is an error, not a silent skip or an endless enumeration. #[test] - fn a_group_past_63_members_is_an_error() { + fn a_group_past_20_members_is_an_error() { let roots = chose( &[ ("quantile_over_time(0.5, lat[5m])", 0.01), @@ -976,7 +984,12 @@ mod tests { ); let mut groups = capability_groups(&roots); let members = groups[0].members.clone(); - groups[0].members = members.iter().cycle().take(64).cloned().collect(); + groups[0].members = members + .iter() + .cycle() + .take(MAX_GROUP_MEMBERS + 1) + .cloned() + .collect(); assert!(matches!( summary_capability_variants(&roots, &groups), Err(PerComputationCandidateError::Unsupported(_)) From 463a8e1b8785ab264cdf2e65fec972368c899033 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:22:53 +0000 Subject: [PATCH 14/14] fix(physical-operators): bound UnivMon key memory by the key bytes inserted Charging every heap slot at the longest key seen overcounts badly: one 1 KB key in a sketch of 19 layers of 20,000 slots was charged 380 MB. Each layer's heap keeps at most one copy of a key, so the retained keys are also bounded by the layers times the bytes of every key inserted (or merged in); the charge takes the smaller bound, still in constant time. asap_sketchlib exposes no eviction hook for an exact running total. --- .../src/summary_kernels/univmon.rs | 63 ++++++++++++++----- 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/crates/asap-physical-operators/src/summary_kernels/univmon.rs b/crates/asap-physical-operators/src/summary_kernels/univmon.rs index 60b6c2376..bd8db2027 100644 --- a/crates/asap-physical-operators/src/summary_kernels/univmon.rs +++ b/crates/asap-physical-operators/src/summary_kernels/univmon.rs @@ -45,6 +45,10 @@ pub struct UnivMonAccumulator { /// one this long, so it never walks the heaps (a build charges memory /// after every row). max_key_bytes: usize, + /// The bytes of every variable-length key inserted (or merged in), + /// counted once per insertion: each layer's heap keeps at most one copy + /// of a key, so the heaps retain at most `layers` times this. + inserted_key_bytes: usize, } impl UnivMonAccumulator { @@ -52,6 +56,7 @@ impl UnivMonAccumulator { pub(crate) fn clear(&mut self) { self.inner.free(); self.max_key_bytes = 0; + self.inserted_key_bytes = 0; } pub fn new(heap_size: usize, rows: usize, cols: usize, layers: usize) -> Result { @@ -59,6 +64,7 @@ impl UnivMonAccumulator { Ok(Self { inner: UnivMon::init_univmon(heap_size, rows, cols, layers), max_key_bytes: 0, + inserted_key_bytes: 0, }) } @@ -76,18 +82,19 @@ impl UnivMonAccumulator { return Err("terminal-mode UnivMon state cannot accept standard updates".into()); } // One walk of the decoded heaps, not one per charge. - let max_key_bytes = (0..sketch.layer_size) + let keys: Vec = (0..sketch.layer_size) .flat_map(|layer| sketch.hh_layers[layer].heap()) .map(|item| match &item.key { asap_sketchlib::HeapItem::String(key) => key.capacity(), asap_sketchlib::HeapItem::Bytes(key) => key.capacity(), _ => 0, }) - .max() - .unwrap_or(0); + .collect(); Ok(Self { inner: sketch, - max_key_bytes, + max_key_bytes: keys.iter().copied().max().unwrap_or(0), + // The retained keys themselves (already counted per layer). + inserted_key_bytes: keys.iter().fold(0, |a, &b| a.saturating_add(b)), }) } @@ -129,6 +136,7 @@ impl UnivMonAccumulator { .ok_or("UnivMon count overflow")?; self.inner.insert(&DataInput::Bytes(&key), 1); self.max_key_bytes = self.max_key_bytes.max(key.len()); + self.inserted_key_bytes = self.inserted_key_bytes.saturating_add(key.len()); Ok(()) } @@ -165,21 +173,26 @@ impl UnivMonAccumulator { .ok_or("UnivMon count overflow")?; self.inner.merge(&other.inner); self.max_key_bytes = self.max_key_bytes.max(other.max_key_bytes); + self.inserted_key_bytes = self + .inserted_key_bytes + .saturating_add(other.inserted_key_bytes); Ok(()) } } impl AggregateCore for UnivMonAccumulator { /// Counters and heap slots of every layer, plus an upper bound on the - /// variable-length keys the heaps retain: every slot holding the - /// longest key seen. Constant time, since a build charges memory after - /// every row. + /// variable-length keys the heaps retain: the smaller of every slot + /// holding the longest key seen and every layer holding a copy of every + /// key inserted. Constant time, since a build charges memory after + /// every row; the library exposes no eviction hook for an exact running + /// total, so a stream of long keys is still overcounted. fn approx_memory_bytes(&self) -> usize { - let key_bytes = self - .inner - .layer_size + let layers = self.inner.layer_size; + let key_bytes = layers .saturating_mul(self.inner.heap_size) - .saturating_mul(self.max_key_bytes); + .saturating_mul(self.max_key_bytes) + .min(layers.saturating_mul(self.inserted_key_bytes)); key_bytes .saturating_add(std::mem::size_of::()) .saturating_add( @@ -365,18 +378,34 @@ mod tests { } // A build charges memory after every row, so the charge must not walk - // the heaps: with keys of one length it stays the same however many - // keys the heaps hold. + // the heaps: once the keys inserted outnumber the heap slots, keys of + // one length are charged the same however many keys the heaps hold. #[test] fn memory_charge_does_not_grow_with_heap_occupancy() { let mut state = UnivMonAccumulator::new(512, 3, 64, 4).unwrap(); let key = |i: i64| crate::values::Value::Utf8(format!("{i:08}").into()); - state.insert_value(&key(0)).unwrap(); - let one = state.approx_memory_bytes(); - for i in 1..2000 { + for i in 0..1000 { state.insert_value(&key(i)).unwrap(); } - assert_eq!(state.approx_memory_bytes(), one); + let full = state.approx_memory_bytes(); + for i in 1000..4000 { + state.insert_value(&key(i)).unwrap(); + } + assert_eq!(state.approx_memory_bytes(), full); + } + + // One long key in a large sketch is charged at most once per layer (a + // key's heap copies), not once per heap slot (19 layers of 20,000 + // slots of 1 KB would be 380 MB). + #[test] + fn one_long_key_is_charged_about_once() { + let mut state = UnivMonAccumulator::new(20_000, 3, 64, 19).unwrap(); + let empty = state.approx_memory_bytes(); + state + .insert_value(&crate::values::Value::Utf8("x".repeat(1024).into())) + .unwrap(); + let charged = state.approx_memory_bytes() - empty; + assert!((1024..=19 * 1100).contains(&charged), "{charged}"); } // Charging memory after every row of a large build stays linear in the