diff --git a/crates/asap-aware-mapping/src/accuracy/estimators/mod.rs b/crates/asap-aware-mapping/src/accuracy/estimators/mod.rs index 922385dcd..3f6dfcec5 100644 --- a/crates/asap-aware-mapping/src/accuracy/estimators/mod.rs +++ b/crates/asap-aware-mapping/src/accuracy/estimators/mod.rs @@ -128,6 +128,7 @@ pub(crate) fn size_params( } } } + /// `P(Binomial(n, p) ≥ k)`, summed term by term in log space and rounded /// up by a relative 1e-12 so rounding never certifies a smaller tail. pub(crate) fn binomial_upper_tail(n: u32, p: f64, k: u32) -> f64 { @@ -165,6 +166,135 @@ pub(crate) fn column_collision(cols: u32) -> f64 { } } +/// The parameters of one state shared by consumers of the same algorithm, +/// each sized alone with one of `params` (Pass 2's summary-capability +/// rule): each parameter's largest value, except where a smaller value is +/// stricter, where it takes the smallest (a KLL's failure probability, +/// DDSketch's α). Not every bound is monotone in its parameters (a Count +/// Sketch's collision bound is tighter at a power-of-two width than one +/// column wider), so the shared state is not assumed to meet each +/// consumer's target: the summary-capability rule composes every guarantee +/// above it again, and plan selection checks each consumer against the +/// shared state. `None` for no parameters, +/// parameters of different algorithms, or a shared UnivMon the library +/// would not build ([`univmon::hash_independent`]). +pub(crate) fn shared_size_params(params: &[&SketchParams]) -> Option { + use SketchParams as P; + let (first, rest) = params.split_first()?; + let shared = rest.iter().try_fold((*first).clone(), |a, b| { + Some(match (a, *b) { + ( + P::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, + distinct_failure_probability, + }, + P::UnivMon { + heap_size: h, + sketch_rows: r, + sketch_cols: c, + layers: l, + distinct_failure_probability: f, + }, + ) => P::UnivMon { + heap_size: heap_size.max(*h), + sketch_rows: sketch_rows.max(*r), + sketch_cols: sketch_cols.max(*c), + layers: layers.max(*l), + // `None`: not sized for a distinct count. + distinct_failure_probability: min_recorded(distinct_failure_probability, *f), + }, + ( + P::Kll { + k, + failure_probability, + }, + P::Kll { + k: k2, + failure_probability: f2, + }, + ) => P::Kll { + k: k.max(*k2), + // `None` is the calibration's δ, above any recorded one. + failure_probability: min_recorded(failure_probability, *f2), + }, + (P::Cms { width, depth }, P::Cms { width: w, depth: d }) => P::Cms { + width: width.max(*w), + depth: depth.max(*d), + }, + (P::CountSketch { width, depth }, P::CountSketch { width: w, depth: d }) => { + P::CountSketch { + width: width.max(*w), + depth: depth.max(*d), + } + } + ( + P::CmsWithHeap { + width, + depth, + heap_size, + }, + P::CmsWithHeap { + width: w, + depth: d, + heap_size: h, + }, + ) => P::CmsWithHeap { + width: width.max(*w), + depth: depth.max(*d), + heap_size: heap_size.max(*h), + }, + ( + P::CountSketchWithHeap { + width, + depth, + heap_size, + }, + P::CountSketchWithHeap { + width: w, + depth: d, + heap_size: h, + }, + ) => P::CountSketchWithHeap { + width: width.max(*w), + depth: depth.max(*d), + heap_size: heap_size.max(*h), + }, + (P::Hll { precision }, P::Hll { precision: p }) => P::Hll { + precision: precision.max(*p), + }, + (P::DDSketch { alpha }, P::DDSketch { alpha: a }) => P::DDSketch { + alpha: alpha.min(*a), + }, + (P::Kmv { k }, P::Kmv { k: k2 }) => P::Kmv { k: k.max(*k2) }, + (P::Theta { k }, P::Theta { k: k2 }) => P::Theta { k: k.max(*k2) }, + _ => return None, + }) + })?; + // The largest rows of one UnivMon with the largest columns of another + // can pass the hash budget `asap_sketchlib` builds within. + match shared { + P::UnivMon { + sketch_rows, + sketch_cols, + layers, + .. + } if !univmon::hash_independent(sketch_rows, sketch_cols, layers) => None, + shared => Some(shared), + } +} + +/// The smaller of two recorded failure probabilities; an unrecorded one +/// (`None`) yields to a recorded one. +fn min_recorded(a: Option, b: Option) -> Option { + match (a, b) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + } +} + /// Accuracy-dependent candidate budget, not a completeness theorem. The /// membership model must still certify the selected set independently. pub(crate) fn topk_capacity(k: usize, eps: f64) -> u32 { @@ -181,3 +311,97 @@ pub(crate) fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { } (x.ceil() as u32).clamp(lo, hi) } + +#[cfg(test)] +mod shared_size_tests { + use super::*; + + /// The shared size takes each parameter's largest value, and the + /// smallest where smaller is stricter (KLL δ, DDSketch α); sizes of + /// different algorithms have none. + #[test] + fn shared_size_takes_each_parameters_strictest_value() { + assert_eq!( + shared_size_params(&[&univmon(256, 5, 1024, 16), &univmon(1, 7, 4096, 1)]), + Some(univmon(256, 7, 4096, 16)) + ); + // The distinct count's failure probability, where one was sized + // for, takes the smallest. + let distinct = |heap_size, delta| SketchParams::UnivMon { + heap_size, + sketch_rows: 1, + sketch_cols: 1, + layers: 19, + distinct_failure_probability: Some(delta), + }; + let Some(SketchParams::UnivMon { + heap_size, + distinct_failure_probability, + .. + }) = shared_size_params(&[ + &univmon(256, 5, 1024, 16), + &distinct(4096, 0.01), + &distinct(2048, 0.001), + ]) + else { + unreachable!() + }; + assert_eq!( + (heap_size, distinct_failure_probability), + (4096, Some(0.001)) + ); + let kll = |k, failure_probability| SketchParams::Kll { + k, + failure_probability, + }; + let SketchParams::Kll { + k, + failure_probability, + } = shared_size_params(&[ + &kll(200, None), + &kll(100, Some(0.001)), + &kll(150, Some(0.005)), + ]) + .unwrap() + else { + unreachable!() + }; + assert_eq!((k, failure_probability), (200, Some(0.001))); + assert_eq!( + shared_size_params(&[ + &SketchParams::DDSketch { alpha: 0.01 }, + &SketchParams::DDSketch { alpha: 0.001 } + ]), + Some(SketchParams::DDSketch { alpha: 0.001 }) + ); + assert_eq!( + shared_size_params(&[&kll(200, None), &SketchParams::Hll { precision: 12 }]), + None + ); + assert_eq!(shared_size_params(&[]), None); + } + + fn univmon(heap_size: u32, sketch_rows: u32, sketch_cols: u32, layers: u8) -> SketchParams { + SketchParams::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, + distinct_failure_probability: None, + } + } + + /// Each UnivMon fits `asap_sketchlib`'s hash budget, but the largest + /// rows of one with the largest columns of the other do not + /// (`12 · (⌈log2 2^16⌉ + 1) = 204 > 128`): the library would not build + /// the shared shape, so there is none. + #[test] + fn no_shared_univmon_past_the_hash_budget() { + assert!(univmon::hash_independent(12, 512, 1)); + assert!(univmon::hash_independent(5, 1 << 16, 1)); + assert_eq!( + shared_size_params(&[&univmon(1, 12, 512, 1), &univmon(1, 5, 1 << 16, 1)]), + None + ); + } +} diff --git a/crates/asap-aware-mapping/src/asap_aware_cse.rs b/crates/asap-aware-mapping/src/asap_aware_cse.rs index 359dccf1a..2187df33f 100644 --- a/crates/asap-aware-mapping/src/asap_aware_cse.rs +++ b/crates/asap-aware-mapping/src/asap_aware_cse.rs @@ -2,7 +2,12 @@ //! common-subexpression elimination, as defined in //! `docs/design_docs/proposals/planner-layering.md` (Algorithm 2). //! -//! Only the identical-expression rule is implemented. A *sharing group* of a +//! Two of its rules are implemented: the identical-expression rule and the +//! summary-capability rule. +//! +//! # Identical-expression rule +//! +//! A *sharing group* of a //! whole-workload candidate is a set of two or more of its nodes with //! identical input and computation semantics that are legal to share: the //! `ir::cse` interning merges only ASAP nodes and nodes with a unique key. @@ -17,7 +22,26 @@ //! same candidate as the subset with that group; such a candidate is //! returned once, under the subset of the groups it actually shares. //! -//! The summary-capability and window-composition rules are not implemented. +//! # Summary-capability rule +//! +//! Composition sizes each consumer's summary build alone, for its own +//! accuracy target. The *members* of a [`CapabilityGroup`] are summary +//! builds of one candidate that chose the same family (same +//! [`SketchAlgorithm`]) over the same input sub-DAG (window, `WHERE` and +//! time range included), item and weight, grouping and filter, and whose +//! every reader is an estimate the family answers +//! ([`FamilyCapability::answers`]); they differ only in their sizes. +//! [`summary_capability_variants`] returns one candidate per choice, for +//! each group, of no sharing or one subset of two or more of its members +//! whose sizes differ (Q1: partial sharing too): the subset becomes one +//! build serving all its readers, sized with each parameter's largest value +//! among them ([`shared_size_params`]: each consumer sized alone, the shared +//! build the per-parameter maximum, a KLL's failure probability the +//! minimum), and every guarantee above it is composed again, so plan +//! selection checks each consumer against the shared state. Members of +//! equal size are the identical-expression rule's; builds that are not +//! shared are untouched, and a consumer never changes family. The window- +//! composition rule is not implemented. use std::collections::HashMap; #[cfg(test)] @@ -25,7 +49,13 @@ use std::collections::HashSet; use std::rc::Rc; use asap_types::ir::cse::share_common_sub_dags; -use asap_types::ir::{OperatorNode, QueryRoot}; +use asap_types::ir::{ASAPOp, Operator, OperatorNode, QueryRoot}; +use asap_types::post_asap::{ + FamilyCapability, FieldDataType, GroupingStrategy, SketchAlgorithm, SketchKind, +}; + +use crate::accuracy::estimators::shared_size_params; +use crate::per_computation_candidates::{with_composed_guarantees, PerComputationCandidateError}; /// A Pass 2 candidate: the candidate's roots with the sharing groups /// `shared_groups` (indices into [`sharing_groups`]) shared. @@ -254,6 +284,354 @@ impl Sharing<'_> { } } +/// One group of the summary-capability rule: summary builds of one family +/// over the same input data that differ only in size, in the order a +/// children-first walk of the roots meets them. +#[derive(Debug, Clone)] +pub struct CapabilityGroup { + pub family: SketchAlgorithm, + pub members: Vec>, +} + +/// One shared build of a summary-capability candidate: the members +/// (indices into the group's [`CapabilityGroup::members`]) of group +/// `group` that read one state. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SharedSummary { + pub group: usize, + pub members: Vec, +} + +/// A Pass 2 candidate of the summary-capability rule: the candidate's +/// roots with the builds in `shared` made one build each, sized for all +/// their consumers, and every guarantee composed again; the error when +/// that composition fails, which leaves the other variants unaffected. +#[derive(Debug)] +pub struct SummaryCapabilityVariant { + pub shared: Vec, + pub roots: Result, PerComputationCandidateError>, +} + +/// The summary-capability groups of the candidate with query roots +/// `roots`: groups of at least two builds whose sizes are not all equal. +pub fn capability_groups(roots: &[QueryRoot]) -> Vec { + let mut nodes = Vec::new(); + let mut seen = std::collections::HashSet::new(); + for node in roots.iter().flat_map(root_operators) { + collect_children_first(node, &mut seen, &mut nodes); + } + // Every node that reads each build, by the build's address. + let mut readers: HashMap<*const OperatorNode, Vec<&Rc>> = HashMap::new(); + for node in &nodes { + for child in node.children() { + readers.entry(Rc::as_ptr(child)).or_default().push(node); + } + } + let mut groups: Vec = Vec::new(); + for node in &nodes { + let Some(family) = shareable_family(node, readers.get(&Rc::as_ptr(node))) else { + continue; + }; + match groups + .iter_mut() + .find(|g| g.family == family && same_input_data(&g.members[0], node)) + { + Some(group) => group.members.push(Rc::clone(node)), + None => groups.push(CapabilityGroup { + family, + members: vec![Rc::clone(node)], + }), + } + } + groups.retain(|g| !all_equal(g.members.iter())); + groups +} + +fn collect_children_first( + node: &Rc, + seen: &mut std::collections::HashSet<*const OperatorNode>, + out: &mut Vec>, +) { + if !seen.insert(Rc::as_ptr(node)) { + return; + } + for child in node.children() { + collect_children_first(child, seen, out); + } + out.push(Rc::clone(node)); +} + +/// The family of a per-subpopulation sketch build whose every reader is an +/// estimate the family answers; `None` for any other node. +fn shareable_family( + node: &OperatorNode, + readers: Option<&Vec<&Rc>>, +) -> Option { + let Some(ASAPOp::SummaryAgg { + family: family @ FieldDataType::Sketch(kind, GroupingStrategy::PerSubpopulationInstance), + .. + }) = node.asap() + else { + return None; + }; + let capability = FamilyCapability::of(family); + let answered = |reader: &&Rc| { + matches!( + reader.asap(), + Some(ASAPOp::SummaryEstimate { query, .. }) if capability.answers(query) + ) + }; + let readers = readers?; + readers + .iter() + .all(answered) + .then(|| kind.algorithm().clone()) +} + +/// Whether two builds of one family read the same data the same way: the +/// same input sub-DAG, item and weight, grouping and filter. +fn same_input_data(a: &OperatorNode, b: &OperatorNode) -> bool { + match (a.asap(), b.asap()) { + ( + Some(ASAPOp::SummaryAgg { + child, + input, + reduction, + grouping, + filter, + .. + }), + Some(ASAPOp::SummaryAgg { + child: child_b, + input: input_b, + reduction: reduction_b, + grouping: grouping_b, + filter: filter_b, + .. + }), + ) => { + (Rc::ptr_eq(child, child_b) || child == child_b) + && input == input_b + && reduction == reduction_b + && grouping == grouping_b + && filter == filter_b + } + _ => false, + } +} + +fn all_equal<'a>(mut nodes: impl Iterator>) -> bool { + let Some(first) = nodes.next() else { + return true; + }; + nodes.all(|node| Rc::ptr_eq(node, first) || **node == **first) +} + +/// The most summary builds one summary-capability group may have: every +/// subset of two or more is a candidate, about a million at 20. +pub const MAX_GROUP_MEMBERS: usize = 20; + +/// Every Pass 2 candidate of the summary-capability rule for the candidate +/// with query roots `roots` and groups `groups` ([`capability_groups`]): +/// one per choice, for each group, of no sharing or one subset of two or +/// more members whose sizes are not all equal, other than sharing nothing. +/// An error only for a group of more than [`MAX_GROUP_MEMBERS`] members; a variant whose +/// composition fails carries its own error. +/// The first group varies slowest; within a group, subsets come in +/// binary-counter order (the first member the lowest bit). +pub fn summary_capability_variants( + roots: &[QueryRoot], + groups: &[CapabilityGroup], +) -> Result, PerComputationCandidateError> { + // Every subset of a group is a candidate: 2^n of them. Past + // MAX_GROUP_MEMBERS that is no longer a list plan selection can price. + if groups + .iter() + .any(|group| group.members.len() > MAX_GROUP_MEMBERS) + { + return Err(PerComputationCandidateError::Unsupported( + "a summary-capability group of more than 20 summary builds (2^n sharing subsets)", + )); + } + // Each group's options: `None` (no sharing), then its subsets. + let options: Vec>>> = groups + .iter() + .map(|group| { + let n = group.members.len(); + let mut options = vec![None]; + for mask in 1u64..(1u64 << n) { + let members: Vec = (0..n).filter(|&m| mask >> m & 1 == 1).collect(); + if members.len() >= 2 && !all_equal(members.iter().map(|&m| &group.members[m])) { + options.push(Some(members)); + } + } + options + }) + .collect(); + let mut choice = vec![0usize; groups.len()]; + let mut variants = Vec::new(); + loop { + // Mixed-radix increment, the last group varying fastest. + let mut carried = true; + for (digit, group_options) in choice.iter_mut().zip(&options).rev() { + *digit += 1; + if *digit < group_options.len() { + carried = false; + break; + } + *digit = 0; + } + if carried { + break; + } + let shared: Vec = choice + .iter() + .enumerate() + .filter_map(|(group, &option)| { + options[group][option] + .clone() + .map(|members| SharedSummary { group, members }) + }) + .collect(); + match share_summaries(roots, groups, &shared) { + Ok(None) => {} + Ok(Some(roots)) => variants.push(SummaryCapabilityVariant { + shared, + roots: Ok(roots), + }), + Err(error) => variants.push(SummaryCapabilityVariant { + shared, + roots: Err(error), + }), + } + } + Ok(variants) +} + +/// `roots` with each of `shared` made one build sized for all its members, +/// and every guarantee composed again; `None` when the members' sizes have +/// no shared size. +fn share_summaries( + roots: &[QueryRoot], + groups: &[CapabilityGroup], + shared: &[SharedSummary], +) -> Result>, PerComputationCandidateError> { + let mut replaced: HashMap<*const OperatorNode, Rc> = HashMap::new(); + for summary in shared { + let members: Vec<&Rc> = summary + .members + .iter() + .map(|&m| &groups[summary.group].members[m]) + .collect(); + let Some(build) = sized_for_all(&members)? else { + return Ok(None); + }; + for member in members { + replaced.insert(Rc::as_ptr(member), Rc::clone(&build)); + } + } + let mut rebuilt: HashMap<*const OperatorNode, Rc> = HashMap::new(); + let mut rebuild = |node: &Rc| rebuild(node, &replaced, &mut rebuilt); + let mut out = Vec::new(); + for root in roots { + out.push(match root { + QueryRoot::Operator(node) => QueryRoot::Operator(rebuild(node)?), + QueryRoot::Scalar(expr) => { + let mut nodes = HashMap::new(); + for node in expr.operator_refs() { + nodes.insert(Rc::as_ptr(node), rebuild(node)?); + } + QueryRoot::Scalar( + expr.map_operator_refs(&mut |node| nodes[&Rc::as_ptr(node)].clone()), + ) + } + }); + } + Ok(Some(with_composed_guarantees(&out)?)) +} + +/// One build serving every member: the first member with the members' +/// shared size ([`shared_size_params`]). +fn sized_for_all( + members: &[&Rc], +) -> Result>, PerComputationCandidateError> { + let kinds: Vec<&SketchKind> = members + .iter() + .filter_map(|member| match member.asap() { + Some(ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(kind, _), + .. + }) => Some(kind), + _ => None, + }) + .collect(); + let params: Vec<_> = kinds.iter().map(|kind| kind.params()).collect(); + let Some(params) = shared_size_params(¶ms) else { + return Ok(None); + }; + let first = members[0]; + let Some(ASAPOp::SummaryAgg { + child, + family: FieldDataType::Sketch(kind, grouping_strategy), + input, + reduction, + grouping, + filter, + }) = first.asap() + else { + return Ok(None); + }; + let operator = Operator::ASAP(ASAPOp::SummaryAgg { + child: Rc::clone(child), + family: FieldDataType::Sketch( + SketchKind::new(kind.algorithm().clone(), params), + grouping_strategy.clone(), + ), + input: input.clone(), + reduction: reduction.clone(), + grouping: grouping.clone(), + filter: filter.clone(), + }); + let mut build = OperatorNode::new(operator)?; + // Keep the member's explicit output names. + for (field, retained) in build.schema.fields.iter_mut().zip(&first.schema.fields) { + field.name = retained.name.clone(); + field.table = retained.table.clone(); + } + Ok(Some(Rc::new(build))) +} + +/// `node` with every replaced build below it swapped for its shared build; +/// the same `Rc` when nothing below changed. +fn rebuild( + node: &Rc, + replaced: &HashMap<*const OperatorNode, Rc>, + rebuilt: &mut HashMap<*const OperatorNode, Rc>, +) -> Result, PerComputationCandidateError> { + if let Some(done) = rebuilt.get(&Rc::as_ptr(node)) { + return Ok(Rc::clone(done)); + } + let out = match replaced.get(&Rc::as_ptr(node)) { + Some(build) => Rc::clone(build), + None => { + let mut changed = false; + let mut children = HashMap::new(); + for child in node.children() { + let new = rebuild(child, replaced, rebuilt)?; + changed |= !Rc::ptr_eq(&new, child); + children.insert(Rc::as_ptr(child), new); + } + if changed { + Rc::new(node.with_new_children(|child| children[&Rc::as_ptr(child)].clone())?) + } else { + Rc::clone(node) + } + } + }; + rebuilt.insert(Rc::as_ptr(node), Rc::clone(&out)); + Ok(out) +} + fn root_operators(root: &QueryRoot) -> Vec<&Rc> { match root { QueryRoot::Operator(node) => vec![node], @@ -454,6 +832,206 @@ mod tests { ); } + /// The composed candidate of `queries` (each at its own ε, δ = 0.01) + /// in which every target chose `family`. + fn chose(queries: &[(&str, f64)], family: SketchAlgorithm) -> Vec { + use crate::per_computation_candidates::{ + compose_logical_candidate, generate_per_computation_candidates, ComputationCandidate, + }; + let roots = queries + .iter() + .enumerate() + .map(|(i, (q, epsilon))| { + let root = lower_promql( + q, + asap_types::types::RequestedAccuracy::epsilon_delta(*epsilon, 0.01) + .with_metric(asap_types::types::ErrorMetric::Rank), + ); + ( + i, + QueryRoot::Operator(with_promql_series_identity(&root).unwrap()), + ) + }) + .collect(); + let inventory = generate_per_computation_candidates(roots).unwrap(); + let choice: Vec = inventory + .targets + .iter() + .map(|t| { + t.choices + .candidates + .iter() + .position(|c| *c == ComputationCandidate::Summary(family.clone())) + .expect("the family is a candidate") + }) + .collect(); + compose_logical_candidate(&inventory, &choice) + .unwrap() + .into_iter() + .map(|(_, root)| root) + .collect() + } + + /// The summary builds `roots` read, by address, with each one's KLL `k`. + fn kll_builds(roots: &[QueryRoot]) -> Vec<(*const OperatorNode, u32)> { + let mut out: Vec<(*const OperatorNode, u32)> = Vec::new(); + for node in roots + .iter() + .flat_map(root_operators) + .flat_map(OperatorNode::reachable) + { + if let Some(ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(kind, _), + .. + }) = node.asap() + { + let asap_types::post_asap::SketchParams::Kll { k, .. } = kind.params() else { + panic!("{kind:?}") + }; + if !out.iter().any(|(p, _)| *p == Rc::as_ptr(&node)) { + out.push((Rc::as_ptr(&node), *k)); + } + } + } + out + } + + fn root_bound(root: &QueryRoot) -> f64 { + let QueryRoot::Operator(node) = root else { + panic!("operator root") + }; + node.guarantee + .as_ref() + .and_then(|g| g.spec.error.evaluate()) + .expect("a constant bound") + } + + /// p50 at ε = 0.01 and p99 at ε = 0.001 over one window, each with its + /// own KLL: the rule shares one KLL with the larger `k`, and p50's + /// guarantee is the shared KLL's, tighter than its own. + #[test] + fn quantiles_at_different_targets_share_one_kll_with_the_larger_k() { + let roots = chose( + &[ + ("quantile_over_time(0.5, lat[5m])", 0.01), + ("quantile_over_time(0.99, lat[5m])", 0.001), + ], + SketchAlgorithm::Kll, + ); + let own = kll_builds(&roots); + assert_eq!(own.len(), 2); + assert!(own[0].1 < own[1].1); + let groups = capability_groups(&roots); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].family, SketchAlgorithm::Kll); + let variants = summary_capability_variants(&roots, &groups).unwrap(); + assert_eq!(variants.len(), 1); + assert_eq!( + variants[0].shared, + [SharedSummary { + group: 0, + members: vec![0, 1] + }] + ); + let shared = kll_builds(variants[0].roots.as_ref().unwrap()); + assert_eq!(shared.len(), 1, "one KLL serves both"); + assert_eq!(shared[0].1, own[1].1); + assert!(root_bound(&variants[0].roots.as_ref().unwrap()[0]) < root_bound(&roots[0])); + assert_eq!( + root_bound(&variants[0].roots.as_ref().unwrap()[1]), + root_bound(&roots[1]) + ); + } + + /// Three consumers at three targets: every subset of two or more of + /// them shares one build (three pairs and all three), the others keep + /// their own. + #[test] + fn every_subset_of_two_or_more_consumers_shares_one_build() { + let roots = chose( + &[ + ("quantile_over_time(0.5, lat[5m])", 0.05), + ("quantile_over_time(0.9, lat[5m])", 0.01), + ("quantile_over_time(0.99, lat[5m])", 0.001), + ], + SketchAlgorithm::Kll, + ); + let groups = capability_groups(&roots); + let variants = summary_capability_variants(&roots, &groups).unwrap(); + assert_eq!( + variants + .iter() + .map(|v| v.shared[0].members.clone()) + .collect::>(), + [vec![0, 1], vec![0, 2], vec![1, 2], vec![0, 1, 2]] + ); + for variant in &variants { + let builds = kll_builds(variant.roots.as_ref().unwrap()); + assert_eq!(builds.len(), 4 - variant.shared[0].members.len()); + } + } + + /// A group of more than 20 members (over a million sharing subsets) + /// is an error, not a silent skip or an endless enumeration. + #[test] + fn a_group_past_20_members_is_an_error() { + let roots = chose( + &[ + ("quantile_over_time(0.5, lat[5m])", 0.01), + ("quantile_over_time(0.99, lat[5m])", 0.001), + ], + SketchAlgorithm::Kll, + ); + let mut groups = capability_groups(&roots); + let members = groups[0].members.clone(); + groups[0].members = members + .iter() + .cycle() + .take(MAX_GROUP_MEMBERS + 1) + .cloned() + .collect(); + assert!(matches!( + summary_capability_variants(&roots, &groups), + Err(PerComputationCandidateError::Unsupported(_)) + )); + } + + /// Equal targets give identical builds, which are the + /// identical-expression rule's; a different window or family is a + /// different input or capability, never shared. + #[test] + fn equal_sizes_other_windows_and_other_families_are_not_capability_groups() { + let equal = chose( + &[ + ("quantile_over_time(0.5, lat[5m])", 0.01), + ("quantile_over_time(0.99, lat[5m])", 0.01), + ], + SketchAlgorithm::Kll, + ); + assert!(capability_groups(&equal).is_empty()); + assert!(!identical_expression_variants(&equal, &sharing_groups(&equal)).is_empty()); + let windows = chose( + &[ + ("quantile_over_time(0.5, lat[5m])", 0.01), + ("quantile_over_time(0.99, lat[10m])", 0.001), + ], + SketchAlgorithm::Kll, + ); + assert!(capability_groups(&windows).is_empty()); + let families = chose( + &[ + ("quantile_over_time(0.5, lat[5m])", 0.01), + ("quantile_over_time(0.99, lat[5m])", 0.001), + ], + SketchAlgorithm::DDSketch, + ); + assert_eq!(capability_groups(&families).len(), 1); + assert_eq!( + capability_groups(&families)[0].family, + SketchAlgorithm::DDSketch + ); + } + /// Each candidate's label (the groups it shares) and its roots agree: /// applying the label gives the same roots, sharing exactly the /// labelled groups. diff --git a/crates/asap-aware-mapping/src/pass/mod.rs b/crates/asap-aware-mapping/src/pass/mod.rs index 3f37907a8..664d4a6a6 100644 --- a/crates/asap-aware-mapping/src/pass/mod.rs +++ b/crates/asap-aware-mapping/src/pass/mod.rs @@ -36,8 +36,8 @@ use crate::stage3_plan_selection::{Selection, SelectionError}; pub use stage1_to_stage3::{ run_stage1_to_stage3, run_stage1_to_stage3_from_pass1, run_stage2_and_stage3, - stage1_logical_asap_optimization, IdenticalExpressionCandidate, LogicalCandidate, - PrunedCandidate, Stage1Candidate, Stage1ToStage3Pass, Stage1ToStage3Run, + stage1_logical_asap_optimization, LogicalCandidate, Pass2Candidate, Pass2Of, PrunedCandidate, + Stage1Candidate, Stage1ToStage3Pass, Stage1ToStage3Run, }; static DEFAULT_COST_MODEL: DefaultCostModel = DefaultCostModel; diff --git a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs index 458e0fce8..bb1c2fa20 100644 --- a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs +++ b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs @@ -10,16 +10,19 @@ //! generation, then composition of one whole-workload candidate per choice //! of a per-computation candidate for every target (the Cartesian product, //! in mixed-radix order, the last target varying fastest), each followed -//! by its Pass 2 candidates from the identical-expression rule +//! by its Pass 2 candidates: those of the identical-expression rule //! ([`identical_expression_variants`]: one per subset of its sharing -//! groups; the independent candidate is kept); +//! groups), then those of the summary-capability rule +//! ([`summary_capability_variants`]: one per choice of shared summary +//! builds), each followed by its own identical-expression candidates, +//! every distinct candidate once; the independent candidate is kept; //! 2. Stage 2, physical ASAP-aware optimization of every Stage 1 candidate //! ([`optimize_physical_asap`]); //! 3. Stage 3, plan selection over the physical candidates ([`select_plan`]). //! //! Stage 0, the query-language frontends, runs before it (the `asap-planner` -//! facade lowers the workload). Pass 2's summary-capability and -//! window-composition rules are not planned yet. +//! facade lowers the workload). Pass 2's window-composition rule is not +//! planned yet. //! //! Every combination is enumerated and plan selection prices every //! candidate; nothing is capped. A search that avoids building every @@ -27,6 +30,7 @@ use std::rc::Rc; +use asap_types::ir::flat::flatten; use asap_types::ir::schema_support::with_promql_series_identity; use asap_types::ir::{OperatorNode, QueryRoot}; use asap_types::post_asap::SketchAlgorithm; @@ -34,7 +38,9 @@ use asap_types::types::{AccuracyTarget, RequestedAccuracy}; use asap_types::workload::{DataWorkload, QueryLanguage, QueryRecurrence}; use super::{OptimizationInput, OptimizationPass, OptimizeError, PlanOutput, QueryPlan}; -use crate::asap_aware_cse::identical_expression_variants; +use crate::asap_aware_cse::{ + capability_groups, identical_expression_variants, sharing_groups, summary_capability_variants, +}; use crate::per_computation_candidates::{ compose_logical_candidate, generate_per_computation_candidates, ComputationCandidate, PerComputationCandidateError, PerComputationCandidates, @@ -57,14 +63,17 @@ pub struct LogicalCandidate { pub roots: Result, PerComputationCandidateError>, } -/// A Stage 1 candidate Pass 2's identical-expression rule added: `id` is -/// `from_logical` with the nodes of each group in `shared` (named by -/// operator kind, children first) made one node. It has `from_logical`'s -/// `choice`. +/// A Stage 1 candidate Pass 2 added: `id` is `from_logical` with each +/// summary build set in `summaries` made one build sized for all its +/// consumers (the summary-capability rule, e.g. "UnivMon for Q1, Q3"), +/// then the nodes of each group in `shared` (named by operator kind, +/// children first) made one node (the identical-expression rule). It has +/// `from_logical`'s `choice`. #[derive(Debug, Clone, PartialEq)] -pub struct IdenticalExpressionCandidate { +pub struct Pass2Candidate { pub id: String, pub from_logical: String, + pub summaries: Vec, pub shared: Vec, } @@ -90,9 +99,9 @@ pub struct Stage1ToStage3Run { /// the size of the Cartesian product of the targets' candidates, /// saturating at `usize::MAX`. Pass 2 adds to them. pub combinations: usize, - /// Stage 1, Pass 2: the candidates the identical-expression rule added, - /// in enumeration order. - pub stage1_pass2_identical_expression: Vec, + /// Stage 1, Pass 2: the candidates the identical-expression and + /// summary-capability rules added, in enumeration order. + pub stage1_pass2: Vec, /// Stage 1: every whole-workload logical candidate, in enumeration /// order: each composed Pass 1 candidate, followed by its Pass 2 /// candidates. @@ -159,17 +168,30 @@ pub struct Stage1Candidate { /// Pass 2 candidate has its Pass 1 candidate's. pub choice: Vec, pub roots: Result, PerComputationCandidateError>, - /// For a Pass 2 candidate: the position, in the enumeration, of the - /// Pass 1 candidate it shares the identical sub-DAGs of, and the shared - /// groups (operator kinds, children first). - pub pass2_of: Option<(usize, Vec)>, + /// For a Pass 2 candidate: what it shares of which Pass 1 candidate. + pub pass2_of: Option, +} + +/// What a Pass 2 candidate shares of a Pass 1 candidate. +#[derive(Debug, Clone, PartialEq)] +pub struct Pass2Of { + /// The position, in the enumeration, of the Pass 1 candidate. + pub from: usize, + /// The summary builds made one build each (summary-capability rule). + pub summaries: Vec, + /// The identical groups shared (operator kinds, children first). + pub shared: Vec, } /// Stage 1 (logical ASAP-aware optimization) over the Pass 1 inventory, one /// candidate at a time: each composed Pass 1 candidate (every choice of a /// per-computation candidate for every target, the last target varying -/// fastest), followed by its Pass 2 identical-expression candidates. Lazy -/// and uncapped: a caller that limits the candidates stops consuming it. +/// fastest), followed by its Pass 2 candidates: the identical-expression +/// rule's, then each summary-capability candidate followed by its own +/// identical-expression candidates. A Pass 2 candidate equal to an earlier +/// one of the same Pass 1 candidate (same nodes, same sharing) is listed +/// once. Lazy and uncapped: a caller that limits the candidates stops +/// consuming it. pub fn stage1_logical_asap_optimization( per_computation: &PerComputationCandidates, ) -> impl Iterator + '_ { @@ -191,32 +213,34 @@ pub fn stage1_logical_asap_optimization( let display_name = display_name(per_computation, &query_of_target, &this); let roots = compose_logical_candidate(per_computation, &this) .map(|roots| roots.into_iter().map(|(_, root)| root).collect::>()); - let (groups, variants) = match &roots { - Ok(roots) => { - let groups = crate::asap_aware_cse::sharing_groups(roots); - let variants = identical_expression_variants(roots, &groups); - (groups, variants) - } - Err(_) => (Vec::new(), Vec::new()), - }; let from = position + queue.len(); + let pass2 = match &roots { + Ok(roots) => pass2_candidates(roots), + Err(_) => Vec::new(), + }; queue.push_back(Stage1Candidate { display_name: display_name.clone(), choice: this.clone(), roots, pass2_of: None, }); - for variant in variants { - let shared: Vec = variant - .shared_groups - .iter() - .map(|&g| format!("{} #{}", groups[g].nodes[0].operator.kind_name(), g + 1)) - .collect(); + for (summaries, shared, roots) in pass2 { + let mut name = display_name.clone(); + if !summaries.is_empty() { + name += &format!(" · one {}", summaries.join(", one ")); + } + if !shared.is_empty() { + name += &format!(" · shared {}", shared.join(", ")); + } queue.push_back(Stage1Candidate { - display_name: format!("{display_name} · shared {}", shared.join(", ")), + display_name: name, choice: this.clone(), - roots: Ok(variant.roots), - pass2_of: Some((from, shared)), + roots, + pass2_of: Some(Pass2Of { + from, + summaries, + shared, + }), }); } } @@ -225,6 +249,109 @@ pub fn stage1_logical_asap_optimization( }) } +/// A Pass 2 candidate: the summary builds it shares (summary-capability +/// rule), the identical groups it shares, and its roots. +type Pass2 = ( + Vec, + Vec, + Result, PerComputationCandidateError>, +); + +/// The Pass 2 candidates of the composed candidate with roots `roots`, in +/// enumeration order, each distinct candidate once. +fn pass2_candidates(roots: &[QueryRoot]) -> Vec { + let mut found = Pass2List { + seen: vec![flatten(roots).0], + out: Vec::new(), + }; + found.identical_variants(&[], roots); + let groups = capability_groups(roots); + let variants = match summary_capability_variants(roots, &groups) { + Ok(variants) => variants, + Err(error) => { + found.out.push((Vec::new(), Vec::new(), Err(error))); + return found.out; + } + }; + for variant in variants { + let summaries: Vec = variant + .shared + .iter() + .map(|summary| { + let group = &groups[summary.group]; + let mut queries: Vec = summary + .members + .iter() + .flat_map(|&m| readers_of(roots, &group.members[m])) + .collect(); + queries.sort_unstable(); + queries.dedup(); + let queries: Vec = queries.iter().map(|q| format!("Q{}", q + 1)).collect(); + format!("{:?} for {}", group.family, queries.join(", ")) + }) + .collect(); + match variant.roots { + Ok(roots) => { + found.push(&summaries, Vec::new(), &roots); + found.identical_variants(&summaries, &roots); + } + // Listed as a candidate that cannot be composed; the other + // variants go on. + Err(error) => found.out.push((summaries, Vec::new(), Err(error))), + } + } + found.out +} + +/// Pass 2 candidates found so far, and the flattened DAG of every candidate +/// listed (the Pass 1 candidate's included), so each is listed once. +struct Pass2List { + seen: Vec, + out: Vec, +} + +impl Pass2List { + fn push(&mut self, summaries: &[String], shared: Vec, roots: &[QueryRoot]) { + let flat = flatten(roots).0; + if self.seen.contains(&flat) { + return; + } + self.seen.push(flat); + self.out + .push((summaries.to_vec(), shared, Ok(roots.to_vec()))); + } + + /// The identical-expression candidates of `roots`, which shares + /// `summaries`. + fn identical_variants(&mut self, summaries: &[String], roots: &[QueryRoot]) { + let groups = sharing_groups(roots); + for variant in identical_expression_variants(roots, &groups) { + let shared = variant + .shared_groups + .iter() + .map(|&g| format!("{} #{}", groups[g].nodes[0].operator.kind_name(), g + 1)) + .collect(); + self.push(summaries, shared, &variant.roots); + } + } +} + +/// The query roots (indices) from which `node` is reachable. +fn readers_of(roots: &[QueryRoot], node: &Rc) -> Vec { + roots + .iter() + .enumerate() + .filter(|(_, root)| { + root_operators(root).into_iter().any(|op| { + OperatorNode::reachable(op) + .iter() + .any(|n| Rc::ptr_eq(n, node)) + }) + }) + .map(|(q, _)| q) + .collect() +} + /// [`run_stage1_to_stage3`] after Pass 1: enumerates every Stage 1 /// candidate of `per_computation` (the Pass 1 inventory of the frontends' /// roots, [`stage1_logical_asap_optimization`]), so a caller can read the @@ -275,11 +402,12 @@ pub fn run_stage2_and_stage3( }), } } - if let Some((from, shared)) = candidate.pass2_of { - pass2.push(IdenticalExpressionCandidate { + if let Some(of) = candidate.pass2_of { + pass2.push(Pass2Candidate { id: id.clone(), - from_logical: format!("L{}", from + 1), - shared, + from_logical: format!("L{}", of.from + 1), + summaries: of.summaries, + shared: of.shared, }); } logical.push(LogicalCandidate { @@ -310,7 +438,7 @@ pub fn run_stage2_and_stage3( stage3_selected_logical: selected_logical, stage1_pass1_per_computation: per_computation, combinations, - stage1_pass2_identical_expression: pass2, + stage1_pass2: pass2, stage1_logical_asap_optimization: logical, stage2_physical_asap_optimization: physical, stage2_pruned: pruned, diff --git a/crates/asap-aware-mapping/src/per_computation_candidates.rs b/crates/asap-aware-mapping/src/per_computation_candidates.rs index 71457a40d..440eb56f8 100644 --- a/crates/asap-aware-mapping/src/per_computation_candidates.rs +++ b/crates/asap-aware-mapping/src/per_computation_candidates.rs @@ -819,6 +819,32 @@ fn rewrite( Ok(rebuilt) } +/// `roots` with every node's guarantee composed again +/// ([`composed_guarantee`]), children first; a sub-DAG shared between roots +/// stays shared. Pass 2 calls it after re-sizing a shared summary, so each +/// consumer's guarantee is the shared state's. +pub(crate) fn with_composed_guarantees( + roots: &[QueryRoot], +) -> Result, PerComputationCandidateError> { + let mut memo = HashMap::new(); + roots + .iter() + .map(|root| { + Ok(match root { + QueryRoot::Operator(node) => QueryRoot::Operator(with_guarantees(node, &mut memo)?), + QueryRoot::Scalar(expr) => { + for node in expr.operator_refs() { + with_guarantees(node, &mut memo)?; + } + QueryRoot::Scalar( + expr.map_operator_refs(&mut |node| memo[&Rc::as_ptr(node)].clone()), + ) + } + }) + }) + .collect() +} + /// `node` and everything below it, each node with its [`composed_guarantee`]. fn with_guarantees( node: &Rc, @@ -952,7 +978,13 @@ fn composed_guarantee(node: &OperatorNode) -> Option { Operator::NonASAP(_) if node.children().into_iter().all(|child| exact(child)) => { Some(AccuracyGuarantee::exact("exact over exact inputs")) } + // Over a join, the child's sample value may be an exact input's + // column, so each column's own guarantee is read instead. + Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) if derives_from_join(child) => { + relational_project_guarantee(cols, child) + } Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) => project_guarantee(cols, child), + Operator::NonASAP(NonASAPOp::Join { .. }) => join_guarantee(node), // A measure filter or `HAVING` over approximate values changes // which rows or groups count: no rule. Operator::NonASAP(NonASAPOp::Aggregate { @@ -1195,14 +1227,189 @@ fn project_guarantee(cols: &[ProjectItem], child: &Rc) -> Option bool { + match node.non_asap() { + Some(NonASAPOp::Join { .. }) => true, + Some( + NonASAPOp::Project { child, .. } + | NonASAPOp::Filter { child, .. } + | NonASAPOp::Sort { child, .. } + | NonASAPOp::Limit { child, .. } + | NonASAPOp::Dedup { child, .. }, + ) => derives_from_join(child), + _ => false, + } +} + +/// The guarantee of each output column of `node`, where it is known: every +/// column of an exact node is exact; a join beside exact inputs +/// ([`join_guarantee`]) gives the exact inputs' columns exact and the +/// approximate input's columns its guarantee; a projection of columns maps +/// its child's. `None` anywhere else, a computed projection included. +/// Guarantees are per node, so every column of an approximate input counts +/// as approximate; column-level tracking is #689. +/// +/// This assumes every approximate node of Stage 1 has an exact row set: +/// its guarantee bounds each row's values, never which rows exist (a +/// top-k estimate, whose rows are approximate, has no bound; +/// `estimate_has_bound`). A guarantee on row membership would need these +/// rules revisited. +fn column_guarantees(node: &OperatorNode) -> Option> { + let width = node.schema.fields.len(); + if let Some(guarantee) = node.guarantee.as_ref().filter(|g| g.is_exact()) { + return Some(vec![guarantee.clone(); width]); + } + match &node.operator { + Operator::NonASAP(NonASAPOp::Join { + kind: + JoinKind::Inner | JoinKind::Left | JoinKind::Right | JoinKind::Full | JoinKind::Cross, + left, + right, + .. + }) => { + let input = join_input_guarantees(left, right)?; + (input.len() == width).then_some(input) + } + Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) => { + let child = column_guarantees(child)?; + cols.iter() + .map(|item| match item.expr { + ScalarExpr::Column(id) => child.get(id).cloned(), + _ => None, + }) + .collect() + } + _ => None, + } +} + +/// The guarantees of a join's input columns (`left ++ right`) when at most +/// one input is approximate and the others are exact: the exact input's +/// columns exact, the approximate input's columns its guarantee. +fn join_input_guarantees( + left: &OperatorNode, + right: &OperatorNode, +) -> Option> { + let exact = |node: &OperatorNode| node.guarantee.as_ref().is_some_and(|g| g.is_exact()); + if !exact(left) && !exact(right) { + return None; + } + let side = |node: &OperatorNode| { + let guarantee = node.guarantee.clone()?; + Some(vec![guarantee; node.schema.fields.len()]) + }; + let mut columns = side(left)?; + columns.extend(side(right)?); + Some(columns) +} + +/// A join whose inputs are exact but one keeps that input's guarantee when +/// its predicate reads only exact columns (a cross join, or a key equality +/// on the exact inputs' columns): which rows match, and how often, then +/// does not depend on an estimate, and each output row carries one row of +/// the approximate input unchanged. Outer joins pad on the same exact +/// decision. A semi- or anti-join, a key on an approximate input's column +/// (per-node guarantees count all its columns approximate; #689), or two +/// approximate inputs have none. An exact join is composed before this. +/// Like [`column_guarantees`], this assumes the approximate input's row set +/// is exact: only its values carry error. +fn join_guarantee(node: &OperatorNode) -> Option { + let Operator::NonASAP(NonASAPOp::Join { + kind: JoinKind::Inner | JoinKind::Left | JoinKind::Right | JoinKind::Full | JoinKind::Cross, + pred, + left, + right, + }) = &node.operator + else { + return None; + }; + let columns = join_input_guarantees(left, right)?; + let pred_exact = + scalar_guarantee(&pred.0, &|id| columns.get(id).cloned()).is_some_and(|g| g.is_exact()); + if !pred_exact { + return None; + } + [left, right] + .into_iter() + .find_map(|input| input.guarantee.clone().filter(|g| !g.is_exact())) +} + +/// The guarantee of a projection over relational columns +/// ([`column_guarantees`]): each output column's as [`scalar_guarantee`] +/// (a `CASE` as [`case_guarantee`]), and the node's the weakest of them. +/// `None` when the child's column guarantees are unknown or any output +/// column has none. +fn relational_project_guarantee( + cols: &[ProjectItem], + child: &Rc, +) -> Option { + let columns = column_guarantees(child)?; + let column = |id: usize| columns.get(id).cloned(); + weakest( + cols.iter() + .map(|item| scalar_guarantee(&item.expr, &column)) + .collect::>>()?, + ) +} + +/// The weakest of `guarantees`: exact when all are, the approximate one +/// when there is one (or all approximate ones are equal), and otherwise +/// as an extremum of them ([`lipschitz_guarantee`]). +fn weakest(guarantees: Vec) -> Option { + let mut approximate: Vec = Vec::new(); + for guarantee in guarantees.into_iter().filter(|g| !g.is_exact()) { + if !approximate.contains(&guarantee) { + approximate.push(guarantee); + } + } + match approximate.len() { + 0 => Some(AccuracyGuarantee::exact("exact over exact operands")), + 1 => approximate.pop(), + _ => lipschitz_guarantee(approximate), + } +} + +/// A `CASE` whose operand and conditions are exact picks its branch by an +/// exact decision, so it keeps the weakest branch's guarantee (a literal or +/// NULL branch, or a missing `ELSE`, is exact). A condition over an +/// approximate value has none. +fn case_guarantee( + operand: Option<&ScalarExpr>, + branches: &[(ScalarExpr, ScalarExpr)], + else_expr: Option<&ScalarExpr>, + column: &dyn Fn(usize) -> Option, +) -> Option { + let exact = |expr: &ScalarExpr| scalar_guarantee(expr, column).is_some_and(|g| g.is_exact()); + if !operand.is_none_or(exact) || !branches.iter().all(|(when, _)| exact(when)) { + return None; + } + weakest( + branches + .iter() + .map(|(_, then)| then) + .chain(else_expr) + .map(|expr| scalar_guarantee(expr, column)) + .collect::>>()?, + ) +} + +/// The guarantee of `column` of a projection's `child`: over a join, the +/// column's own ([`column_guarantees`]); otherwise the child's sample +/// value has the child's as an operand ([`operand_guarantee`]) and other /// columns (labels, timestamps) are exact. `None` when the child has no -/// sample value. +/// sample value or the column's guarantee is unknown. pub(crate) fn project_column_guarantee( child: &Rc, column: usize, ) -> Option { + // Over a join, each column's own guarantee ([`column_guarantees`]). + if derives_from_join(child) { + return column_guarantees(child)?.get(column).cloned(); + } let value = resolve_column_ref(&ColumnRef::SampleValue, &child.schema).ok()?; if column == value { operand_guarantee(child) @@ -1241,6 +1448,11 @@ pub(crate) fn scalar_guarantee( ScalarExpr::Negative { expr, .. } => { lipschitz_guarantee(vec![scalar_guarantee(expr, column)?]) } + ScalarExpr::Case { + operand, + branches, + else_expr, + } => case_guarantee(operand.as_deref(), branches, else_expr.as_deref(), column), ScalarExpr::Arithmetic { op, left, right, .. } => arithmetic_guarantee( @@ -2734,6 +2946,212 @@ mod tests { assert!(close(delta(&vector(&shared, &shared)), 0.001)); assert!(close(delta(&vector(&shared, &other)), 0.002)); } + + /// `left JOIN right ON pred`, composed. + fn join( + kind: JoinKind, + left: &Rc, + right: &Rc, + pred: ScalarExpr, + ) -> OperatorNode { + let mut node = OperatorNode::new(Operator::NonASAP(NonASAPOp::Join { + kind, + pred: Predicate(pred), + left: left.clone(), + right: right.clone(), + })) + .unwrap(); + node.guarantee = composed_guarantee(&node); + node + } + fn column(id: usize) -> ScalarExpr { + ScalarExpr::Column(id) + } + fn equals(left: ScalarExpr, right: ScalarExpr) -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(left), + op: CompareOpKind::Eq, + right: Box::new(right), + semantics: ExprSemantics::Sql, + } + } + /// `CASE WHEN condition THEN NULL ELSE otherwise END` over `child`, + /// composed. + fn case_over( + child: &OperatorNode, + condition: ScalarExpr, + otherwise: ScalarExpr, + ) -> Option { + let node = OperatorNode::new(Operator::NonASAP(NonASAPOp::Project { + cols: vec![ProjectItem { + alias: Some("v".into()), + expr: ScalarExpr::Case { + operand: None, + branches: vec![(condition, ScalarExpr::Literal(ScalarValue::Null))], + else_expr: Some(Box::new(otherwise)), + }, + }], + qualifier: None, + child: Rc::new(child.clone()), + })) + .unwrap(); + composed_guarantee(&node) + } + /// An approximate node and an exact one, and the approximate one's + /// column count (its columns come first in a join). + fn approximate_and_exact() -> (Rc, Rc, usize) { + let approximate = node(rel(0.01)); + let exact = node_over("count(m)", AccuracyGuarantee::exact("test")); + let width = approximate.schema.fields.len(); + (approximate, exact, width) + } + + /// A join whose other input is exact and whose predicate reads only + /// exact columns keeps the approximate input's guarantee: a cross + /// join, or a key equality on the exact input's columns. + #[test] + fn join_beside_an_exact_input_on_exact_columns_keeps_the_guarantee() { + let (approximate, exact, width) = approximate_and_exact(); + for (kind, pred) in [ + ( + JoinKind::Cross, + ScalarExpr::Literal(ScalarValue::Boolean(true)), + ), + (JoinKind::Inner, equals(column(width), lit(0.0))), + (JoinKind::Left, equals(column(width), lit(0.0))), + ] { + assert_eq!( + join(kind.clone(), &approximate, &exact, pred).guarantee, + Some(rel(0.01)), + "{kind:?}" + ); + } + // Either side. + assert_eq!( + join( + JoinKind::Cross, + &exact, + &approximate, + ScalarExpr::Literal(ScalarValue::Boolean(true)) + ) + .guarantee, + Some(rel(0.01)) + ); + } + + /// A join keyed on an approximate input's column, a semi-join, or + /// two approximate inputs have no guarantee (column-level tracking + /// is #689). + #[test] + fn join_on_an_approximate_key_has_no_guarantee() { + let (approximate, exact, width) = approximate_and_exact(); + assert_eq!( + join( + JoinKind::Inner, + &approximate, + &exact, + equals(column(0), column(width)) + ) + .guarantee, + None + ); + assert_eq!( + join( + JoinKind::Semi, + &exact, + &approximate, + ScalarExpr::Literal(ScalarValue::Boolean(true)) + ) + .guarantee, + None + ); + assert_eq!( + join( + JoinKind::Cross, + &approximate, + &approximate, + ScalarExpr::Literal(ScalarValue::Boolean(true)) + ) + .guarantee, + None + ); + } + + /// A `CASE` whose condition reads only exact columns keeps each + /// branch's guarantee (a NULL branch is exact); one testing an + /// approximate column has none. + #[test] + fn case_on_exact_columns_keeps_the_branch_guarantee() { + let (approximate, exact, width) = approximate_and_exact(); + let joined = join( + JoinKind::Cross, + &approximate, + &exact, + ScalarExpr::Literal(ScalarValue::Boolean(true)), + ); + let value = approximate.schema.fields.len() - 1; + assert_eq!( + case_over(&joined, equals(column(width), lit(0.0)), column(value)), + Some(rel(0.01)) + ); + assert_eq!( + case_over(&joined, equals(column(value), lit(0.0)), column(value)), + None + ); + } + + /// `node` projecting `cols` over `child`, composed. + fn project(child: &OperatorNode, cols: Vec) -> OperatorNode { + let mut node = OperatorNode::new(Operator::NonASAP(NonASAPOp::Project { + cols: cols + .into_iter() + .enumerate() + .map(|(i, expr)| ProjectItem { + alias: Some(format!("c{i}")), + expr, + }) + .collect(), + qualifier: None, + child: Rc::new(child.clone()), + })) + .unwrap(); + node.guarantee = composed_guarantee(&node); + node + } + + /// Over a join, a projection reads each column's own guarantee, + /// not the child's sample value (which may be an exact input's + /// column): `sum(m) * 1000` beside the exact side's `value` has + /// the scaled error 1000, not the approximate side's 1. A + /// projection over that computed projection has no per-column + /// guarantees, so it has none (#689). + #[test] + fn projection_over_a_join_reads_each_columns_guarantee() { + let approximate = node(abs(1.0, 0.01)); + let exact = node_over("m", AccuracyGuarantee::exact("test")); + let joined = join( + JoinKind::Cross, + &approximate, + &exact, + ScalarExpr::Literal(ScalarValue::Boolean(true)), + ); + assert_eq!(joined.guarantee, Some(abs(1.0, 0.01))); + let sum = approximate.schema.fields.len() - 1; + let value = approximate.schema.fields.len() + exact.schema.fields.len() - 1; + let scaled = project( + &joined, + vec![ + arith(ArithmeticOpKind::Mul, column(sum), lit(1000.0)), + column(value), + ], + ); + assert_eq!(scaled.guarantee.as_ref().map(bound), Some(1000.0)); + let stacked = project( + &scaled, + vec![arith(ArithmeticOpKind::Add, column(1), lit(1.0)), column(0)], + ); + assert_eq!(stacked.guarantee, None); + } } /// Which stream norm bounds a Count-Min or Count Sketch estimate (#669). diff --git a/crates/asap-aware-mapping/src/query_physical_lowering.rs b/crates/asap-aware-mapping/src/query_physical_lowering.rs index bb7f914e5..49bdc6e91 100644 --- a/crates/asap-aware-mapping/src/query_physical_lowering.rs +++ b/crates/asap-aware-mapping/src/query_physical_lowering.rs @@ -1252,6 +1252,9 @@ fn supports_hash_aggregate( | AggIntent::PearsonCorr { .. } | AggIntent::Group | AggIntent::CountValues { .. } + | AggIntent::Cardinality { .. } + | AggIntent::FrequencyL2 { .. } + | AggIntent::FrequencyEntropy { .. } ) }) } @@ -1555,6 +1558,78 @@ mod tests { assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).is_ok()); } + // The exact distinct count, L2 norm and entropy are hash aggregates: the + // executor runs them as native reducers over one identity column. + #[test] + fn exact_frequency_intents_lower_to_physical_hash_aggregates() { + use asap_types::pre_asap::{AggIntent, DataType, Field, Reduction, Schema, Source}; + use asap_types::types::AccuracyTarget; + let source = Source::Table { + table_ref: "flows".into(), + }; + for measure in [ + AggIntent::Cardinality { + cols: vec![0], + accuracy: AccuracyTarget::exact(), + }, + AggIntent::FrequencyL2 { + col: Some(0), + accuracy: AccuracyTarget::exact(), + }, + AggIntent::FrequencyEntropy { + col: Some(0), + accuracy: AccuracyTarget::exact(), + }, + ] { + let root = + OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![]), + measures: vec![measure.clone()], + output_names: vec!["v".into()], + filters: vec![], + having: None, + child: OperatorNode::new_shared(asap_types::ir::Operator::NonASAP( + NonASAPOp::Scan { + source: source.clone(), + predicates: vec![], + schema: Schema::new(vec![Field::plain( + "src_ip", + DataType::Utf8, + false, + )]), + }, + )) + .unwrap(), + })) + .unwrap(); + let scope = scope(vec![coverage(source.clone(), vec![])]); + let provided = HashMap::from([ + ( + "query-1".into(), + evidence(scan_stats(edge(100, 1600), 1600)), + ), + ( + "query-0".into(), + evidence(OperatorStatistics::HashAggregate { + edges: unary_edges(edge(100, 1600), edge(1, 8)), + group_count: 1, + key_bytes: 0, + accumulator_bytes_per_group: 48, + }), + ), + ]); + let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)) + .unwrap_or_else(|e| panic!("{measure:?}: {e:?}")); + assert!(matches!( + dag.nodes.last().unwrap().operator, + PhysicalOperator::HashAggregate { + grouping_key_count: 0, + accumulator_count: 1, + } + )); + } + } + #[test] fn query_lowering_recurses_and_fuses_global_sort_limit() { use asap_types::pre_asap::{AggIntent, GroupKeys, Reduction, Source}; diff --git a/crates/asap-physical-operators/src/capability.rs b/crates/asap-physical-operators/src/capability.rs index c8441313e..6e0441ec2 100644 --- a/crates/asap-physical-operators/src/capability.rs +++ b/crates/asap-physical-operators/src/capability.rs @@ -112,7 +112,7 @@ pub fn validate_summary_kernel( } _ => return Err(format!("unsupported summary kernel {family:?}")), }; - if keyed != input.item.is_some() && !is_unit_sample_frequency(input) { + if keyed != input.item.is_some() && unit_frequency_column(input).is_none() { return Err("Planner item expression does not match kernel layout".into()); } Ok(()) @@ -217,6 +217,25 @@ fn valid_matrix(width: u32, depth: u32) -> bool { .is_some() } +/// The one item column of a unit-count frequency update (each value is one +/// occurrence), the update an unkeyed frequency family (HLL, UnivMon) builds +/// from that column's values. +pub(crate) fn unit_frequency_column( + update: &planner_types::post_asap::SummaryUpdate, +) -> Option<&planner_types::pre_asap::ColumnRef> { + use planner_types::post_asap::{NonNegativeWeightProof, SummaryInputExpr, WeightDomain}; + match (&update.item, &update.weight, &update.weight_domain) { + ( + Some(SummaryInputExpr::Column(column)), + SummaryInputExpr::Constant(weight), + WeightDomain::NonNegative { + proof: NonNegativeWeightProof::UnitCount, + }, + ) if *weight == 1.0 => Some(column), + _ => None, + } +} + pub(crate) fn is_unit_sample_frequency(update: &planner_types::post_asap::SummaryUpdate) -> bool { use planner_types::post_asap::{NonNegativeWeightProof, SummaryInputExpr, WeightDomain}; matches!( @@ -266,7 +285,10 @@ pub fn validate_native_family(family: &SummaryFamilyType) -> Result<(), Error> { match family { SummaryFamilyType::ExactAggregate(..) => {} SummaryFamilyType::Sketch(kind, _) - if matches!(kind.algorithm(), A::Kll | A::DDSketch | A::Hll | A::Kmv) => {} + if matches!( + kind.algorithm(), + A::Kll | A::DDSketch | A::Hll | A::Kmv | A::UnivMon + ) => {} _ => { return Err(Error::Invalid( "summary family has no native DAG state implementation".into(), @@ -323,6 +345,15 @@ pub fn validate_sketch_evaluation( } (A::Kmv, SketchStatistic::Cardinality) => true, (A::Kmv, _) => bare_count, + // One frequency state answers its total, distinct count, L2 norm + // and entropy. + ( + A::UnivMon, + SketchStatistic::Cardinality + | SketchStatistic::FrequencyL2 + | SketchStatistic::FrequencyEntropy, + ) => true, + (A::UnivMon, _) => bare_count, // Only count intents read a Count-Min bare count, and their // updates have unit weight; the evaluation is typed Int64 on that basis. (A::Cms, _) => bare_count, diff --git a/crates/asap-physical-operators/src/operators/aggregate/mod.rs b/crates/asap-physical-operators/src/operators/aggregate/mod.rs index 0ef4c6f7d..8f69dba57 100644 --- a/crates/asap-physical-operators/src/operators/aggregate/mod.rs +++ b/crates/asap-physical-operators/src/operators/aggregate/mod.rs @@ -13,6 +13,28 @@ impl Operator { for (name, reduction) in &measures { let (t, n) = match reduction { Reduction::Count => (DataType::Int64, false), + Reduction::Cardinality(columns) => { + if columns.is_empty() { + return Err(invalid( + "distinct aggregate requires at least one identity column", + )); + } + for column in columns { + plain(&input, *column)?; + } + (DataType::Int64, false) + } + Reduction::FrequencyL2(i) | Reduction::FrequencyEntropy(i) => { + if !matches!( + plain(&input, *i)?.0, + DataType::Bool | DataType::Int64 | DataType::Float64 | DataType::Utf8 + ) { + return Err(invalid( + "frequency aggregate requires a Boolean, Int64, Float64 or Utf8 identity", + )); + } + (DataType::Float64, false) + } Reduction::Sum(i) | Reduction::Avg(i) => { let (t, _) = plain(&input, *i)?; if !matches!(t, DataType::Int64 | DataType::Float64) { @@ -170,6 +192,14 @@ impl Operator { #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub enum Reduction { Count, + /// Exact distinct tuple count; a tuple with any NULL component is skipped. + Cardinality(Vec), + /// L2 norm of the unit-update frequencies of one identity column; NULL + /// identities are skipped. + FrequencyL2(usize), + /// Shannon entropy in bits of the frequencies of one identity column; + /// NULL identities are skipped. + FrequencyEntropy(usize), Sum(usize), Avg(usize), Min(usize), @@ -276,7 +306,7 @@ async fn reduce( selected.push(row); } } - result.push(reduce_one(&selected, measure, input, &mut work).await?); + result.push(reduce_one(&selected, measure, input, &mut work, context).await?); } workspace.grow(row_bytes(&result))?; output.push(result); @@ -314,6 +344,7 @@ async fn reduce_one( measure: &Reduction, input: &SchemaRef, work: &mut Cooperative, + context: &RunContext, ) -> Result { let column = match measure { Reduction::Count => { @@ -322,6 +353,66 @@ async fn reduce_one( )) } Reduction::Sum(i) | Reduction::Avg(i) | Reduction::Min(i) | Reduction::Max(i) => *i, + Reduction::Cardinality(columns) => { + let mut workspace = Workspace::new(context)?; + let mut identities = std::collections::BTreeSet::new(); + for row in rows { + work.checkpoint().await?; + if columns + .iter() + .any(|column| matches!(row[*column], Value::Null)) + { + continue; + } + let key = group_key(row, columns)?; + if !identities.contains(&key) { + workspace.grow(key_bytes(&key))?; + identities.insert(key); + } + } + return Ok(Value::Int64( + i64::try_from(identities.len()).map_err(|_| invalid("distinct count overflow"))?, + )); + } + Reduction::FrequencyL2(column) | Reduction::FrequencyEntropy(column) => { + let mut workspace = Workspace::new(context)?; + let mut counts = BTreeMap::, u64>::new(); + let mut total = 0_u64; + for row in rows { + work.checkpoint().await?; + let value = &row[*column]; + if matches!(value, Value::Null) { + continue; + } + if matches!(value, Value::Float64(v) if !v.is_finite()) { + return Err(invalid( + "frequency aggregate requires a finite floating identity", + )); + } + let key = value.key()?; + if !counts.contains_key(&key) { + workspace.grow( + 64 + std::mem::size_of::>() + + key.len() + + std::mem::size_of::(), + )?; + } + *counts.entry(key).or_default() += 1; + total += 1; + } + let mut result = 0.0_f64; + for count in counts.into_values() { + work.checkpoint().await?; + let count = count as f64; + if matches!(measure, Reduction::FrequencyL2(_)) { + result = result.hypot(count); + } else { + let probability = count / total as f64; + result -= probability * probability.log2(); + } + } + return Ok(Value::Float64(result)); + } Reduction::Quantile { column, q } => { let mut values = Vec::with_capacity(rows.len()); for row in rows { diff --git a/crates/asap-physical-operators/src/operators/summary/mod.rs b/crates/asap-physical-operators/src/operators/summary/mod.rs index a180948ac..7e8edcfb7 100644 --- a/crates/asap-physical-operators/src/operators/summary/mod.rs +++ b/crates/asap-physical-operators/src/operators/summary/mod.rs @@ -167,8 +167,14 @@ impl Operator { ) -> Result { crate::values::validate_family(&family)?; validate_groups(&input, &groups)?; - if plain(&input, value)?.0 != &DataType::Float64 { - return Err(invalid("summary numeric update requires Float64")); + let dtype = plain(&input, value)?.0; + let typed_frequency = matches!(&family, SummaryFamilyType::Sketch(kind, _) + if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::UnivMon); + if *dtype != DataType::Float64 + && !(typed_frequency + && matches!(dtype, DataType::Utf8 | DataType::Int64 | DataType::Bool)) + { + return Err(invalid("summary update type is unsupported by its family")); } if let Some(time) = time { if plain(&input, time)? != (&DataType::Timestamp, false) { @@ -532,11 +538,10 @@ async fn build_summary( states.get_mut(&key).expect("inserted group"); // SQL aggregates ignore NULL samples while retaining the group. // A missing counter sample also contributes no observation. - let value = match row[value] { - Value::Float64(value) => value, - Value::Null => continue, - _ => return Err(invalid("summary update type")), - }; + let value = &row[value]; + if matches!(value, Value::Null) { + continue; + } let timestamp = if let Some(time) = time { let Value::Timestamp(time) = row[time] else { return Err(invalid("summary time type")); @@ -552,9 +557,8 @@ async fn build_summary( )); } updater - .validate_single_input(value) + .update_value(value, timestamp) .map_err(Error::Operator)?; - updater.update_single(value, timestamp); *previous = Some(timestamp); memory.resize(updater.memory_usage_bytes() + *overhead)?; } diff --git a/crates/asap-physical-operators/src/physical_planner/mod.rs b/crates/asap-physical-operators/src/physical_planner/mod.rs index 36fc2a247..3b0612bbf 100644 --- a/crates/asap-physical-operators/src/physical_planner/mod.rs +++ b/crates/asap-physical-operators/src/physical_planner/mod.rs @@ -315,6 +315,90 @@ fn compile_internal( )?; continue; } + // An exact per-series range reduction with no exact accumulator + // form (`distinct_over_time`, `l2_over_time`, `entropy_over_time`) + // reduces each series' raw samples in the window natively. + if let Payload::NonASAP(NonASAPOp::Aggregate { + reduction: PlannerReduction::PerEntity, + measures, + filters, + having: None, + .. + }) = &node.payload + { + let [measure] = measures.as_slice() else { + return Err(invalid("per-entity aggregate has one measure")); + }; + if filters.iter().any(Option::is_some) { + return Err(invalid("filtered aggregate has no native implementation")); + } + let [input] = schemas.as_slice() else { + return Err(invalid("per-entity aggregate requires one input")); + }; + let value = match measure.input_cols().first() { + Some(column) => *column, + None => named_column(input, &ColumnRef::SampleValue)?, + }; + let reduction = match measure { + AggIntent::Cardinality { .. } => Reduction::Cardinality(vec![value]), + AggIntent::FrequencyL2 { .. } => Reduction::FrequencyL2(value), + AggIntent::FrequencyEntropy { .. } => Reduction::FrequencyEntropy(value), + _ => { + return Err(invalid(format!( + "node {id}: per-entity aggregate requires an explicit entity binding" + ))) + } + }; + let coordinate = input + .time_index + .ok_or_else(|| invalid("temporal input lacks time"))?; + let groups = (0..input.fields.len()) + .filter(|&column| column != value && column != coordinate) + .collect(); + let aggregate = + Operator::aggregate(input.clone(), groups, vec![("value".into(), reduction)])?; + // A PromQL sample is Float64; the distinct count is Int64. + let reduced = aggregate.schema(); + let project = Operator::project( + reduced.clone(), + reduced + .fields + .iter() + .enumerate() + .map(|(i, field)| { + let expression = if field.dtype == FieldDataType::Plain(DataType::Int64) + && i + 1 == reduced.fields.len() + { + Expression::ExactFloat64(i) + } else { + Expression::Column(i) + }; + (field.name.clone(), expression) + }) + .collect(), + )?; + let projected = project.schema(); + physical_dag.add(auxiliary, inputs, aggregate)?; + physical_dag.add(helper_id(id, 1)?, vec![auxiliary], project)?; + let stamped = Operator::scope_timestamp(projected, output.clone())?; + if output + .fields + .iter() + .any(|field| field.name == promql_rows::SERIES_IDENTITY_COLUMN) + { + // Range functions drop the metric name, as their + // summary evaluations do. + physical_dag.add(helper_id(id, 2)?, vec![helper_id(id, 1)?], stamped)?; + physical_dag.add( + id, + vec![helper_id(id, 2)?], + Operator::series_without_name(output)?, + )?; + } else { + physical_dag.add(id, vec![helper_id(id, 1)?], stamped)?; + } + continue; + } // A closed row must include either all source labels or the explicit // complete-label identity. Projected labels alone are insufficient. if let Payload::ASAP(ASAPOp::SummaryAgg { @@ -343,17 +427,25 @@ fn compile_internal( let Some(NonASAPOp::Scan { schema, .. }) = source.non_asap() else { return Err(invalid("per-entity summary requires a resolved source")); }; - if !schema.closed || update.item.is_some() { + // An unkeyed frequency family (UnivMon) counts each value of + // its one item column once, as a value-frequency build. + let frequency_item = crate::capability::unit_frequency_column(update) + .filter(|_| unkeyed_frequency_family(family)); + if !schema.closed || (update.item.is_some() && frequency_item.is_none()) { return Err(invalid( "per-entity summary requires complete source identity", )); } crate::capability::validate_summary_kernel(family, update, grouping) .map_err(Error::Invalid)?; - let SummaryInputExpr::Column(value) = &update.weight else { - return Err(invalid( - "per-entity update requires a projected value column", - )); + let value = match (frequency_item, &update.weight) { + (Some(item), _) => item, + (None, SummaryInputExpr::Column(value)) => value, + (None, _) => { + return Err(invalid( + "per-entity update requires a projected value column", + )) + } }; let input = schemas[0].clone(); let value = named_column(&input, value)?; @@ -766,6 +858,19 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result Reduction::Count, + AggIntent::Cardinality { cols, .. } => { + Reduction::Cardinality(if cols.is_empty() { + vec![column(None)?] + } else { + cols.clone() + }) + } + AggIntent::FrequencyL2 { col, .. } => { + Reduction::FrequencyL2(column(*col)?) + } + AggIntent::FrequencyEntropy { col, .. } => { + Reduction::FrequencyEntropy(column(*col)?) + } AggIntent::Sum { col } => Reduction::Sum(column(*col)?), AggIntent::Avg { col } => Reduction::Avg(column(*col)?), AggIntent::Min { col } => Reduction::Min(column(*col)?), @@ -869,6 +974,27 @@ fn summary_build( groups(input, keys)?, ); } + // An unkeyed frequency family (UnivMon) counts each value of its + // one item column once: a value-frequency build over that column. + if let (Some(item), true) = ( + crate::capability::unit_frequency_column(update), + unkeyed_frequency_family(family), + ) { + crate::capability::validate_summary_kernel(family, update, grouping) + .map_err(Error::Invalid)?; + let PlannerReduction::Reduce(keys) = reduction else { + return Err(invalid( + "summary construction requires explicit grouping columns", + )); + }; + return Operator::summary_build( + input.clone(), + family.clone(), + named_column(input, item)?, + input.time_index, + groups(input, keys)?, + ); + } if let Some(item) = &update.item { let PlannerReduction::Reduce(keys) = reduction else { return Err(invalid("keyed summary requires explicit partitions")); @@ -931,6 +1057,15 @@ fn summary_build( groups(input, keys)?, ) } +/// Families whose state counts values of one column without keyed items: +/// their unit-frequency update builds over that column. +fn unkeyed_frequency_family(family: &FieldDataType) -> bool { + matches!(family, FieldDataType::Sketch(kind, _) if matches!( + kind.algorithm(), + planner_types::post_asap::SketchAlgorithm::UnivMon + | planner_types::post_asap::SketchAlgorithm::Hll + )) +} fn summary_column(input: &SchemaRef) -> Result { let columns = input .fields diff --git a/crates/asap-physical-operators/src/summary_kernels/factory.rs b/crates/asap-physical-operators/src/summary_kernels/factory.rs index 1d9da341e..caa1e6b17 100644 --- a/crates/asap-physical-operators/src/summary_kernels/factory.rs +++ b/crates/asap-physical-operators/src/summary_kernels/factory.rs @@ -46,6 +46,21 @@ pub trait AccumulatorUpdater: Send { } } + /// Update from one typed row value. Numeric kernels keep their Float64 + /// contract; a frequency kernel may also take non-numeric identities. + fn update_value( + &mut self, + value: &crate::values::Value, + timestamp_ms: i64, + ) -> Result<(), String> { + let crate::values::Value::Float64(value) = value else { + return Err("summary update requires Float64".into()); + }; + self.validate_single_input(*value)?; + self.update_single(*value, timestamp_ms); + Ok(()) + } + /// Feed a single (value, timestamp_ms) pair — for SingleSubpopulation types. fn update_single(&mut self, value: f64, timestamp_ms: i64); @@ -645,7 +660,7 @@ pub fn create_planner_accumulator( } }; if updater.is_keyed() != input.item.is_some() - && !crate::capability::is_unit_sample_frequency(input) + && crate::capability::unit_frequency_column(input).is_none() { return Err("Planner item expression does not match the selected kernel layout".into()); } @@ -718,6 +733,11 @@ impl AccumulatorUpdater for HllUpdater { } impl AccumulatorUpdater for UnivMonUpdater { + fn update_value(&mut self, value: &crate::values::Value, _: i64) -> Result<(), String> { + self.acc + .insert_value(value) + .map_err(|error| error.to_string()) + } fn is_keyed(&self) -> bool { false } diff --git a/crates/asap-physical-operators/src/summary_kernels/univmon.rs b/crates/asap-physical-operators/src/summary_kernels/univmon.rs index 63104a22b..bd8db2027 100644 --- a/crates/asap-physical-operators/src/summary_kernels/univmon.rs +++ b/crates/asap-physical-operators/src/summary_kernels/univmon.rs @@ -40,18 +40,31 @@ pub(crate) fn check_dimensions( #[derive(Debug, Clone)] pub struct UnivMonAccumulator { inner: UnivMon, + /// The longest variable-length key the heaps may retain, in bytes: the + /// memory charge bounds the retained keys by every heap slot holding + /// one this long, so it never walks the heaps (a build charges memory + /// after every row). + max_key_bytes: usize, + /// The bytes of every variable-length key inserted (or merged in), + /// counted once per insertion: each layer's heap keeps at most one copy + /// of a key, so the heaps retain at most `layers` times this. + inserted_key_bytes: usize, } impl UnivMonAccumulator { /// Empty the sketch in place, keeping its shape. pub(crate) fn clear(&mut self) { self.inner.free(); + self.max_key_bytes = 0; + self.inserted_key_bytes = 0; } pub fn new(heap_size: usize, rows: usize, cols: usize, layers: usize) -> Result { check_dimensions(heap_size, rows, cols, layers)?; Ok(Self { inner: UnivMon::init_univmon(heap_size, rows, cols, layers), + max_key_bytes: 0, + inserted_key_bytes: 0, }) } @@ -68,7 +81,21 @@ impl UnivMonAccumulator { if !sketch.accepts_standard_updates() { return Err("terminal-mode UnivMon state cannot accept standard updates".into()); } - Ok(Self { inner: sketch }) + // One walk of the decoded heaps, not one per charge. + let keys: Vec = (0..sketch.layer_size) + .flat_map(|layer| sketch.hh_layers[layer].heap()) + .map(|item| match &item.key { + asap_sketchlib::HeapItem::String(key) => key.capacity(), + asap_sketchlib::HeapItem::Bytes(key) => key.capacity(), + _ => 0, + }) + .collect(); + Ok(Self { + inner: sketch, + max_key_bytes: keys.iter().copied().max().unwrap_or(0), + // The retained keys themselves (already counted per layer). + inserted_key_bytes: keys.iter().fold(0, |a, &b| a.saturating_add(b)), + }) } /// The underlying sketch, so a deployment can encode it with sketchlib's codec. @@ -90,6 +117,29 @@ impl UnivMonAccumulator { Ok(()) } + /// One occurrence of a typed identity. A finite Float64 is a sample + /// ([`Self::insert_sample`]); Bool, Int64 and Utf8 keep their type as + /// type-tagged key bytes, since converting Int64 to Float64 would collapse + /// neighbouring keys above 2^53. NULL is skipped. + pub fn insert_value(&mut self, value: &crate::values::Value) -> Result<(), Error> { + use crate::values::Value; + match value { + Value::Null => return Ok(()), + Value::Float64(number) if number.is_finite() => return self.insert_sample(*number), + Value::Bool(_) | Value::Int64(_) | Value::Utf8(_) => {} + _ => return Err("unsupported UnivMon identity type or non-finite sample".into()), + } + let key = value.key()?; + self.inner + .bucket_size + .checked_add(1) + .ok_or("UnivMon count overflow")?; + self.inner.insert(&DataInput::Bytes(&key), 1); + self.max_key_bytes = self.max_key_bytes.max(key.len()); + self.inserted_key_bytes = self.inserted_key_bytes.saturating_add(key.len()); + Ok(()) + } + fn compatible(&self, other: &Self) -> bool { ( self.inner.heap_size, @@ -122,21 +172,38 @@ impl UnivMonAccumulator { .checked_add(other.inner.bucket_size) .ok_or("UnivMon count overflow")?; self.inner.merge(&other.inner); + self.max_key_bytes = self.max_key_bytes.max(other.max_key_bytes); + self.inserted_key_bytes = self + .inserted_key_bytes + .saturating_add(other.inserted_key_bytes); Ok(()) } } impl AggregateCore for UnivMonAccumulator { + /// Counters and heap slots of every layer, plus an upper bound on the + /// variable-length keys the heaps retain: the smaller of every slot + /// holding the longest key seen and every layer holding a copy of every + /// key inserted. Constant time, since a build charges memory after + /// every row; the library exposes no eviction hook for an exact running + /// total, so a stream of long keys is still overcounted. fn approx_memory_bytes(&self) -> usize { - std::mem::size_of::().saturating_add( - self.inner.layer_size.saturating_mul( - self.inner - .sketch_row - .saturating_mul(self.inner.sketch_col) - .saturating_mul(16) - .saturating_add(self.inner.heap_size.saturating_mul(256)), - ), - ) + let layers = self.inner.layer_size; + let key_bytes = layers + .saturating_mul(self.inner.heap_size) + .saturating_mul(self.max_key_bytes) + .min(layers.saturating_mul(self.inserted_key_bytes)); + key_bytes + .saturating_add(std::mem::size_of::()) + .saturating_add( + self.inner.layer_size.saturating_mul( + self.inner + .sketch_row + .saturating_mul(self.inner.sketch_col) + .saturating_mul(16) + .saturating_add(self.inner.heap_size.saturating_mul(256)), + ), + ) } fn clone_boxed_core(&self) -> Box { Box::new(self.clone()) @@ -248,4 +315,117 @@ mod tests { assert!(UnivMonAccumulator::new(4, 11, 2047, 1).is_err()); assert!(UnivMonAccumulator::new(4, 10, 2047, 1).is_ok()); } + + // Typed keys stay distinct, and merge, when entered as type-tagged bytes: + // Int64 identities above 2^53 do not collapse. + #[test] + fn typed_keys_merge_and_stay_distinct() { + use crate::values::Value; + for keys in [ + [ + Value::Utf8("192.0.2.1".into()), + Value::Utf8("192.0.2.2".into()), + ], + [ + Value::Int64(9_007_199_254_740_992), + Value::Int64(9_007_199_254_740_993), + ], + [Value::Bool(false), Value::Bool(true)], + ] { + let mut left = UnivMonAccumulator::new(32, 5, 1024, 4).unwrap(); + let mut right = UnivMonAccumulator::new(32, 5, 1024, 4).unwrap(); + for _ in 0..2 { + left.insert_value(&keys[0]).unwrap(); + right.insert_value(&keys[1]).unwrap(); + } + left.insert_value(&Value::Null).unwrap(); + left.merge_in_place(&right).unwrap(); + for (statistic, expected) in [ + (count(), 4.0), + (SketchStatistic::Cardinality, 2.0), + (SketchStatistic::FrequencyL2, 8.0_f64.sqrt()), + (SketchStatistic::FrequencyEntropy, 1.0), + ] { + let estimate = left.estimate(&statistic).unwrap(); + assert!( + (estimate - expected).abs() < 0.01, + "{keys:?} {statistic:?}: {estimate}" + ); + } + } + } + + // A non-finite sample or an unsupported identity type is an error. + #[test] + fn rejects_non_finite_and_untyped_identities() { + use crate::values::Value; + let mut state = UnivMonAccumulator::new(32, 5, 1024, 4).unwrap(); + assert!(state.insert_value(&Value::Float64(f64::INFINITY)).is_err()); + assert!(state.insert_value(&Value::Timestamp(1)).is_err()); + } + + // Retained variable-length identities count toward the state's memory. + #[test] + fn memory_accounts_for_string_identities() { + let mut state = UnivMonAccumulator::new(32, 5, 1024, 4).unwrap(); + let empty = state.approx_memory_bytes(); + state + .insert_value(&crate::values::Value::Utf8("x".repeat(4096).into())) + .unwrap(); + assert!(state.approx_memory_bytes() >= empty + 4096); + state.clear(); + assert_eq!(state.approx_memory_bytes(), empty); + } + + // A build charges memory after every row, so the charge must not walk + // the heaps: once the keys inserted outnumber the heap slots, keys of + // one length are charged the same however many keys the heaps hold. + #[test] + fn memory_charge_does_not_grow_with_heap_occupancy() { + let mut state = UnivMonAccumulator::new(512, 3, 64, 4).unwrap(); + let key = |i: i64| crate::values::Value::Utf8(format!("{i:08}").into()); + for i in 0..1000 { + state.insert_value(&key(i)).unwrap(); + } + let full = state.approx_memory_bytes(); + for i in 1000..4000 { + state.insert_value(&key(i)).unwrap(); + } + assert_eq!(state.approx_memory_bytes(), full); + } + + // One long key in a large sketch is charged at most once per layer (a + // key's heap copies), not once per heap slot (19 layers of 20,000 + // slots of 1 KB would be 380 MB). + #[test] + fn one_long_key_is_charged_about_once() { + let mut state = UnivMonAccumulator::new(20_000, 3, 64, 19).unwrap(); + let empty = state.approx_memory_bytes(); + state + .insert_value(&crate::values::Value::Utf8("x".repeat(1024).into())) + .unwrap(); + let charged = state.approx_memory_bytes() - empty; + assert!((1024..=19 * 1100).contains(&charged), "{charged}"); + } + + // Charging memory after every row of a large build stays linear in the + // rows (it was quadratic: 3 s for this build). Timing-based, so ignored + // by default; run with `--ignored --release`. + #[test] + #[ignore = "timing; run in release"] + fn per_row_memory_charge_of_a_large_build_is_fast() { + let mut state = UnivMonAccumulator::new(20_000, 3, 1024, 19).unwrap(); + let start = std::time::Instant::now(); + let mut charged = 0usize; + for i in 0..40_000i64 { + state.insert_value(&crate::values::Value::Int64(i)).unwrap(); + charged = charged.max(state.approx_memory_bytes()); + } + assert!(charged > 0); + assert!( + start.elapsed() < std::time::Duration::from_millis(500), + "{:?}", + start.elapsed() + ); + } } diff --git a/crates/asap-physical-operators/src/values.rs b/crates/asap-physical-operators/src/values.rs index 33f8ff2b4..830a624a8 100644 --- a/crates/asap-physical-operators/src/values.rs +++ b/crates/asap-physical-operators/src/values.rs @@ -277,7 +277,7 @@ fn validate_state(family: &SummaryFamilyType, state: &dyn AggregateCore) -> Resu use crate::summary_kernels::{ count_min_sketch::CountMinSketchAccumulator, datasketches_kll::DatasketchesKLLAccumulator, dd_sketch::DDSketchAccumulator, exact::ExactAccumulator, hll_sketch::HllSketchAccumulator, - kmv::KmvAccumulator, + kmv::KmvAccumulator, univmon::UnivMonAccumulator, }; use planner_types::post_asap::SketchParams; validate_family(family)?; @@ -328,6 +328,24 @@ fn validate_state(family: &SummaryFamilyType, state: &dyn AggregateCore) -> Resu .as_any() .downcast_ref::() .is_some_and(|s| s.inner.k == *k as usize), + SketchParams::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, + .. + } => state + .as_any() + .downcast_ref::() + .is_some_and(|s| { + s.dimensions() + == ( + *heap_size as usize, + *sketch_rows as usize, + *sketch_cols as usize, + usize::from(*layers), + ) + }), SketchParams::Cms { width, depth } => state .as_any() .downcast_ref::() diff --git a/crates/asap-physical-operators/tests/blocking_resources.rs b/crates/asap-physical-operators/tests/blocking_resources.rs index 09cee7a4e..24bc1244b 100644 --- a/crates/asap-physical-operators/tests/blocking_resources.rs +++ b/crates/asap-physical-operators/tests/blocking_resources.rs @@ -248,3 +248,49 @@ fn weighted_summary_build_yields_within_a_batch() { drop(output); assert_eq!(run.retained_bytes(), 0); } + +// Frequency and distinct dictionaries count against the budget and release +// their reservations on failure. +#[test] +fn frequency_dictionary_enforces_memory_budget() { + use asap_physical_operators::operators::Reduction; + let input = schema(1); + let mut sources = PhysicalDAG::default(); + sources + .add( + 0, + vec![], + Operator::source( + input.clone(), + vec![Batch::try_new( + input.clone(), + (0..64).map(|i| vec![Value::Int64(i)]).collect(), + ) + .unwrap()], + ) + .unwrap(), + ) + .unwrap(); + for (reduction, succeeds) in [ + (Reduction::Count, true), + (Reduction::FrequencyL2(0), false), + (Reduction::FrequencyEntropy(0), false), + (Reduction::Cardinality(vec![0]), false), + ] { + // Count needs about 13.4 kB here, the dictionaries at least 14.5 kB. + let run = context(14_000); + let inputs = sources.execute(&[0], run.clone()).unwrap(); + let operator = + Operator::aggregate(input.clone(), vec![], vec![("result".into(), reduction)]).unwrap(); + let mut output = operator.start(inputs, run.clone()).unwrap(); + let result = block_on(output.next()).unwrap(); + if succeeds { + assert!(result.is_ok(), "{result:?}"); + } else { + assert!(matches!(result, Err(Error::MemoryLimit)), "{result:?}"); + } + drop(result); + drop(output); + assert_eq!(run.retained_bytes(), 0); + } +} diff --git a/crates/asap-physical-operators/tests/physical_semantics.rs b/crates/asap-physical-operators/tests/physical_semantics.rs index a934b99ce..627a14421 100644 --- a/crates/asap-physical-operators/tests/physical_semantics.rs +++ b/crates/asap-physical-operators/tests/physical_semantics.rs @@ -1155,3 +1155,242 @@ fn measure_filters_are_rejected_over_promql_series_identity() { .expect("filtered measure over PromQL series identity"); assert!(error.to_string().contains("PromQL"), "{error}"); } + +/// An exact aggregate node over `input` with one `measure` named `name`, +/// whose output is one non-null `dtype` column. +fn exact_aggregate_node( + measure: planner_types::pre_asap::AggIntent, + name: &str, + dtype: DataType, +) -> PhysicalASAPDAGNode { + use planner_types::{ + post_asap::ExecutionDataState, + pre_asap::{GroupKeys, Reduction as PlanReduction}, + }; + PhysicalASAPDAGNode { + id: 1, + payload: PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Aggregate { + reduction: PlanReduction::Reduce(GroupKeys::by(vec![])), + measures: vec![measure], + output_names: vec![name.into()], + filters: vec![], + having: None, + child: 0, + }), + output_state: ExecutionDataState::QUERY_ROWS, + materialization: Default::default(), + output_schema: (*schema(&[(name, dtype, false)])).clone(), + guarantee: None, + } +} + +// Exact frequency intents bind to native reducers over typed keys, without a +// sketch or numeric key conversion; empty and all-NULL input give 0. +#[test] +fn exact_frequency_intents_execute_typed_keys_and_empty_input() { + use asap_physical_operators::physical_planner::compile_node; + use planner_types::pre_asap::AggIntent; + use planner_types::types::AccuracyTarget; + for (dtype, values) in [ + ( + DataType::Utf8, + vec![Value::Utf8("a".into()), Value::Utf8("b".into())], + ), + ( + DataType::Int64, + vec![ + Value::Int64(9_007_199_254_740_992), + Value::Int64(9_007_199_254_740_993), + ], + ), + (DataType::Bool, vec![Value::Bool(false), Value::Bool(true)]), + ( + DataType::Float64, + vec![Value::Float64(-0.0), Value::Float64(1.0)], + ), + ] { + let input = schema(&[("key", dtype, true)]); + for (measure, name, expected) in [ + ( + AggIntent::FrequencyL2 { + col: Some(0), + accuracy: AccuracyTarget::exact(), + }, + "frequency_l2", + 8.0_f64.sqrt(), + ), + ( + AggIntent::FrequencyEntropy { + col: Some(0), + accuracy: AccuracyTarget::exact(), + }, + "frequency_entropy", + 1.0, + ), + ] { + let node = exact_aggregate_node(measure, name, DataType::Float64); + let operator = compile_node(&node, std::slice::from_ref(&input)) + .expect("exact frequency intent binds"); + let rows = values + .iter() + .flat_map(|v| [vec![v.clone()], vec![v.clone()]]) + .chain([vec![Value::Null]]) + .collect(); + let result = unary(input.clone(), vec![rows], operator.clone()); + assert!( + matches!(result[0][0], Value::Float64(v) if (v - expected).abs() < 1e-12), + "{name}: {result:?}" + ); + for batches in [vec![], vec![vec![vec![Value::Null]]]] { + let result = unary(input.clone(), batches, operator.clone()); + assert!(matches!(result[0][0], Value::Float64(v) if v == 0.0)); + } + } + } +} + +// Each group gets its own frequency population, with one canonical +// signed-zero identity; entropy is in bits. +#[test] +fn exact_frequency_grouping_and_entropy_bits() { + let input = schema(&[ + ("group", DataType::Int64, false), + ("key", DataType::Float64, true), + ]); + let operator = Operator::aggregate( + input.clone(), + vec![0], + vec![ + ("l2".into(), Reduction::FrequencyL2(1)), + ("entropy".into(), Reduction::FrequencyEntropy(1)), + ], + ) + .unwrap(); + let rows = vec![ + vec![Value::Int64(1), Value::Float64(-0.0)], + vec![Value::Int64(1), Value::Float64(0.0)], + vec![Value::Int64(1), Value::Float64(0.0)], + vec![Value::Int64(1), Value::Float64(1.0)], + vec![Value::Int64(2), Value::Float64(2.0)], + vec![Value::Int64(2), Value::Null], + vec![Value::Int64(3), Value::Null], + ]; + let result = unary(input.clone(), vec![rows], operator.clone()); + assert_eq!(result.len(), 3); + let expected_entropy = -0.75_f64 * 0.75_f64.log2() - 0.25_f64 * 0.25_f64.log2(); + for (row, l2, entropy) in [ + (&result[0], 10.0_f64.sqrt(), expected_entropy), + (&result[1], 1.0, 0.0), + (&result[2], 0.0, 0.0), + ] { + assert!(matches!(row[1], Value::Float64(v) if (v - l2).abs() < 1e-12)); + assert!(matches!(row[2], Value::Float64(v) if (v - entropy).abs() < 1e-12)); + } + assert!(unary(input, vec![], operator).is_empty()); +} + +// A non-finite floating identity is an error, not a frequency key. +#[test] +fn exact_frequency_rejects_non_finite_identities() { + let input = schema(&[("key", DataType::Float64, false)]); + let operator = Operator::aggregate( + input.clone(), + vec![], + vec![("l2".into(), Reduction::FrequencyL2(0))], + ) + .unwrap(); + let mut dag = PhysicalDAG::default(); + dag.add( + 0, + vec![], + Operator::source( + input.clone(), + vec![Batch::try_new(input, vec![vec![Value::Float64(f64::INFINITY)]]).unwrap()], + ) + .unwrap(), + ) + .unwrap(); + dag.add(1, vec![0], operator).unwrap(); + let run = context(); + let result = block_on(async { + let mut stream = dag.execute(&[1], run.clone()).unwrap().remove(0); + stream.next().await.unwrap() + }); + assert!(result.is_err()); +} + +// Exact distinct binding keeps typed tuples, skips any tuple with a NULL +// and returns zero on no input. +#[test] +fn exact_cardinality_binds_and_executes_typed_tuples() { + use asap_physical_operators::physical_planner::compile_node; + use planner_types::pre_asap::AggIntent; + use planner_types::types::AccuracyTarget; + let input = schema(&[ + ("key", DataType::Int64, true), + ("tag", DataType::Utf8, true), + ]); + for (cols, expected) in [(vec![0], 2), (vec![0, 1], 3)] { + let node = exact_aggregate_node( + AggIntent::Cardinality { + cols, + accuracy: AccuracyTarget::exact(), + }, + "distinct", + DataType::Int64, + ); + let operator = + compile_node(&node, std::slice::from_ref(&input)).expect("exact distinct intent binds"); + let rows = vec![ + vec![Value::Int64(9_007_199_254_740_992), Value::Utf8("a".into())], + vec![Value::Int64(9_007_199_254_740_992), Value::Utf8("a".into())], + vec![Value::Int64(9_007_199_254_740_992), Value::Utf8("b".into())], + vec![Value::Int64(9_007_199_254_740_993), Value::Utf8("a".into())], + vec![Value::Null, Value::Utf8("c".into())], + ]; + let result = unary(input.clone(), vec![rows], operator.clone()); + assert!(matches!(result[0][0], Value::Int64(v) if v == expected)); + for rows in [vec![], vec![vec![Value::Null, Value::Null]]] { + let result = unary(input.clone(), vec![rows], operator.clone()); + assert!(matches!(result[0][0], Value::Int64(0))); + } + } +} + +// Distinct uses grouped equality: signed zero and NaN payloads each form one +// identity. +#[test] +fn exact_cardinality_grouping_normalizes_float_identities() { + let input = schema(&[ + ("group", DataType::Int64, false), + ("key", DataType::Float64, true), + ]); + let operator = Operator::aggregate( + input.clone(), + vec![0], + vec![("distinct".into(), Reduction::Cardinality(vec![1]))], + ) + .unwrap(); + let result = unary( + input, + vec![vec![ + vec![Value::Int64(1), Value::Float64(0.0)], + vec![Value::Int64(1), Value::Float64(-0.0)], + vec![Value::Int64(1), Value::Float64(f64::NAN)], + vec![ + Value::Int64(1), + Value::Float64(f64::from_bits(f64::NAN.to_bits() + 1)), + ], + vec![Value::Int64(2), Value::Null], + ]], + operator, + ); + assert!(matches!( + result[0].as_slice(), + [Value::Int64(1), Value::Int64(2)] + )); + assert!(matches!( + result[1].as_slice(), + [Value::Int64(2), Value::Int64(0)] + )); +} diff --git a/crates/asap-physical-operators/tests/univmon_execution.rs b/crates/asap-physical-operators/tests/univmon_execution.rs new file mode 100644 index 000000000..fc8725117 --- /dev/null +++ b/crates/asap-physical-operators/tests/univmon_execution.rs @@ -0,0 +1,208 @@ +//! One native UnivMon build supplies the three statistics of #509 Example 2: +//! a distinct count, an entropy and an L2 norm of one identity column. +use asap_physical_operators::{ + operators::{Operator, SummaryEvaluation}, + physical_planner::compile_node, + plan::{PhysicalDAG, PhysicalOperator}, + runtime::{Limits, RunContext, Scope}, + values::{Batch, SchemaRef, Value}, + Error, +}; +use futures::{executor::block_on, StreamExt}; +use planner_types::ir::physical_export::{PhysicalASAPDAGNode, PhysicalASAPOperatorPayload}; +use planner_types::ir::ASAPOp; +use planner_types::post_asap::{ + ExecutionDataState, Field, FieldDataType, GroupingStrategy, NonNegativeWeightProof, + SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, SummaryInputExpr, SummaryUpdate, + WeightDomain, +}; +use planner_types::pre_asap::{ColumnRef, DataType, GroupKeys, Reduction, Schema}; +use std::sync::Arc; + +fn family() -> FieldDataType { + FieldDataType::Sketch( + SketchKind::new( + SketchAlgorithm::UnivMon, + SketchParams::UnivMon { + heap_size: 64, + sketch_rows: 5, + sketch_cols: 128, + layers: 4, + distinct_failure_probability: None, + }, + ), + GroupingStrategy::default(), + ) +} + +fn context() -> RunContext { + RunContext::new( + Scope::Query { + evaluation_time_ms: 0, + revision: 1, + }, + Limits::default(), + ) + .unwrap() +} + +fn value(row: &[Value]) -> f64 { + match row { + [Value::Float64(value)] => *value, + other => panic!("expected one floating point result, got {other:?}"), + } +} + +/// Build `build` over `batch` and read the three statistics from its state. +fn three_statistics(batch: Batch, build: Operator) -> Result, Error> { + let state = build.output_schema(); + let mut dag = PhysicalDAG::default(); + dag.add( + 0, + vec![], + Operator::source(batch.schema().clone(), vec![batch])?, + )?; + dag.add(1, vec![0], build)?; + for (id, statistic) in [ + (2, SketchStatistic::Cardinality), + (3, SketchStatistic::FrequencyL2), + (4, SketchStatistic::FrequencyEntropy), + ] { + dag.add( + id, + vec![1], + Operator::evaluation(state.clone(), 0, SummaryEvaluation::Sketch(statistic))?, + )?; + } + let outputs = block_on(futures::future::join_all( + dag.execute(&[2, 3, 4], context())? + .into_iter() + .map(|stream| stream.collect::>()), + )); + outputs + .into_iter() + .map(|stream| { + let batches = stream.into_iter().collect::, _>>()?; + Ok(value(&batches[0].rows()[0])) + }) + .collect() +} + +// One UnivMon state over Float64 samples answers distinct, L2 and entropy. +#[test] +fn one_univmon_state_answers_distinct_l2_and_entropy() -> Result<(), Error> { + let input: SchemaRef = Arc::new(Schema::new(vec![Field::plain( + "value", + DataType::Float64, + false, + )])); + let batch = Batch::try_new( + input.clone(), + [1.0, 1.0, 2.0, 2.0, 2.0] + .into_iter() + .map(|value| vec![Value::Float64(value)]) + .collect(), + )?; + let build = Operator::summary_build(input, family(), 0, None, vec![])?; + let [distinct, l2, entropy] = three_statistics(batch, build)?[..] else { + unreachable!() + }; + assert!((distinct - 2.0).abs() < 0.01); + assert!((l2 - 13.0_f64.sqrt()).abs() < 0.01); + let expected = -0.4_f64 * 0.4_f64.log2() - 0.6 * 0.6_f64.log2(); + assert!((entropy - expected).abs() < 0.01, "{entropy}"); + Ok(()) +} + +// SQL source IPs and integer identifiers enter frequency state without +// numeric coercion; NULL identities are skipped. +#[test] +fn typed_frequency_keys_preserve_identity() -> Result<(), Error> { + for (dtype, keys) in [ + ( + DataType::Utf8, + [ + Value::Utf8("192.0.2.1".into()), + Value::Utf8("192.0.2.2".into()), + ], + ), + ( + DataType::Int64, + [ + Value::Int64(9_007_199_254_740_992), + Value::Int64(9_007_199_254_740_993), + ], + ), + (DataType::Bool, [Value::Bool(false), Value::Bool(true)]), + ] { + let input: SchemaRef = Arc::new(Schema::new(vec![Field::plain("src_ip", dtype, true)])); + let batch = Batch::try_new( + input.clone(), + vec![ + vec![keys[0].clone()], + vec![keys[0].clone()], + vec![keys[1].clone()], + vec![keys[1].clone()], + vec![Value::Null], + ], + )?; + let build = Operator::summary_build(input, family(), 0, None, vec![])?; + for (found, expected) in + three_statistics(batch, build)? + .into_iter() + .zip([2.0, 8.0_f64.sqrt(), 1.0]) + { + assert!((found - expected).abs() < 0.01, "{found} vs {expected}"); + } + } + Ok(()) +} + +// Pass 1's UnivMon update (each item column value is one unit occurrence) +// binds as a value-frequency build over that column, a Utf8 one included. +#[test] +fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<(), Error> { + let input: SchemaRef = Arc::new(Schema::new(vec![Field::plain( + "src_ip", + DataType::Utf8, + false, + )])); + let state = Arc::new(Schema::new(vec![Field { + table: None, + name: "state".into(), + dtype: family(), + nullable: false, + }])); + let node = PhysicalASAPDAGNode { + id: 1, + payload: PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { + child: 0, + family: family(), + input: SummaryUpdate { + item: Some(SummaryInputExpr::Column(ColumnRef::Named("src_ip".into()))), + weight: SummaryInputExpr::Constant(1.0), + weight_domain: WeightDomain::NonNegative { + proof: NonNegativeWeightProof::UnitCount, + }, + }, + reduction: Reduction::Reduce(GroupKeys::by(vec![])), + grouping: GroupingStrategy::default(), + filter: None, + }), + output_state: ExecutionDataState::QUERY_ROWS, + materialization: Default::default(), + output_schema: (*state).clone(), + guarantee: None, + }; + let build = compile_node(&node, std::slice::from_ref(&input))?; + let batch = Batch::try_new( + input, + ["a", "a", "b", "b"] + .into_iter() + .map(|key| vec![Value::Utf8(key.into())]) + .collect(), + )?; + let found = three_statistics(batch, build)?; + assert!((found[0] - 2.0).abs() < 0.01, "{found:?}"); + Ok(()) +} diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index 323df79ab..f965c8a7e 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -1,7 +1,8 @@ // cargo run -p asap-devtools --bin stage_pipeline -- \ // --example planner-layering-1 --out planner-layering-example1.json -// (also planner-layering-3a and planner-layering-3b: #509 Example 3, -// Patterns A and B; 3a has 972 candidates, all priced, 64 shown by default) +// (also planner-layering-2: #509 Example 2, three SQL flow statistics; +// planner-layering-3a and planner-layering-3b: #509 Example 3, Patterns A +// and B; 3a has 972 candidates, all priced, 64 shown by default) // cargo run -p asap-devtools --bin stage_pipeline -- \ // --promql "topk by (job) (10, rate(x[1m]))" --epsilon 0.01 --delta 0.001 --out run.json // cargo run -p asap-devtools --bin stage_pipeline -- \ @@ -14,8 +15,10 @@ // - stage0_query_language_frontends: the frontends' workload DAG, one root per query; // - stage1_logical_asap_optimization: Pass 1 whole-workload candidates, one per // choice of a per-computation candidate for every target (the Cartesian -// product), each followed by its Pass 2 identical-expression candidates -// (one per subset of its sharing groups), in enumeration order. Every candidate is optimized and +// product), each followed by its Pass 2 candidates (the +// identical-expression rule's, one per subset of its sharing groups, +// then the summary-capability rule's, one per choice of shared summary +// builds), in enumeration order. Every candidate is optimized and // priced; the document shows the first `--max-displayed` (default 64) // and the selected one, with `combinations` (how many exist) and // `capped` (whether some are not shown). A candidate that cannot be @@ -59,7 +62,7 @@ use std::collections::HashSet; use asap_aware_mapping::stage3_plan_selection::{Rejection, Selection, SelectionError}; use asap_devtools::planner_layering_examples::{ - declared, example1, pattern_a, pattern_b, PatternARecurrence, + declared, example1, example2, example2_catalog, pattern_a, pattern_b, PatternARecurrence, }; use asap_devtools::planning_stages::{ lower_promql_roots, lower_sql_roots, run_planning_stages, run_planning_stages_over, @@ -78,7 +81,7 @@ use asap_types::workload::{ use serde_json::{json, Value}; const USAGE: &str = - "usage: stage_pipeline (--example planner-layering-{1,3a,3b} | ((--promql | --sql) \ + "usage: stage_pipeline (--example planner-layering-{1,2,3a,3b} | ((--promql | --sql) \ [--name ]... [--table ...] [--epsilon --delta --metric ] [--interval-ms ])) \ [--max-combinations ] [--max-displayed ] --out "; @@ -211,13 +214,17 @@ fn run(args: Vec) -> Result<(), Failure> { (Some(example), true) => { let workload = match example { "planner-layering-1" => example1(), + "planner-layering-2" => example2(), "planner-layering-3a" => { pattern_a(PatternARecurrence::OnceAdHoc, DataArrival::Mixed) } "planner-layering-3b" => pattern_b(), other => return Err(format!("unknown example {other}").into()), }; - let roots = lower_promql_roots(&workload)?; + let roots = match workload.query_workload.language { + QueryLanguage::PromQL => lower_promql_roots(&workload)?, + _ => lower_sql_roots(&workload, &example2_catalog())?, + }; let result = run_planning_stages(&workload, roots.clone(), Some(max_combinations)); document_of(workload_queries(&workload), &roots, result, max_displayed)? } diff --git a/crates/devtools/src/planner_layering_examples.rs b/crates/devtools/src/planner_layering_examples.rs index 6789bd995..a964b956a 100644 --- a/crates/devtools/src/planner_layering_examples.rs +++ b/crates/devtools/src/planner_layering_examples.rs @@ -1,4 +1,4 @@ -//! The workloads of #509 Examples 1, 3 and 4 +//! The workloads of #509 Examples 1, 2, 3 and 4 //! (`docs/design_docs/proposals/planner-layering.md`), shared by //! `stage_pipeline` and the planner-layering acceptance tests. @@ -7,7 +7,7 @@ use asap_types::workload::{ AccuracyRequirement, BatchEntry, DataArrival, DataDistribution, DataWorkload, DurationMs, Evidence, EvidenceSource, LatencyRequirement, MetricType, PlanningWorkload, Predictability, Query, QueryLanguage, QueryRequirements, QueryTimeScope, QueryWorkload, Rate, RepeatedDemand, - RepeatingEntry, RepetitionInterval, TimeSelection, TimestampMs, + RepeatingEntry, RepetitionInterval, SqlDialect, TimeSelection, TimestampMs, }; pub fn declared(value: T) -> Evidence { @@ -88,6 +88,78 @@ pub fn example1() -> PlanningWorkload { } } +/// Example 2's three SQL queries over the last minute of `flows`: the +/// distinct sources, the entropy of the per-source counts and their L2 +/// norm. Q3 casts the counts to DOUBLE: the design's Int64 `c * c` can +/// overflow, so the SQL frontend recognizes only the Float64 product. +pub const EXAMPLE2_SQL: [&str; 3] = [ + "SELECT COUNT(DISTINCT src_ip) FROM flows WHERE ts >= now() - INTERVAL '1 minute'", + "SELECT -SUM(p * LN(p)) FROM (\ + SELECT COUNT(*) * 1.0 / SUM(COUNT(*)) OVER () AS p FROM flows \ + WHERE ts >= now() - INTERVAL '1 minute' GROUP BY src_ip)", + "SELECT SQRT(SUM(CAST(c AS DOUBLE) * CAST(c AS DOUBLE))) FROM (\ + SELECT src_ip, COUNT(*) AS c FROM flows \ + WHERE ts >= now() - INTERVAL '1 minute' GROUP BY src_ip)", +]; + +/// (ε, δ) of Example 2's Q1, Q2 and Q3; Q3 is the strictest. +pub const EXAMPLE2_ACCURACY: [(f64, f64); 3] = [(0.02, 0.01), (0.05, 0.01), (0.01, 0.01)]; + +/// Example 2's `flows(ts, src_ip)` table: the design names only these +/// two columns. +pub fn example2_catalog() -> asap_frontend_sql::SqlCatalog { + use asap_types::pre_asap::schema::{DataType, Field, Schema}; + asap_frontend_sql::SqlCatalog::new().with_table( + "flows", + Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("src_ip", DataType::Utf8, false), + ], + 0, + vec![], + ), + ) +} + +/// Example 2: three dashboard panels over one flow stream, every 10 s +/// over the last minute, with 10,000,000 distinct sources. +pub fn example2() -> PlanningWorkload { + PlanningWorkload { + query_workload: QueryWorkload { + language: QueryLanguage::SQL(SqlDialect::DataFusionSQL), + query_batch: None, + repeating_queries: Some( + EXAMPLE2_SQL + .iter() + .zip(EXAMPLE2_ACCURACY) + .map(|(query, (epsilon, delta))| RepeatingEntry { + query: Query((*query).into()), + demand: RepeatedDemand::FixedInterval(RepetitionInterval(10_000)), + requirements: QueryRequirements { + accuracy: AccuracyRequirement::Explicit( + RequestedAccuracy::epsilon_delta(epsilon, delta), + ), + response_latency: LatencyRequirement::Unspecified, + }, + predictability: Predictability::Predictable { known_at: None }, + time_selection: TimeSelection { + scope: QueryTimeScope::RealTime, + lookback: Some(DurationMs(MINUTE_MS)), + as_of: None, + }, + }) + .collect(), + ), + }, + data_workload: Some(DataWorkload { + data_ingestion_interval: Evidence::default(), + input_cardinality: declared(10_000_000), + ..shared_data_workload(DataArrival::ContinuouslyIngesting) + }), + } +} + /// Pattern A's five queries: (PromQL, `lookback`, T − `as_of`). pub const PATTERN_A: [(&str, u64, u64); 5] = [ ("quantile_over_time(0.99, latency_ms[5y])", 5 * YEAR_MS, 0), diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index 17fc08ef0..869915abc 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -39,21 +39,31 @@ pub fn lower_promql_roots(workload: &PlanningWorkload) -> Result, .collect() } -/// Lower a SQL workload over the tables of `catalog`. +/// Lower a SQL workload over the tables of `catalog`: every entry, batch +/// and repeating, in workload order, each with its own accuracy target. pub fn lower_sql_roots( workload: &PlanningWorkload, catalog: &asap_frontend_sql::SqlCatalog, ) -> Result, String> { - tokio::runtime::Builder::new_current_thread() + let dialect = match &workload.query_workload.language { + asap_types::workload::QueryLanguage::SQL(dialect) => dialect.clone(), + _ => asap_types::workload::SqlDialect::DataFusionSQL, + }; + let runtime = tokio::runtime::Builder::new_current_thread() .build() - .map_err(|e| e.to_string())? - .block_on(asap_frontend_sql::lower_sql_batch( - &workload.query_workload, - catalog, - )) - .into_iter() - .map(|root| { - root.map(QueryRoot::Operator) + .map_err(|e| e.to_string())?; + workload + .query_workload + .entries() + .map(|entry| { + runtime + .block_on(asap_frontend_sql::lower_sql_dialect( + &entry.query.0, + catalog, + dialect.clone(), + entry.requirements.accuracy.requested(), + )) + .map(QueryRoot::Operator) .map_err(|e| format!("lowering: {e}")) }) .collect() diff --git a/crates/devtools/tests/stage_pipeline.rs b/crates/devtools/tests/stage_pipeline.rs index 27184d8bb..c8cdda2e7 100644 --- a/crates/devtools/tests/stage_pipeline.rs +++ b/crates/devtools/tests/stage_pipeline.rs @@ -1,4 +1,4 @@ -//! `stage_pipeline` writes a valid four-stage viewer document for #509 Example 1. +//! `stage_pipeline` writes valid four-stage viewer documents for #509's examples. use std::collections::HashSet; use std::path::PathBuf; use std::process::Command; @@ -210,6 +210,42 @@ fn example3a_selects_the_cheapest_of_all_candidates() { assert!(physical.iter().any(|p| p["id"] == expected.as_str())); } +/// Example 2 plans its three SQL queries: Stage 1 lists a candidate in +/// which one UnivMon build serves all three, and plan selection selects a +/// plan. +#[test] +fn example2_plans_the_sql_workload_with_one_univmon_for_all_three() { + let document = generate_example("planner-layering-2", &["--max-displayed", "100000"]); + let queries = document["workload"]["queries"].as_array().unwrap(); + assert_eq!(queries.len(), 3); + assert!(queries.iter().all(|q| q["language"] == "sql")); + validated_dag(&document["stage0_query_language_frontends"]["dag"], 3); + let univmon_builds = |candidate: &Value| { + candidate["dag"]["nodes"] + .as_array() + .unwrap() + .iter() + .filter(|n| { + n["operator"]["ASAP"]["SummaryAgg"]["family"]["Sketch"][0]["algorithm"] == "UnivMon" + }) + .count() + }; + let candidates = document["stage1_logical_asap_optimization"]["candidates"] + .as_array() + .unwrap(); + assert!( + candidates.iter().any(|c| { + c["display_name"] + .as_str() + .unwrap() + .starts_with("Q1 UnivMon · Q2 UnivMon (Count acc) · Q3 UnivMon (Count acc)") + && univmon_builds(c) == 1 + }), + "no candidate reads one UnivMon for all three" + ); + assert!(document["stage3_plan_selection"]["selected"].is_string()); +} + /// SQL queries over tables declared with `--table` are planned through the /// same stages, and the document records each query's language. #[test] diff --git a/crates/frontend-sql/src/frequency.rs b/crates/frontend-sql/src/frequency.rs new file mode 100644 index 000000000..fe6334178 --- /dev/null +++ b/crates/frontend-sql/src/frequency.rs @@ -0,0 +1,403 @@ +//! Recognition of the SQL frequency idioms at the query root. +//! +//! #509 Example 2 writes `L2(x)` and `Entropy(x)` as SQL idioms over grouped +//! unit counts. [`recognize_frequency_idioms`] names them as +//! `FrequencyL2` / `FrequencyEntropy` intents, so Pass 1 can offer exact and +//! summary candidates (UnivMon among them) for each. It preserves the +//! output schema, the predicates below the grouping, the query's accuracy +//! target, SQL's NULL result on empty input (decided by an exact population +//! count, never by an estimate) and the entropy unit: the IR's entropy is in +//! bits, the SQL idiom's `LN` gives nats. Integer products and nullable +//! keys are refused, because SQL integer overflow and NULL groups are +//! observable; such queries stay relational. +use asap_types::ir::non_asap::any_measure_filtered; +use asap_types::ir::{ + ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, ProjectItem, ScalarExpr, +}; +use asap_types::pre_asap::{ + AggIntent, ArithmeticOpKind, CompareOpKind, DataType, GroupKeys, JoinKind, Reduction, + ScalarValue, WindowFrameBound, WindowFrameOffset, WindowFuncKind, +}; +use asap_types::types::{AccuracyTarget, ErrorMetric}; +use std::rc::Rc; + +/// The recognized form of a query whose root is the L2 or natural-log entropy +/// idiom over grouped unit counts, or `None` when neither rule applies. +pub fn recognize_frequency_idioms(root: &Rc) -> Option> { + frequency_entropy_rewrite(root).or_else(|| frequency_l2_rewrite(root)) +} + +/// `expr` over `node`, followed down through projections only: crossing a +/// filter or a limit would change the population. +fn expand( + mut expr: ScalarExpr, + mut node: Rc, +) -> Option<(ScalarExpr, Rc)> { + while let Some(NonASAPOp::Project { cols, child, .. }) = node.non_asap() { + expr = substitute(&expr, cols)?; + node = Rc::clone(child); + } + Some((expr, node)) +} +fn substitute(expr: &ScalarExpr, cols: &[ProjectItem]) -> Option { + Some(match expr { + ScalarExpr::Column(id) => cols.get(*id)?.expr.clone(), + ScalarExpr::Literal(_) => expr.clone(), + ScalarExpr::Negative { expr, semantics } => ScalarExpr::Negative { + expr: Box::new(substitute(expr, cols)?), + semantics: *semantics, + }, + ScalarExpr::Cast { + expr, + to, + try_cast: false, + } if *to == DataType::Float64 => ScalarExpr::Cast { + expr: Box::new(substitute(expr, cols)?), + to: to.clone(), + try_cast: false, + }, + ScalarExpr::Arithmetic { + op, + left, + right, + semantics, + } => ScalarExpr::Arithmetic { + op: op.clone(), + left: Box::new(substitute(left, cols)?), + right: Box::new(substitute(right, cols)?), + semantics: *semantics, + }, + ScalarExpr::FunctionCall { name, args } => ScalarExpr::FunctionCall { + name: name.clone(), + args: args + .iter() + .map(|arg| substitute(arg, cols)) + .collect::>()?, + }, + _ => return None, + }) +} +fn uncast(expr: &ScalarExpr) -> &ScalarExpr { + match expr { + ScalarExpr::Cast { + expr, + to: DataType::Float64, + try_cast: false, + } => uncast(expr), + _ => expr, + } +} + +fn frequency_l2_rewrite(root: &Rc) -> Option> { + let NonASAPOp::Project { cols, child, .. } = root.non_asap()? else { + return None; + }; + let [item] = cols.as_slice() else { + return None; + }; + let (expr, outer) = expand(item.expr.clone(), Rc::clone(child))?; + let ScalarExpr::FunctionCall { name, args } = &expr else { + return None; + }; + if !name.eq_ignore_ascii_case("sqrt") { + return None; + } + let [arg] = args.as_slice() else { + return None; + }; + if !matches!(uncast(arg), ScalarExpr::Column(0)) { + return None; + } + let NonASAPOp::Aggregate { + reduction: Reduction::Reduce(keys), + measures, + filters, + having: None, + child, + .. + } = outer.non_asap()? + else { + return None; + }; + if keys.is_without() || !keys.keys().is_empty() || any_measure_filtered(filters) { + return None; + } + let [AggIntent::Sum { col: Some(col) }] = measures.as_slice() else { + return None; + }; + let (product, inner) = expand(ScalarExpr::Column(*col), Rc::clone(child))?; + let ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Mul, + left, + right, + semantics: ExprSemantics::Sql, + } = &product + else { + return None; + }; + // SQL Int64 multiplication can overflow. Admit only products already typed Float64. + if product.scalar_type(&inner.schema).ok()?.0 != DataType::Float64 { + return None; + } + if !matches!( + (uncast(left), uncast(right)), + (ScalarExpr::Column(1), ScalarExpr::Column(1)) + ) { + return None; + } + let (key, accuracy, input) = grouped_unit_count(&inner)?; + sql_frequency_result( + root, + input, + AggIntent::FrequencyL2 { + col: Some(key), + // The count's ε and δ, stated as the norm's relative error. + accuracy: accuracy.with_metric(ErrorMetric::Relative), + }, + "frequency_l2", + ScalarExpr::Column(0), + ) +} + +// Both frequency rules require a complete, unfiltered count per non-NULL identity. +fn grouped_unit_count( + node: &Rc, +) -> Option<(usize, AccuracyTarget, Rc)> { + let NonASAPOp::Aggregate { + reduction: Reduction::Reduce(keys), + measures, + filters, + having: None, + child, + .. + } = node.non_asap()? + else { + return None; + }; + let [key] = keys.keys() else { + return None; + }; + let [AggIntent::Count { accuracy }] = measures.as_slice() else { + return None; + }; + let field = child.schema.fields.get(*key)?; + if keys.is_without() + || any_measure_filtered(filters) + || field.nullable + || !matches!( + field.plain_dtype()?, + DataType::Bool | DataType::Int64 | DataType::Utf8 + ) + { + return None; + } + Some((*key, accuracy.clone(), Rc::clone(child))) +} + +fn probability_term(product: &ScalarExpr) -> Option<&ScalarExpr> { + let ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Mul, + left, + right, + semantics: ExprSemantics::Sql, + } = product + else { + return None; + }; + for (probability, logarithm) in [(left, right), (right, left)] { + let ScalarExpr::FunctionCall { name, args } = logarithm.as_ref() else { + continue; + }; + if name.eq_ignore_ascii_case("ln") && args.as_slice() == [probability.as_ref().clone()] { + return Some(probability); + } + } + None +} + +fn unit_count_term(expr: &ScalarExpr) -> bool { + match uncast(expr) { + ScalarExpr::Column(1) => true, + ScalarExpr::Arithmetic { op: ArithmeticOpKind::Mul, left, right, semantics: ExprSemantics::Sql } => { + [(left, right), (right, left)].into_iter().any(|(count, scale)| matches!(uncast(count), ScalarExpr::Column(1)) && matches!(scale.as_ref(), ScalarExpr::Literal(ScalarValue::Float64(value)) if *value == 1.0)) + }, + _ => false, + } +} + +fn frequency_entropy_rewrite(root: &Rc) -> Option> { + let NonASAPOp::Project { cols, child, .. } = root.non_asap()? else { + return None; + }; + let [item] = cols.as_slice() else { + return None; + }; + let (expr, outer) = expand(item.expr.clone(), Rc::clone(child))?; + let ScalarExpr::Negative { + expr, + semantics: ExprSemantics::Sql, + } = expr + else { + return None; + }; + if !matches!(uncast(&expr), ScalarExpr::Column(0)) { + return None; + } + let NonASAPOp::Aggregate { + reduction: Reduction::Reduce(keys), + measures, + filters, + having: None, + child, + .. + } = outer.non_asap()? + else { + return None; + }; + if keys.is_without() || !keys.keys().is_empty() || any_measure_filtered(filters) { + return None; + } + let [AggIntent::Sum { col: Some(col) }] = measures.as_slice() else { + return None; + }; + let (product, window) = expand(ScalarExpr::Column(*col), Rc::clone(child))?; + let probability = probability_term(&product)?; + let ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Div, + left, + right, + semantics: ExprSemantics::Sql, + } = probability + else { + return None; + }; + if !unit_count_term(left) + || !matches!(uncast(right), ScalarExpr::Column(2)) + || probability.scalar_type(&window.schema).ok()?.0 != DataType::Float64 + { + return None; + } + let NonASAPOp::SQLWindowFunc { + func: WindowFuncKind::Sum, + args, + partition_by, + order_by, + frame: Some(frame), + child, + .. + } = window.non_asap()? + else { + return None; + }; + if partition_by.is_without() + || !partition_by.keys().is_empty() + || !order_by.is_empty() + || args.as_slice() != [ScalarExpr::Column(1)] + { + return None; + } + if !matches!( + frame.start_bound, + WindowFrameBound::Preceding(WindowFrameOffset::Scalar(ScalarValue::Null)) + ) || !matches!( + frame.end_bound, + WindowFrameBound::Following(WindowFrameOffset::Scalar(ScalarValue::Null)) + ) { + return None; + } + let (key, accuracy, input) = grouped_unit_count(child)?; + let nats = ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Mul, + left: Box::new(ScalarExpr::Column(0)), + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64( + std::f64::consts::LN_2, + ))), + semantics: ExprSemantics::Sql, + }; + // -SUM(p*LN(p)) is negative zero for a single-identity population. + let nats = ScalarExpr::Negative { + expr: Box::new(ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Sub, + left: Box::new(ScalarExpr::Literal(ScalarValue::Float64(0.0))), + right: Box::new(nats), + semantics: ExprSemantics::Sql, + }), + semantics: ExprSemantics::Sql, + }; + sql_frequency_result( + root, + input, + AggIntent::FrequencyEntropy { + col: Some(key), + // The count's ε and δ, stated as an absolute error in bits. + accuracy: accuracy.with_metric(ErrorMetric::ABSOLUTE), + }, + "frequency_entropy", + nats, + ) +} + +// Both rules use an exact population guard: an approximate statistic may be +// zero even for nonempty input, and must not control SQL's NULL result. +fn sql_frequency_result( + root: &Rc, + input: Rc, + measure: AggIntent, + name: &str, + value: ScalarExpr, +) -> Option> { + let NonASAPOp::Project { qualifier, .. } = root.non_asap()? else { + return None; + }; + let aggregate = |measure, name: &str| { + OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::Reduce(GroupKeys::none()), + measures: vec![measure], + output_names: vec![name.into()], + filters: vec![], + having: None, + child: Rc::clone(&input), + })) + .ok() + }; + let statistic = aggregate(measure, name)?; + let count = aggregate( + AggIntent::Count { + accuracy: AccuracyTarget::exact(), + }, + "population_count", + )?; + let child = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Join { + kind: JoinKind::Cross, + pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), + left: statistic, + right: count, + })) + .ok()?; + let rewritten = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { + cols: vec![ProjectItem { + alias: Some(root.schema.fields.first()?.name.clone()), + expr: ScalarExpr::Case { + operand: None, + branches: vec![( + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(1)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(0))), + semantics: ExprSemantics::Sql, + }, + ScalarExpr::Cast { + expr: Box::new(ScalarExpr::Literal(ScalarValue::Null)), + to: DataType::Float64, + try_cast: false, + }, + )], + else_expr: Some(Box::new(value)), + }, + }], + qualifier: qualifier.clone(), + child, + })) + .ok()?; + (root.schema == rewritten.schema).then_some(rewritten) +} diff --git a/crates/frontend-sql/src/lib.rs b/crates/frontend-sql/src/lib.rs index e65a25f0a..d2c40dade 100644 --- a/crates/frontend-sql/src/lib.rs +++ b/crates/frontend-sql/src/lib.rs @@ -9,6 +9,7 @@ //! Depends on DataFusion only — never on the PromQL parser. pub mod error; +pub mod frequency; pub mod sql; use std::rc::Rc; @@ -55,7 +56,9 @@ pub async fn lower_sql_dialect( // Binding resolves names and derives every node's schema; result-type // checks (such as temporal subtraction, whose duration unit the IR cannot // represent) surface here as `ResolveDAGError::Schema`. - Ok(resolve_root(&unresolved)?) + let root = resolve_root(&unresolved)?; + // #509 Example 2: the L2 and entropy idioms become frequency intents. + Ok(frequency::recognize_frequency_idioms(&root).unwrap_or(root)) } /// Lower every SQL batch entry in `workload` to an operator DAG. diff --git a/crates/frontend-sql/tests/frequency_entropy.rs b/crates/frontend-sql/tests/frequency_entropy.rs new file mode 100644 index 000000000..4fa911e4d --- /dev/null +++ b/crates/frontend-sql/tests/frequency_entropy.rs @@ -0,0 +1,95 @@ +//! The SQL frontend names the natural-log entropy idiom as a +//! `FrequencyEntropy` intent (#509 Example 2's Q2). +use asap_frontend_common::resolve_root; +use asap_frontend_sql::{lower_sql, SqlCatalog, SqlLowerer}; +use asap_types::{ + ir::{NonASAPOp, OperatorNode}, + pre_asap::{AggIntent, DataType, Field, Schema}, + types::{ErrorMetric, RequestedAccuracy}, +}; + +fn catalog(nullable: bool) -> SqlCatalog { + SqlCatalog::new().with_table( + "flows", + Schema::new(vec![Field::plain("src_ip", DataType::Utf8, nullable)]), + ) +} + +fn has_entropy(node: &OperatorNode) -> bool { + if let Some(NonASAPOp::Aggregate { measures, .. }) = node.non_asap() { + if measures + .iter() + .any(|m| matches!(m, AggIntent::FrequencyEntropy { .. })) + { + return true; + } + } + node.children().iter().any(|child| has_entropy(child)) +} + +/// The natural-log idiom lowers to an entropy intent with the original +/// output schema; the relational form before recognition has none. +#[tokio::test] +async fn recognizes_sql_entropy_in_nats() { + let sql = "SELECT -SUM(p * LN(p)) AS entropy FROM (SELECT COUNT(*) * 1.0 / SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip) f"; + let rewritten = lower_sql(sql, &catalog(false), RequestedAccuracy::exact()) + .await + .unwrap(); + let original = resolve_root( + &SqlLowerer::new(&catalog(false)) + .lower(sql, &RequestedAccuracy::exact()) + .await + .unwrap(), + ) + .unwrap(); + assert!(has_entropy(&rewritten)); + assert_eq!(original.schema, rewritten.schema); + assert!(!has_entropy(&original)); +} + +/// Other units, normalizations, window frames or counted populations are +/// not entropy: scaled probabilities, `LOG2`, partitioned or ordered +/// windows, `HAVING` and a nullable key stay relational. +#[tokio::test] +async fn declines_non_equivalent_entropy_shapes() { + for (sql, nullable) in [ + ("SELECT -SUM(p*LN(p)) FROM (SELECT COUNT(*)*2.0/SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip) f", false), + ("SELECT -SUM(p*LOG2(p)) FROM (SELECT COUNT(*)*1.0/SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip) f", false), + ("SELECT -SUM(p*LN(p)) FROM (SELECT COUNT(*)*1.0/SUM(COUNT(*)) OVER (PARTITION BY src_ip) AS p FROM flows GROUP BY src_ip) f", false), + ("SELECT -SUM(p*LN(p)) FROM (SELECT COUNT(*)*1.0/SUM(COUNT(*)) OVER (ORDER BY src_ip) AS p FROM flows GROUP BY src_ip) f", false), + ("SELECT -SUM(p*LN(p)) FROM (SELECT COUNT(*)*1.0/SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip HAVING COUNT(*) > 1) f", false), + ("SELECT -SUM(p*LN(p)) FROM (SELECT COUNT(*)*1.0/SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip) f", true), + ] { + let root = lower_sql(sql, &catalog(nullable), RequestedAccuracy::exact()) + .await + .unwrap(); + assert!(!has_entropy(&root), "{sql}"); + } +} + +/// Entropy carries the query's accuracy target; its empty-population guard +/// is an exact count. +#[tokio::test] +async fn entropy_accuracy_and_population_guard_are_separate() { + let target = RequestedAccuracy::epsilon_delta(0.05, 0.01); + let node = lower_sql("SELECT -SUM(p*LN(p)) FROM (SELECT COUNT(*)*1.0/SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip) f", &catalog(false), target.clone()).await.unwrap(); + let NonASAPOp::Project { child, .. } = node.expect_non_asap() else { + panic!("project"); + }; + let NonASAPOp::Join { left, right, .. } = child.expect_non_asap() else { + panic!("guarded statistic"); + }; + let NonASAPOp::Aggregate { measures, .. } = left.expect_non_asap() else { + panic!("entropy"); + }; + assert!( + matches!(&measures[0], AggIntent::FrequencyEntropy { accuracy, .. } if *accuracy == target.target_in(ErrorMetric::ABSOLUTE)) + ); + let NonASAPOp::Aggregate { measures, .. } = right.expect_non_asap() else { + panic!("count"); + }; + assert!(matches!( + measures.as_slice(), + [AggIntent::Count { accuracy }] if accuracy.is_exact() + )); +} diff --git a/crates/frontend-sql/tests/frequency_l2.rs b/crates/frontend-sql/tests/frequency_l2.rs new file mode 100644 index 000000000..153dab4ba --- /dev/null +++ b/crates/frontend-sql/tests/frequency_l2.rs @@ -0,0 +1,225 @@ +//! The SQL frontend names the frequency L2 idiom as a `FrequencyL2` intent +//! (#509 Example 2's Q3, in its Float64 form). +use asap_frontend_common::resolve_root; +use asap_frontend_sql::{lower_sql, lower_sql_batch, SqlCatalog, SqlLowerer}; +use asap_types::{ + ir::{NonASAPOp, OperatorNode}, + pre_asap::{AggIntent, DataType, Field, Schema}, + types::{ErrorMetric, RequestedAccuracy}, +}; +use std::rc::Rc; + +fn catalog(nullable: bool) -> SqlCatalog { + SqlCatalog::new().with_table( + "flows", + Schema::new(vec![ + Field::plain("src_ip", DataType::Utf8, nullable), + Field::plain("keep", DataType::Bool, false), + ]), + ) +} + +fn has_l2(node: &OperatorNode) -> bool { + if let Some(NonASAPOp::Aggregate { measures, .. }) = node.non_asap() { + if measures + .iter() + .any(|m| matches!(m, AggIntent::FrequencyL2 { .. })) + { + return true; + } + } + node.children().iter().any(|child| has_l2(child)) +} + +/// The relational DAG the frontend lowers before frequency recognition. +async fn relational(sql: &str, catalog: &SqlCatalog) -> Rc { + resolve_root( + &SqlLowerer::new(catalog) + .lower(sql, &RequestedAccuracy::exact()) + .await + .unwrap(), + ) + .unwrap() +} + +const CAST_PRODUCT: &str = "SELECT SQRT(SUM(CAST(c AS DOUBLE) * CAST(c AS DOUBLE))) AS norm FROM (SELECT src_ip, COUNT(*) AS c FROM flows WHERE keep GROUP BY src_ip) f"; +const CAST_COUNT: &str = "SELECT SQRT(SUM(c * c)) AS norm FROM (SELECT src_ip, CAST(COUNT(*) AS DOUBLE) AS c FROM flows GROUP BY src_ip) f"; + +/// Float64 count products lower to an L2 intent that keeps the alias, the +/// filter, the type and the empty-input nullability of the original. +#[tokio::test] +async fn recognizes_float_frequency_l2() { + for sql in [CAST_PRODUCT, CAST_COUNT] { + let rewritten = lower_sql(sql, &catalog(false), RequestedAccuracy::exact()) + .await + .unwrap(); + let original = relational(sql, &catalog(false)).await; + assert!(has_l2(&rewritten), "{sql}"); + assert_eq!(original.schema, rewritten.schema); + assert!(!has_l2(&original)); + } +} + +/// Nearby forms with other overflow, NULL or frequency semantics stay +/// relational: the Int64 product (SQL integer overflow is observable), a +/// nullable key, `HAVING`, and a sum that is not a unit count. +#[tokio::test] +async fn declines_non_equivalent_frequency_shapes() { + for (sql, nullable) in [ + ("SELECT SQRT(SUM(c*c)) FROM (SELECT src_ip, COUNT(*) AS c FROM flows GROUP BY src_ip) f", false), + ("SELECT SQRT(SUM(CAST(c AS DOUBLE)*CAST(c AS DOUBLE))) FROM (SELECT src_ip, COUNT(*) AS c FROM flows GROUP BY src_ip) f", true), + ("SELECT SQRT(SUM(c*c)) FROM (SELECT src_ip, CAST(COUNT(*) AS DOUBLE) AS c FROM flows GROUP BY src_ip HAVING COUNT(*) > 1) f", false), + ("SELECT SQRT(SUM(c*c)) FROM (SELECT src_ip, CAST(SUM(CAST(keep AS BIGINT)) AS DOUBLE) AS c FROM flows GROUP BY src_ip) f", false), + ] { + let root = lower_sql(sql, &catalog(nullable), RequestedAccuracy::exact()) + .await + .unwrap(); + assert!(!has_l2(&root), "{sql}"); + } +} + +/// The L2 intent carries the query's accuracy target, not a default. +#[tokio::test] +async fn frequency_l2_preserves_accuracy_target() { + let target = RequestedAccuracy::epsilon_delta(0.01, 0.01); + let node = lower_sql(CAST_COUNT, &catalog(false), target.clone()) + .await + .unwrap(); + let NonASAPOp::Project { child, .. } = node.expect_non_asap() else { + panic!("project"); + }; + let NonASAPOp::Join { left, .. } = child.expect_non_asap() else { + panic!("guarded statistic"); + }; + let NonASAPOp::Aggregate { measures, .. } = left.expect_non_asap() else { + panic!("aggregate"); + }; + assert!( + matches!(&measures[0], AggIntent::FrequencyL2 { accuracy, .. } if *accuracy == target.target_in(ErrorMetric::Relative)) + ); +} + +/// An approximate L2 estimate never decides whether SQL returns NULL: the +/// empty-input guard is an exact count. +#[tokio::test] +async fn frequency_empty_input_guard_uses_an_exact_count() { + let target = RequestedAccuracy::epsilon_delta(0.01, 0.01); + let node = lower_sql(CAST_COUNT, &catalog(false), target) + .await + .unwrap(); + let NonASAPOp::Project { child, .. } = node.expect_non_asap() else { + panic!("project"); + }; + let NonASAPOp::Join { right, .. } = child.expect_non_asap() else { + panic!("exact population guard"); + }; + let NonASAPOp::Aggregate { measures, .. } = right.expect_non_asap() else { + panic!("count"); + }; + assert!(matches!( + measures.as_slice(), + [AggIntent::Count { accuracy }] if accuracy.is_exact() + )); +} + +/// The batch path recognizes the idiom as the single-query path does. +#[tokio::test] +async fn batch_lowering_recognizes_frequency_l2() { + use asap_types::workload::{ + AccuracyRequirement, BatchEntry, LatencyRequirement, Predictability, Query, QueryLanguage, + QueryRequirements, QueryWorkload, SqlDialect, TimeSelection, + }; + let workload = QueryWorkload { + language: QueryLanguage::SQL(SqlDialect::DataFusionSQL), + query_batch: Some(vec![BatchEntry { + query: Query(CAST_COUNT.into()), + requirements: QueryRequirements { + accuracy: AccuracyRequirement::Explicit(RequestedAccuracy::exact()), + response_latency: LatencyRequirement::Unspecified, + }, + predictability: Predictability::Unknown, + invocations: 1, + execute_at: None, + time_selection: TimeSelection::default(), + }]), + repeating_queries: None, + }; + let roots = lower_sql_batch(&workload, &catalog(false)).await; + assert!(has_l2(roots[0].as_ref().unwrap())); +} + +/// Stage 1 offers the exact option and summary candidates for the +/// recognized L2 intent. +#[tokio::test] +async fn stage1_offers_exact_and_summary_l2_candidates() { + use asap_aware_mapping::per_computation_candidates::{ + generate_per_computation_candidates, ComputationCandidate, + }; + use asap_types::ir::QueryRoot; + let target = RequestedAccuracy::epsilon_delta(0.01, 0.01); + let root = lower_sql(CAST_COUNT, &catalog(false), target) + .await + .unwrap(); + let inventory = + generate_per_computation_candidates(vec![(0, QueryRoot::Operator(root))]).unwrap(); + let l2 = inventory + .targets + .iter() + .find(|t| has_l2(&t.target)) + .expect("L2 target"); + let candidates = &l2.choices.candidates; + assert!( + candidates.iter().any(|c| matches!( + c, + ComputationCandidate::Exact | ComputationCandidate::ExactAccumulator { .. } + )), + "{candidates:?}" + ); + assert!( + candidates + .iter() + .any(|c| matches!(c, ComputationCandidate::Summary(_))), + "{candidates:?}" + ); +} + +/// #509 Example 2 over `flows(ts, src_ip)` with its one-minute window: Q2 +/// lowers to entropy and the Float64 Q3 to L2; the design's literal Int64 +/// Q3 stays relational. +#[tokio::test] +async fn example2_queries_lower_to_frequency_intents() { + let catalog = SqlCatalog::new().with_table( + "flows", + Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("src_ip", DataType::Utf8, false), + ], + 0, + vec![], + ), + ); + const WINDOW: &str = "WHERE ts >= now() - INTERVAL '1 minute'"; + let q2 = format!("SELECT -SUM(p * LN(p)) FROM (SELECT COUNT(*) * 1.0 / SUM(COUNT(*)) OVER () AS p FROM flows {WINDOW} GROUP BY src_ip)"); + let q3_float = format!("SELECT SQRT(SUM(CAST(c AS DOUBLE) * CAST(c AS DOUBLE))) FROM (SELECT src_ip, COUNT(*) AS c FROM flows {WINDOW} GROUP BY src_ip)"); + let q3_int = format!("SELECT SQRT(SUM(c * c)) FROM (SELECT src_ip, COUNT(*) AS c FROM flows {WINDOW} GROUP BY src_ip)"); + let lower = |sql: String| { + let catalog = &catalog; + async move { + lower_sql(&sql, catalog, RequestedAccuracy::exact()) + .await + .unwrap() + } + }; + let q2 = lower(q2).await; + assert!( + q2.children().iter().any(|join| join.children().iter().any( + |aggregate| matches!(aggregate.non_asap(), Some(NonASAPOp::Aggregate { measures, .. }) + if matches!(measures[..], [AggIntent::FrequencyEntropy { col: Some(1), .. }])) + )), + "{q2:#?}" + ); + let q3_float = lower(q3_float).await; + assert!(has_l2(&q3_float)); + assert!(!has_l2(&*lower(q3_int).await)); +} diff --git a/crates/integration-tests/tests/frequency_execution.rs b/crates/integration-tests/tests/frequency_execution.rs new file mode 100644 index 000000000..a1a89cb5c --- /dev/null +++ b/crates/integration-tests/tests/frequency_execution.rs @@ -0,0 +1,152 @@ +//! #509 Example 2's PromQL stand-in executes natively: the distinct count, +//! entropy and L2 norm of one series' sample values over 1 min, through the +//! exact candidate and through the one that builds a UnivMon, at query time. + +use std::collections::BTreeMap; +use std::rc::Rc; +use std::sync::Arc; + +use asap_integration_tests::fixtures::lower_promql; +use asap_integration_tests::planning::logical_candidates; +use asap_physical_operators::physical_planner::{ + compile_candidate, frontier_from_timing, promql_rows, InputContract, +}; +use asap_physical_operators::runtime::Scope; +use asap_physical_operators::values::{Batch, Value}; +use asap_types::ir::physical_export::{compile_physical_asap_dag, PhysicalASAPOperatorPayload}; +use asap_types::ir::{ + apply_materialization_timings, ASAPOp, MaterializationAssignment, NonASAPOp, OperatorNode, + TimingMemo, +}; +use asap_types::post_asap::{FieldDataType, SketchAlgorithm}; +use asap_types::types::RequestedAccuracy; + +mod physical_common; + +const EVALUATION_MS: i64 = 300_000; +/// `flows_src_ip` samples: source 1 twice, 2 three times, 3 once. +const SAMPLES: [f64; 6] = [1.0, 2.0, 1.0, 2.0, 3.0, 2.0]; + +/// The summary family a candidate builds: `None` for the exact plan. +fn built_family(candidate: &Rc) -> Option { + OperatorNode::reachable(candidate) + .iter() + .find_map(|node| match node.asap() { + Some(ASAPOp::SummaryAgg { family, .. }) => Some(family.clone()), + _ => None, + }) +} + +/// Execute `candidate` at query time over `SAMPLES`, all inside the window. +fn execute(candidate: &Rc) -> Vec { + let timed = apply_materialization_timings( + candidate, + &MaterializationAssignment::default(), + &mut TimingMemo::new(), + ) + .unwrap(); + let dag = compile_physical_asap_dag(&timed).unwrap(); + let raw: BTreeMap = dag + .nodes + .iter() + .filter(|node| { + matches!( + node.payload, + PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::TimeRange { .. }) + ) + }) + .map(|node| (node.id as u64, Arc::new(node.output_schema.clone()))) + .collect(); + assert_eq!(raw.len(), 1, "one range selector"); + let frontier = frontier_from_timing(&dag).unwrap(); + let compiled = compile_candidate( + &dag, + raw.iter() + .map(|(id, schema)| (*id, InputContract::bounded(schema.clone()))) + .collect(), + &[dag.roots[0] as u64], + &frontier, + ) + .unwrap(); + assert!(compiled.precompute.is_none()); + let labels = BTreeMap::from([("__name__".to_string(), "flows_src_ip".to_string())]); + let inputs = raw + .iter() + .map(|(id, schema)| { + let rows = SAMPLES + .iter() + .enumerate() + .map(|(i, value)| { + promql_rows::series_row(schema, &labels, 290_000 + i as i64, *value).unwrap() + }) + .collect(); + (*id, Batch::try_new(schema.clone(), rows).unwrap()) + }) + .collect(); + let result = physical_common::execute( + &compiled.query, + inputs, + Scope::Query { + evaluation_time_ms: EVALUATION_MS, + revision: 1, + }, + ); + result[0] + .iter() + .flat_map(|batch| batch.rows()) + .flat_map(|row| row.iter()) + .filter_map(|value| match value { + Value::Float64(value) => Some(*value), + Value::Int64(value) => Some(*value as f64), + _ => None, + }) + .collect() +} + +// The exact candidate and the UnivMon candidate of each statistic both +// execute and give the statistic of the window's sample values (entropy in +// bits). +#[test] +fn standin_exact_and_univmon_candidates_execute() { + let entropy = -[2.0_f64, 3.0, 1.0] + .iter() + .map(|count| count / 6.0 * (count / 6.0).log2()) + .sum::(); + // A UnivMon sized for a distinct count at ε = 0.01 holds a heap of + // millions of keys on each of its 19 layers, past the default 64 MiB. + for (query, expected, epsilon, delta) in [ + ("distinct_over_time(flows_src_ip[1m])", 3.0, 0.2, 0.1), + ("entropy_over_time(flows_src_ip[1m])", entropy, 0.01, 0.01), + ( + "l2_over_time(flows_src_ip[1m])", + 14.0_f64.sqrt(), + 0.01, + 0.01, + ), + ] { + let accuracy = RequestedAccuracy::epsilon_delta(epsilon, delta); + let root = lower_promql(query, accuracy.clone()).unwrap(); + let root = promql_rows::with_series_identity(&root).unwrap(); + let mut executed = Vec::new(); + for candidate in logical_candidates(root, accuracy) { + let family = built_family(&candidate); + let name = match &family { + None => "exact".to_string(), + Some(FieldDataType::Sketch(kind, _)) + if kind.algorithm() == &SketchAlgorithm::UnivMon => + { + "UnivMon".to_string() + } + _ => continue, + }; + let values = execute(&candidate); + assert!( + matches!(values.as_slice(), [value] if (value - expected).abs() < 1e-9), + "{query} {name}: {values:?}, expected {expected}" + ); + executed.push(name); + } + executed.sort(); + assert_eq!(executed, ["UnivMon", "exact"], "{query}"); + } +} diff --git a/crates/integration-tests/tests/physical_common/mod.rs b/crates/integration-tests/tests/physical_common/mod.rs index 7bd2b1157..37eb6edfe 100644 --- a/crates/integration-tests/tests/physical_common/mod.rs +++ b/crates/integration-tests/tests/physical_common/mod.rs @@ -73,3 +73,60 @@ fn compile_with( &root, )?) } + +/// Execute a relational DAG at query time against an in-memory connector +/// for its one scan, scan predicates included; checks that no memory stays +/// reserved afterwards. +#[allow(dead_code)] +pub fn execute_raw_rows( + root: &std::rc::Rc, + rows: Vec>, +) -> Vec> { + use asap_physical_operators::dag::{ + planner::bind_with_data_sources, + scan::{DataSources, MemorySource}, + }; + use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; + use asap_types::ir::NonASAPOp; + use std::sync::Arc; + let wire = compile_physical_asap_dag(root).unwrap(); + let (source, schema) = wire + .nodes + .iter() + .find_map(|node| match &node.payload { + PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { source, .. }) => { + Some((source.clone(), node.output_schema.clone())) + } + _ => None, + }) + .expect("one scan"); + let input = Arc::new(schema); + let batch = Batch::try_new(input.clone(), rows).unwrap(); + let mut sources = DataSources::default(); + sources + .register( + source, + Arc::new(MemorySource::new(input, vec![batch]).unwrap()), + ) + .unwrap(); + let root_id = wire.roots[0] as u64; + let plan = bind_with_data_sources(&wire, BTreeMap::new(), &[root_id], &sources).unwrap(); + let context = RunContext::new( + Scope::Query { + evaluation_time_ms: 0, + revision: 1, + }, + Limits::default(), + ) + .unwrap(); + let result = block_on(async { + let mut output = plan.execute(&[root_id], context.clone()).unwrap().remove(0); + let mut rows = vec![]; + while let Some(batch) = output.next().await { + rows.extend_from_slice(batch.unwrap().rows()); + } + rows + }); + assert_eq!(context.retained_bytes(), 0); + result +} diff --git a/crates/integration-tests/tests/planner_layering_example2.rs b/crates/integration-tests/tests/planner_layering_example2.rs index b0c87134d..c6440f1a9 100644 --- a/crates/integration-tests/tests/planner_layering_example2.rs +++ b/crates/integration-tests/tests/planner_layering_example2.rs @@ -6,10 +6,11 @@ //! need an unimplemented feature are `#[ignore]`d, naming it and the PR that //! delivers it. //! -//! The doc's workload is SQL. The SQL frontend lowers Q1 to a distinct count -//! but not Q2 and Q3 to `Entropy` and `L2` (the doc's own TODO, #598), so -//! the planning stages run on a PromQL stand-in with the same structure: -//! three statistics of one input over the same 1-min window, through the +//! The doc's workload is SQL; the SQL frontend lowers Q1 to a distinct +//! count and recognizes Q2 and Q3 as `Entropy` and `L2` (Q3 with a Float64 +//! product). Its time filter stays a scan predicate, so the planning +//! stages also run on a PromQL stand-in with the same structure: three +//! statistics of one input over the same 1-min window, through the //! frontend's `distinct_over_time`, `entropy_over_time` and `l2_over_time`. mod physical_common; @@ -39,7 +40,9 @@ const SQL_Q1: &str = const SQL_Q2: &str = "SELECT -SUM(p * LN(p)) FROM (\ SELECT COUNT(*) * 1.0 / SUM(COUNT(*)) OVER () AS p FROM flows \ WHERE ts >= now() - INTERVAL '1 minute' GROUP BY src_ip)"; -const SQL_Q3: &str = "SELECT SQRT(SUM(c * c)) FROM (\ +/// Q3 casts the counts to DOUBLE: the design's Int64 `SUM(c * c)` can +/// overflow, so the SQL frontend recognizes only the Float64 product. +const SQL_Q3: &str = "SELECT SQRT(SUM(CAST(c AS DOUBLE) * CAST(c AS DOUBLE))) FROM (\ SELECT src_ip, COUNT(*) AS c FROM flows \ WHERE ts >= now() - INTERVAL '1 minute' GROUP BY src_ip)"; @@ -271,7 +274,6 @@ async fn stage0_query_language_frontends_sql_q1_lowers_to_a_distinct_count() { /// The SQL frontend recognizes Q2 as `Entropy(src_ip)` and Q3 as `L2(src_ip)`. #[tokio::test] -#[ignore = "needs SQL frontend recognition of the Entropy and L2 idioms (#598)"] async fn stage0_query_language_frontends_sql_q2_q3_lower_to_entropy_and_l2() { let roots = lower_sql(&sql_workload()).await; assert!(frontend_operations(&roots[1]).contains(&"aggregate:frequency_entropy".to_string())); @@ -280,7 +282,6 @@ async fn stage0_query_language_frontends_sql_q2_q3_lower_to_entropy_and_l2() { /// Once the SQL forms are recognized, the SQL workload has the stand-in's Pass 1 options. #[tokio::test] -#[ignore = "needs SQL frontend recognition of the Entropy and L2 idioms (#598)"] async fn stage1_logical_asap_optimization_sql_has_the_standin_options() { let workload = sql_workload(); let sql = run_stages(&workload, lower_sql(&workload).await); @@ -382,7 +383,6 @@ fn stage1_logical_asap_optimization_keeps_every_independent_combination() { /// The summary-capability rule adds one UnivMon build feeding a distinct-count, an entropy and an L2 estimate. #[test] -#[ignore = "needs the Pass 2 summary-capability rule for frequency moments (#595, #596)"] fn stage1_logical_asap_optimization_summary_capability_adds_one_univmon_for_all_three() { let run = pipeline(); let shared = run.logical.iter().find(|c| { @@ -402,7 +402,6 @@ fn stage1_logical_asap_optimization_summary_capability_adds_one_univmon_for_all_ /// For each pair, a candidate shares one UnivMon between the two while the third keeps each of its own options. #[test] -#[ignore = "needs the Pass 2 summary-capability rule for frequency moments (#595, #596)"] fn stage1_logical_asap_optimization_summary_capability_adds_pairwise_shared_univmons() { let run = pipeline(); for (pair, third) in [([0, 1], 2), ([0, 2], 1), ([1, 2], 0)] { @@ -416,23 +415,42 @@ fn stage1_logical_asap_optimization_summary_capability_adds_pairwise_shared_univ } } -/// A UnivMon shared by several statistics is sized for the strictest consumer. +/// A UnivMon shared by several statistics is sized for every consumer: +/// each consumer is sized alone, and the shared build takes each +/// parameter's largest value among them. #[test] -#[ignore = "needs the Pass 2 summary-capability rule for frequency moments (#595, #596)"] -fn stage1_logical_asap_optimization_shared_univmon_is_sized_for_the_strictest_consumer() { +fn stage1_logical_asap_optimization_shared_univmon_takes_each_parameters_largest_own_size() { let run = pipeline(); - let strictest = |queries: &[usize]| { - *queries - .iter() - .min_by(|a, b| ACCURACY[**a].0.total_cmp(&ACCURACY[**b].0)) - .unwrap() + let own = |q| match own_univmon(&run, q) { + SketchParams::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, + .. + } => [heap_size, sketch_rows, sketch_cols, u32::from(layers)], + other => panic!("{other:?}"), }; let mut seen = 0; for queries in [vec![0, 1, 2], vec![0, 1], vec![0, 2], vec![1, 2]] { + let mut want = [0u32; 4]; + for &q in &queries { + for (w, p) in want.iter_mut().zip(own(q)) { + *w = (*w).max(p); + } + } + let want = SketchParams::UnivMon { + heap_size: want[0], + sketch_rows: want[1], + sketch_cols: want[2], + layers: want[3] as u8, + // Ignored by equality. + distinct_failure_probability: None, + }; for c in &run.logical { if let Some(params) = shared_univmon(c, &queries) { seen += 1; - assert_eq!(params, own_univmon(&run, strictest(&queries)), "{}", c.id); + assert_eq!(params, want, "{} {queries:?}", c.id); } } } @@ -475,7 +493,6 @@ fn stage2_physical_asap_optimization_keeps_every_logical_candidate() { /// Physical ASAP-aware optimization keeps the shared UnivMon as one build node read by all three queries. #[test] -#[ignore = "needs the Pass 2 summary-capability rule for frequency moments (#595, #596)"] fn stage2_physical_asap_optimization_keeps_the_shared_univmon_as_one_build() { let run = pipeline(); let mut seen = 0; @@ -741,9 +758,22 @@ fn stage3_plan_selection_accepts_univmon_l2() { } } +/// SQL Q3's L2 through UnivMon keeps its guarantee: the exact population +/// count joined beside it (a cross join, no key) and the `CASE` deciding the +/// empty-input NULL on that exact count keep the estimate's guarantee, so +/// plan selection accepts it. +#[tokio::test] +async fn stage3_plan_selection_accepts_sql_univmon_l2_beside_its_exact_population_count() { + let workload = sql_workload(); + let run = run_stages(&workload, lower_sql(&workload).await); + for id in univmon_only_candidates(&run, 2) { + assert!(run.cost(id).is_some(), "{id}: {:?}", run.invalid().get(id)); + } +} + /// One materialized UnivMon for all three costs no more than three separate UnivMons. #[test] -#[ignore = "needs one UnivMon for all three (#595, #596), a UnivMon accuracy model (#621) and materializing it (#604): not materialized, it is charged once per consuming query"] +#[ignore = "needs a UnivMon entropy accuracy model and materializing the shared UnivMon (#604): not materialized, it is charged once per consuming query"] fn stage3_plan_selection_shared_univmon_costs_no_more_than_three() { let run = pipeline(); let cost = |c: &Logical| run.cost(&run.physical_of(c).next().unwrap().id); diff --git a/crates/integration-tests/tests/precompute_raw_samples.rs b/crates/integration-tests/tests/precompute_raw_samples.rs index da1fdaea9..71fe0cb37 100644 --- a/crates/integration-tests/tests/precompute_raw_samples.rs +++ b/crates/integration-tests/tests/precompute_raw_samples.rs @@ -277,6 +277,14 @@ fn evaluations(state: &dyn AggregateCore, family: &FieldDataType) -> Vec { SketchAlgorithm::Hll | SketchAlgorithm::Kmv => { vec![state.estimate(&SketchStatistic::Cardinality).unwrap()] } + SketchAlgorithm::UnivMon => [ + SketchStatistic::Cardinality, + SketchStatistic::FrequencyL2, + SketchStatistic::FrequencyEntropy, + ] + .iter() + .map(|statistic| state.estimate(statistic).unwrap()) + .collect(), other => panic!("unexpected unkeyed sketch {other:?}"), } } @@ -314,8 +322,7 @@ fn check( let stored_only = matches!(family, FieldDataType::Sketch(kind, _) if kind.algorithm() == &asap_types::post_asap::SketchAlgorithm::Cms); if stored_only || asap_physical_operators::capability::validate_native_family(family).is_err() { - // Families without a native state (e.g. UnivMon), or with native - // stored state only (plain CMS), are outside precompute execution; + // Families without a native state, or with native stored state only (plain CMS), are outside precompute execution; // their compile must fail. assert!(precompute::compile(dag, &[source], &[root]).is_err()); return Err(format!("{family:?}")); @@ -331,7 +338,7 @@ fn check( &input.item, matches!( kind.algorithm(), - SketchAlgorithm::Hll | SketchAlgorithm::Kmv + SketchAlgorithm::Hll | SketchAlgorithm::Kmv | SketchAlgorithm::UnivMon ), ) { // Keyed heaps: every item's estimated weight is its exact @@ -391,6 +398,9 @@ fn check( fn raw_sample_summaries_compile_and_match_their_kernels() { let exact = RequestedAccuracy::exact(); let sketch = RequestedAccuracy::epsilon(0.02); + // A UnivMon sized for a distinct count at ε = 0.02 holds a heap of + // millions of keys on each of its 19 layers, past the default 64 MiB. + let distinct = RequestedAccuracy::epsilon_delta(0.2, 0.1); let queries = [ ("sum_over_time(m[5m])", &exact), ("count_over_time(m[5m])", &exact), @@ -404,7 +414,7 @@ fn raw_sample_summaries_compile_and_match_their_kernels() { ("quantile_over_time(0.9, m[5m])", &sketch), ("sum by (service) (quantile_over_time(0.9, m[5m]))", &sketch), ("quantile by (service) (0.9, m)", &sketch), - ("distinct_over_time(m[5m])", &sketch), + ("distinct_over_time(m[5m])", &distinct), ("count(m)", &sketch), ("topk(2, m)", &sketch), ( @@ -456,6 +466,7 @@ fn raw_sample_summaries_compile_and_match_their_kernels() { "DDSketch", "Hll", "Kmv", + "UnivMon", // The whole-expression heaps over raw samples. "CmsWithHeap", "CountSketchWithHeap", diff --git a/crates/integration-tests/tests/sql_cardinality.rs b/crates/integration-tests/tests/sql_cardinality.rs new file mode 100644 index 000000000..df6e76d24 --- /dev/null +++ b/crates/integration-tests/tests/sql_cardinality.rs @@ -0,0 +1,48 @@ +//! The design's SQL distinct query executes natively and exactly. +mod physical_common; +use asap_frontend_sql::{lower_sql, SqlCatalog}; +use asap_physical_operators::values::Value; +use asap_types::{ + pre_asap::{DataType, Field, Schema}, + types::RequestedAccuracy, +}; + +// COUNT(DISTINCT src_ip) skips NULL and keeps the WHERE, Utf8 identities and +// zero on no input. +#[tokio::test] +async fn sql_distinct_executes_through_raw_scan_and_native_binding() { + let catalog = SqlCatalog::new().with_table( + "flows", + Schema::new(vec![ + Field::plain("src_ip", DataType::Utf8, true), + Field::plain("keep", DataType::Bool, false), + ]), + ); + let root = lower_sql( + "SELECT COUNT(DISTINCT src_ip) AS sources FROM flows WHERE keep", + &catalog, + RequestedAccuracy::exact(), + ) + .await + .unwrap(); + for (rows, expected) in [ + (vec![], 0), + (vec![vec![Value::Null, Value::Bool(true)]], 0), + ( + vec![ + vec![Value::Utf8("a".into()), Value::Bool(true)], + vec![Value::Utf8("a".into()), Value::Bool(true)], + vec![Value::Utf8("b".into()), Value::Bool(true)], + vec![Value::Utf8("discard".into()), Value::Bool(false)], + vec![Value::Null, Value::Bool(true)], + ], + 2, + ), + ] { + let result = physical_common::execute_raw_rows(&root, rows); + assert!( + matches!(result.as_slice(), [row] if matches!(row.as_slice(), [Value::Int64(v)] if *v == expected)), + "{result:?}" + ); + } +} diff --git a/crates/integration-tests/tests/sql_frequency.rs b/crates/integration-tests/tests/sql_frequency.rs new file mode 100644 index 000000000..7cb40514b --- /dev/null +++ b/crates/integration-tests/tests/sql_frequency.rs @@ -0,0 +1,65 @@ +//! The SQL entropy and L2 idioms, recognized by the SQL frontend, execute +//! natively and exactly: entropy in nats (SQL's `LN`), L2 of the +//! per-source counts, and NULL on empty input, decided by an exact count. +mod physical_common; +use asap_frontend_sql::{lower_sql, SqlCatalog}; +use asap_physical_operators::values::Value; +use asap_types::{ + pre_asap::{DataType, Field, Schema}, + types::RequestedAccuracy, +}; + +const ENTROPY: &str = "SELECT -SUM(p * LN(p)) FROM (\ + SELECT COUNT(*) * 1.0 / SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip)"; +const L2: &str = "SELECT SQRT(SUM(CAST(c AS DOUBLE) * CAST(c AS DOUBLE))) FROM (\ + SELECT src_ip, COUNT(*) AS c FROM flows GROUP BY src_ip)"; + +fn catalog() -> SqlCatalog { + SqlCatalog::new().with_table( + "flows", + Schema::new(vec![Field::plain("src_ip", DataType::Utf8, false)]), + ) +} + +fn rows(sources: &[&str]) -> Vec> { + sources + .iter() + .map(|s| vec![Value::Utf8((*s).into())]) + .collect() +} + +async fn run(sql: &str, sources: &[&str]) -> Option { + let root = lower_sql(sql, &catalog(), RequestedAccuracy::exact()) + .await + .unwrap(); + let result = physical_common::execute_raw_rows(&root, rows(sources)); + match result.as_slice() { + [row] => match row.as_slice() { + [Value::Float64(v)] => Some(*v), + [Value::Null] => None, + other => panic!("{other:?}"), + }, + other => panic!("{other:?}"), + } +} + +/// Counts 3 and 1 over four rows: entropy −0.75 ln 0.75 − 0.25 ln 0.25 +/// nats; one source gives −0.0; no rows give NULL. +#[tokio::test] +async fn recognized_sql_entropy_executes_in_nats() { + let expected = -0.75f64 * 0.75f64.ln() - 0.25 * 0.25f64.ln(); + let got = run(ENTROPY, &["a", "a", "a", "b"]).await.unwrap(); + assert!((got - expected).abs() < 1e-12, "{got} vs {expected}"); + let single = run(ENTROPY, &["a", "a"]).await.unwrap(); + assert_eq!(single, 0.0); + assert!(single.is_sign_negative(), "SQL's -SUM(1 * LN 1) is -0.0"); + assert_eq!(run(ENTROPY, &[]).await, None); +} + +/// Counts 2 and 1: L2 √5; no rows give NULL. +#[tokio::test] +async fn recognized_sql_l2_executes_exactly() { + let got = run(L2, &["a", "a", "b"]).await.unwrap(); + assert!((got - 5f64.sqrt()).abs() < 1e-12, "{got}"); + assert_eq!(run(L2, &[]).await, None); +} diff --git a/crates/types/src/post_asap/family_capability.rs b/crates/types/src/post_asap/family_capability.rs index a6d4c5ac8..25f087ee2 100644 --- a/crates/types/src/post_asap/family_capability.rs +++ b/crates/types/src/post_asap/family_capability.rs @@ -3,9 +3,10 @@ //! //! The IR stays permissive: a `SummaryMerge` over any state family is //! structurally valid. The planner rules and the executor consult this -//! table to decide whether a composition is sound. Today the table holds -//! the mergeability facts; the summary-capability facts (which statistics -//! a family answers) are added as further fields of [`FamilyCapability`]. +//! table to decide whether a composition is sound. It holds the +//! mergeability facts and the summary-capability facts: which readouts +//! ([`Readout`]) one state of the family answers, so one state can serve +//! every consumer whose statistic it answers. use super::sketch::{ExactKind, GroupingStrategy, HydraKind, SketchAlgorithm, SketchStatistic}; use crate::pre_asap::schema::FieldDataType; @@ -20,6 +21,44 @@ pub struct FamilyCapability { /// (`SummarySubtract`, reserved): the subtrahend's selection must lie /// inside the minuend's, and the result is the state of the difference. pub subtractable: bool, + /// The readouts one state of the family answers, whatever its + /// parameters; none for a family read only as a finalized value (exact + /// accumulators) or one not listed. + pub readouts: &'static [Readout], +} + +/// One kind of readout (`SummaryEstimate` statistic) of a summary state. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Readout { + /// The total of the updates: a bare `PointCount` (no item value). + Total, + /// One item's frequency: a `PointCount` with an item value. + ItemFrequency, + /// The value at any quantile rank. + Quantile, + /// The number of distinct items. + Distinct, + /// The L2 norm of the item frequencies. + FrequencyL2, + /// The entropy of the item frequencies. + FrequencyEntropy, + /// The heaviest items, any `k` up to the state's heap. + TopK, +} + +impl Readout { + /// The kind of readout `statistic` is. + pub fn of(statistic: &SketchStatistic) -> Self { + match statistic { + SketchStatistic::PointCount { value: None, .. } => Self::Total, + SketchStatistic::PointCount { value: Some(_), .. } => Self::ItemFrequency, + SketchStatistic::Quantile { .. } => Self::Quantile, + SketchStatistic::Cardinality => Self::Distinct, + SketchStatistic::FrequencyL2 => Self::FrequencyL2, + SketchStatistic::FrequencyEntropy => Self::FrequencyEntropy, + SketchStatistic::TopK { .. } => Self::TopK, + } + } } /// How a family's states merge. @@ -66,9 +105,21 @@ pub enum MergedError { } impl FamilyCapability { + /// Whether one state of the family answers `statistic`. + pub fn answers(&self, statistic: &SketchStatistic) -> bool { + self.readouts.contains(&Readout::of(statistic)) + } + /// The capability of `family`. Families not listed have no capability /// (they fail closed). pub fn of(family: &FieldDataType) -> Self { + Self { + readouts: readouts(family), + ..Self::merge_facts(family) + } + } + + fn merge_facts(family: &FieldDataType) -> Self { use ExactKind as E; use SelectionRelation::{Disjoint, DisjointInTime, OverlapAllowed}; use SketchAlgorithm as S; @@ -150,6 +201,7 @@ impl FamilyCapability { Self { merge, subtractable, + readouts: &[], } } @@ -163,3 +215,168 @@ impl FamilyCapability { }) } } + +/// The readouts one state of `family` answers. +fn readouts(family: &FieldDataType) -> &'static [Readout] { + use Readout::*; + use SketchAlgorithm as S; + match family { + FieldDataType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { kind, .. }) => { + match kind { + // Per subpopulation, as the inner family answers. + HydraKind::HydraCms | HydraKind::HydraCountSketch => &[ItemFrequency], + HydraKind::HydraKll => &[Quantile], + } + } + FieldDataType::Sketch(kind, GroupingStrategy::PerSubpopulationInstance) => { + match kind.algorithm() { + // One frequency vector serves its total and every moment. + S::UnivMon => &[Total, Distinct, FrequencyL2, FrequencyEntropy], + // A quantile sketch answers every rank. + S::Kll => &[Quantile], + S::DDSketch => &[Quantile, Total], + S::Cms => &[ItemFrequency, Total], + // Each row's squared counters estimate F2 (AMS). + S::CountSketch => &[ItemFrequency, FrequencyL2], + S::CmsWithHeap | S::CountSketchWithHeap => &[TopK, ItemFrequency], + S::Hll | S::Kmv | S::Theta => &[Distinct], + } + } + FieldDataType::ExactAggregate(..) + | FieldDataType::Plain(_) + | FieldDataType::Sample(..) + | FieldDataType::Wavelet(..) + | FieldDataType::StatModel(..) => &[], + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::post_asap::{ExactParams, SketchKind, SketchParams}; + use crate::pre_asap::expr_ir::ColumnRef; + + fn sketch(algorithm: SketchAlgorithm, params: SketchParams) -> FieldDataType { + FieldDataType::Sketch( + SketchKind::new(algorithm, params), + GroupingStrategy::default(), + ) + } + + fn total() -> SketchStatistic { + SketchStatistic::PointCount { + key: ColumnRef::SampleValue, + value: None, + } + } + + fn item() -> SketchStatistic { + SketchStatistic::PointCount { + key: ColumnRef::Named("src_ip".into()), + value: Some("10.0.0.1".into()), + } + } + + /// One UnivMon answers its total, the distinct count, L2 and entropy, + /// so it can serve all four; it answers no quantile or top-k. + #[test] + fn univmon_answers_its_total_and_every_frequency_moment() { + let univmon = FamilyCapability::of(&sketch( + SketchAlgorithm::UnivMon, + SketchParams::UnivMon { + heap_size: 256, + sketch_rows: 5, + sketch_cols: 1024, + layers: 16, + distinct_failure_probability: None, + }, + )); + for statistic in [ + total(), + SketchStatistic::Cardinality, + SketchStatistic::FrequencyL2, + SketchStatistic::FrequencyEntropy, + ] { + assert!(univmon.answers(&statistic), "{statistic:?}"); + } + for statistic in [ + SketchStatistic::Quantile { q: 0.5 }, + SketchStatistic::TopK { k: 3 }, + ] { + assert!(!univmon.answers(&statistic), "{statistic:?}"); + } + } + + /// A quantile sketch answers every rank; heap sketches answer top-k and + /// item frequencies; CountSketch answers item frequencies and L2. + #[test] + fn summary_capability_facts() { + let kll = sketch( + SketchAlgorithm::Kll, + SketchParams::Kll { + k: 200, + failure_probability: None, + }, + ); + let count_sketch = sketch( + SketchAlgorithm::CountSketch, + SketchParams::CountSketch { + width: 64, + depth: 5, + }, + ); + let cms_heap = sketch( + SketchAlgorithm::CmsWithHeap, + SketchParams::CmsWithHeap { + width: 64, + depth: 5, + heap_size: 10, + }, + ); + let hll = sketch(SketchAlgorithm::Hll, SketchParams::Hll { precision: 12 }); + for (family, yes, no) in [ + ( + &kll, + vec![ + SketchStatistic::Quantile { q: 0.5 }, + SketchStatistic::Quantile { q: 0.99 }, + ], + vec![SketchStatistic::Cardinality, total()], + ), + ( + &count_sketch, + vec![item(), SketchStatistic::FrequencyL2], + vec![ + SketchStatistic::FrequencyEntropy, + SketchStatistic::TopK { k: 3 }, + ], + ), + ( + &cms_heap, + vec![SketchStatistic::TopK { k: 10 }, item()], + vec![SketchStatistic::FrequencyL2], + ), + ( + &hll, + vec![SketchStatistic::Cardinality], + vec![SketchStatistic::FrequencyL2, item()], + ), + ] { + let capability = FamilyCapability::of(family); + for statistic in yes { + assert!(capability.answers(&statistic), "{family:?} {statistic:?}"); + } + for statistic in no { + assert!(!capability.answers(&statistic), "{family:?} {statistic:?}"); + } + } + } + + /// An exact accumulator is read as its finalized value, not a readout. + #[test] + fn exact_accumulators_answer_no_readout() { + let sum = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); + assert!(FamilyCapability::of(&sum).readouts.is_empty()); + assert!(!FamilyCapability::of(&sum).answers(&total())); + } +} diff --git a/crates/types/src/post_asap/mod.rs b/crates/types/src/post_asap/mod.rs index 93dba180e..6da106382 100644 --- a/crates/types/src/post_asap/mod.rs +++ b/crates/types/src/post_asap/mod.rs @@ -37,7 +37,9 @@ pub use execution_data_state::{ lift_plain, DataPrimitive, ExactOperationSchemaError, ExecutionDataState, ExecutionDataStateError, ExecutionTiming, Materialization, Medium, }; -pub use family_capability::{FamilyCapability, MergeCapability, MergedError, SelectionRelation}; +pub use family_capability::{ + FamilyCapability, MergeCapability, MergedError, Readout, SelectionRelation, +}; pub use guarantee::{ AccuracyError, AccuracyGuarantee, BoundExpr, CompositionOperator, ErrorMetric, GuaranteeSource, ProbabilityExpr, diff --git a/crates/types/src/types.rs b/crates/types/src/types.rs index 4de62c478..09fa74f89 100644 --- a/crates/types/src/types.rs +++ b/crates/types/src/types.rs @@ -394,6 +394,16 @@ fn node_metric(node: &OperatorNode) -> Result { node_metric(child) } + // A `CASE` choosing between columns and literals, e.g. SQL's + // empty-input NULL guard over a statistic cross-joined with an exact + // count: the metric of the columns its value branches read. + NonASAPOp::Project { .. } => (0..node.schema.fields.len()) + .map(|column| column_metric(node, column)) + .collect::, _>>() + .map(|metrics| match metrics.split_first() { + Some((first, rest)) if rest.iter().all(|metric| metric == first) => *first, + _ => ErrorMetric::ABSOLUTE, + }), NonASAPOp::Aggregate { measures, .. } => { let mut metrics = Vec::new(); for intent in measures { @@ -422,6 +432,83 @@ fn node_metric(node: &OperatorNode) -> Result } } +/// The metric of `node`'s output `column`, following columns through +/// projections, `CASE` value branches, joins and operators that only +/// select, order or rename rows, down to an aggregate's measure. A group +/// key or any other expression is absolute. +fn column_metric(node: &OperatorNode, column: usize) -> Result { + use crate::ir::operator_properties::JoinKind; + let Operator::NonASAP(op) = &node.operator else { + return Ok(ErrorMetric::ABSOLUTE); + }; + match op { + NonASAPOp::Filter { child, .. } + | NonASAPOp::Sort { child, .. } + | NonASAPOp::Limit { child, .. } + | NonASAPOp::Dedup { child, .. } => column_metric(child, column), + NonASAPOp::Project { cols, child, .. } => match cols.get(column).map(|item| &item.expr) { + Some(ScalarExpr::Column(id)) => column_metric(child, *id), + Some(ScalarExpr::Case { + branches, + else_expr, + .. + }) => { + let mut metrics = Vec::new(); + for value in branches + .iter() + .map(|(_, then)| then) + .chain(else_expr.as_deref()) + { + match value { + // A literal, possibly cast (SQL's typed NULL). + ScalarExpr::Literal(_) => {} + ScalarExpr::Cast { expr, .. } + if matches!(**expr, ScalarExpr::Literal(_)) => {} + ScalarExpr::Column(id) => metrics.push(column_metric(child, *id)?), + _ => return Ok(ErrorMetric::ABSOLUTE), + } + } + Ok(match metrics.split_first() { + Some((first, rest)) if rest.iter().all(|metric| metric == first) => *first, + _ => ErrorMetric::ABSOLUTE, + }) + } + _ => Ok(ErrorMetric::ABSOLUTE), + }, + NonASAPOp::Join { + kind: + JoinKind::Inner | JoinKind::Left | JoinKind::Right | JoinKind::Full | JoinKind::Cross, + left, + right, + .. + } => { + let width = left.schema.fields.len(); + if column < width { + column_metric(left, column) + } else { + column_metric(right, column - width) + } + } + NonASAPOp::Aggregate { measures, .. } => { + // Group keys first, then one column per measure. + let keys = node.schema.fields.len().saturating_sub(measures.len()); + match column.checked_sub(keys).and_then(|m| measures.get(m)) { + Some(AggIntent::Quantile { .. }) => { + Err(AccuracyRequestError::MissingQuantileMetric { intent: "quantile" }) + } + Some(AggIntent::HistogramQuantile { .. }) => { + Err(AccuracyRequestError::MissingQuantileMetric { + intent: "histogram_quantile", + }) + } + Some(intent) => Ok(intent_metric(intent).unwrap_or(ErrorMetric::ABSOLUTE)), + None => Ok(ErrorMetric::ABSOLUTE), + } + } + _ => Ok(ErrorMetric::ABSOLUTE), + } +} + #[cfg(test)] mod tests { use super::*; @@ -572,6 +659,80 @@ mod tests { ); } + /// SQL's empty-input NULL guard (a statistic cross-joined with an exact + /// row count, and `CASE WHEN count = 0 THEN NULL ELSE statistic END`) + /// keeps the statistic's metric: the root's value is the statistic's + /// column. A `CASE` whose value branches read columns of different + /// metrics, or any other expression, is absolute. + #[test] + fn root_metric_looks_through_a_null_guard_to_its_statistic() { + use crate::ir::operator_properties::JoinKind; + use crate::ir::{Predicate, ProjectItem}; + use crate::pre_asap::{CompareOpKind, ScalarValue}; + let exact = AccuracyTarget::exact(); + let QueryRoot::Operator(l2) = aggregate(vec![AggIntent::FrequencyL2 { + col: None, + accuracy: exact.clone(), + }]) else { + unreachable!() + }; + let QueryRoot::Operator(count) = aggregate(vec![AggIntent::Count { accuracy: exact }]) + else { + unreachable!() + }; + let joined = node(NonASAPOp::Join { + kind: JoinKind::Cross, + pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), + left: l2, + right: count, + }); + let guard = |otherwise: ScalarExpr| { + QueryRoot::Operator(node(NonASAPOp::Project { + cols: vec![ProjectItem { + alias: Some("v".into()), + expr: ScalarExpr::Case { + operand: None, + branches: vec![( + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(1)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(0))), + semantics: crate::ir::ExprSemantics::Sql, + }, + ScalarExpr::Cast { + expr: Box::new(ScalarExpr::Literal(ScalarValue::Null)), + to: DataType::Float64, + try_cast: false, + }, + )], + else_expr: Some(Box::new(otherwise)), + }, + }], + qualifier: None, + child: joined.clone(), + })) + }; + let metric = |root: QueryRoot| { + RequestedAccuracy::epsilon(0.01) + .for_root(&root) + .unwrap() + .metric + }; + assert_eq!(metric(guard(ScalarExpr::Column(0))), ErrorMetric::Relative); + // The count's column, or arithmetic over the statistic. + assert_eq!( + metric(guard(ScalarExpr::Column(1))), + ErrorMetric::ABSOLUTE_STREAM_L1 + ); + assert_eq!( + metric(guard(ScalarExpr::Negative { + expr: Box::new(ScalarExpr::Column(0)), + semantics: crate::ir::ExprSemantics::Sql, + })), + ErrorMetric::ABSOLUTE + ); + } + /// A quantile root's request must name its metric; a named metric or an /// exact request needs no default. #[test] diff --git a/docs/user_guide_docs/stage-viewer.md b/docs/user_guide_docs/stage-viewer.md index 14024c8f0..3043c00b2 100644 --- a/docs/user_guide_docs/stage-viewer.md +++ b/docs/user_guide_docs/stage-viewer.md @@ -69,7 +69,7 @@ plan wins. | Tab | Workload | What to look for | | --- | --- | --- | | 1 · dashboard panels | Two PromQL panels over 1M counter series every 10 s: an exact per-job rate, and the approximate top-10 series per job | The exact query stays exact; the CMS+heap and CountSketch+heap top-k plans are rejected with their reasons, so the all-exact plan wins; the plans that share the two queries' scan and range selector tie with it, and the independent plan is kept on enumeration order | -| 2 · SQL flow statistics | Distinct sources, entropy and L2 of per-source counts over the last minute of flows | All 64 plans compose and are priced; none is pruned. The SQL frontend does not yet recognize entropy and L2, so Q2 and Q3 are per-source counts with exact, Count-Min, CountSketch or UnivMon options. 56 plans are invalid: HLL for Q1 because its bound has "failure probability None", and UnivMon for Q1 or Count-Min or CountSketch for Q2 and Q3 because "the output has no guarantee, and the deployment has no evidence that it meets target". 8 are valid: Q1 exact or KMV with Q2 and Q3 each exact or UnivMon. Theta is not offered: `asap_sketchlib` has no Theta sketch. P1 (all exact) is selected at 8.64 per second, tied with its shared-input plans; KMV on Q1 costs more | +| 2 · SQL flow statistics | Distinct sources, entropy and L2 of per-source counts over the last minute of flows | All 82 plans compose and are priced; none is pruned. The SQL frontend recognizes Q2 as entropy and Q3 (its counts cast to DOUBLE) as L2, each with an exact or UnivMon option and an exact population count for the empty-input NULL. 16 plans come from Pass 1. Pass 2 adds the plans that share identical nodes and, by the summary-capability rule, the plans in which one UnivMon serves two of the queries or all three, sized with each parameter's largest value among them. 70 plans are invalid: HLL or UnivMon for Q1 (50) because its bound has "failure probability None" (both need the source's distinct bound, which this example does not declare), and UnivMon for Q2 (20) because entropy has no certified bound. 12 are valid: Q1 exact or KMV, Q2 exact, and Q3 exact or UnivMon, each alone or sharing identical nodes. Q3's UnivMon keeps its L2 guarantee through the exact population count joined beside it and the `CASE` that decides the empty-input NULL on that count. Theta is not offered: `asap_sketchlib` has no Theta sketch. P1 (all exact) is selected at 6.08 per second; KMV on Q1 costs the same to three decimals, UnivMon on Q3 costs 8.08, and the shared plans cost 7.44 or more, because a shared node that is not materialized is charged once per query that reads it | | 3a · historical p99 batch | Five p99 reports over 1–5 years, run once | All 972 plans are priced: 243 Pass 1 plans, each with 3 shared-input plans. Each query builds its own sketch at query time. P485 is selected: KLL for all five queries. DDSketch in place of KLL ties, and the tie goes to enumeration order. The all-exact plan costs about 16% more | | 3b · live p99 panel | p99 over the last 5 min every minute, 1M series | P2, KLL, is selected at 2.100 per second, tied with DDSketch and chosen on enumeration order; exact over the window's raw samples costs 2.400. Memory a query holds only while it runs is not charged. Everything runs at query time and nothing is materialized. KLL panes kept from ingestion time and the 200 ms latency check come with the window-summary PRs | | 4a · monthly p99 reports | Example 3a repeated monthly | The same plan, with its cost amortized over a month |