From 15d1b0f9f2a4c9c04f427a4a49cd1464fd7a4a29 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:47:14 +0000 Subject: [PATCH 01/15] fix(logical-asap-optimization): weight Top-K heap sketches by the ranked value Pass 1's Top-K sketch update weighted every heap by the sample value. Over SQL rows that column is the one named `value`, or else the only numeric column: an Int64 key made the heap candidates fail to compose, and a key named `value` would have weighted the heap by the key. The update now reads the column the top-k ranks (the aggregate's measure over grouped rows, the sample value over per-series rows), the same column exact top-k sorts by. `ranked_column` moves from Stage 2 into the shared realization helpers so both stages use it. --- .../src/per_computation_candidates.rs | 30 +++-- crates/asap-aware-mapping/src/realization.rs | 62 +++++++++- .../src/stage2_physical_asap_optimization.rs | 54 +-------- .../tests/per_computation_candidates_sql.rs | 111 +++++++++++++++++- 4 files changed, 194 insertions(+), 63 deletions(-) diff --git a/crates/asap-aware-mapping/src/per_computation_candidates.rs b/crates/asap-aware-mapping/src/per_computation_candidates.rs index 39546ab15..7e223f9ed 100644 --- a/crates/asap-aware-mapping/src/per_computation_candidates.rs +++ b/crates/asap-aware-mapping/src/per_computation_candidates.rs @@ -1816,6 +1816,14 @@ fn measure_input( AggIntent::Count { .. } if input.result_kind == OperatorResultKind::Relation => { SummaryInputExpr::Column(counted_column(schema, filter)?) } + // A top-k ranks its child's aggregate output, not a column of its own. + AggIntent::TopK { .. } => SummaryInputExpr::Column( + crate::realization::ranked_value(input) + .map_err(|_| { + PerComputationCandidateError::Unsupported("Top-K has no ranked column") + })? + .1, + ), _ => crate::realization::summarised_input(intent, schema).map_err(|_| { PerComputationCandidateError::Unsupported("input column outside child schema") })?, @@ -1906,14 +1914,17 @@ fn summary_update( asap_types::pre_asap::schema::PROMQL_SERIES_IDENTITY.into(), )) } else if child.closed { - // A closed PromQL child (an aggregate's output, say) lists - // every label: the item is its labels outside the partition, - // as the legacy current-series and counter rules rank them. - // SQL rows carry no sample value to rank by. - let value = resolve_column_ref(&ColumnRef::SampleValue, child).map_err(|_| { - PerComputationCandidateError::Unsupported( - "Top-K item identity for closed schemas without a sample value", - ) + // A closed child (an aggregate's output, say) lists every + // column: the item is its columns outside the partition and + // the ranked value, as the legacy current-series and counter + // rules rank them. + let SummaryInputExpr::Column(ranked) = &weight else { + return Err(PerComputationCandidateError::Unsupported( + "Top-K ranked value is not a column", + )); + }; + let value = resolve_column_ref(ranked, child).map_err(|_| { + PerComputationCandidateError::Unsupported("Top-K ranked value does not resolve") })?; let items: Vec<_> = child .fields @@ -1940,7 +1951,8 @@ fn summary_update( }; SummaryUpdate { item: Some(item), - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), + // The ranked value ([`measure_input`]). + weight, // Not proven non-negative; selection decides whether CMS is admissible. weight_domain: WeightDomain::UnknownOrSigned, } diff --git a/crates/asap-aware-mapping/src/realization.rs b/crates/asap-aware-mapping/src/realization.rs index bb7df9d15..44661ba75 100644 --- a/crates/asap-aware-mapping/src/realization.rs +++ b/crates/asap-aware-mapping/src/realization.rs @@ -3,11 +3,16 @@ //! the intent's accuracy target and `(ε, δ)` budget, default sketch sizing, //! and the column/tuple a summary consumes. -use asap_types::ir::SchemaDerivationError; +use asap_types::ir::{ + ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr, SchemaDerivationError, +}; use asap_types::post_asap::{AccuracyError, ExactOperationSchemaError, ExecutionDataStateError}; use asap_types::post_asap::{Field, Schema, SketchAlgorithm, SketchParams, SummaryInputExpr}; use asap_types::pre_asap::agg_intent::AggIntent; +use asap_types::pre_asap::column_resolution::{resolve_column_ref, ResolveError}; use asap_types::pre_asap::expr_ir::ColumnRef; +use asap_types::pre_asap::schema::ColumnId; +use asap_types::pre_asap::Reduction; use asap_types::types::AccuracyTarget; use thiserror::Error; @@ -124,6 +129,61 @@ pub(crate) fn summarised_column(intent: &AggIntent, child_schema: &Schema) -> Co } } +/// The column a top-k ranks `child` by. Over a one-measure grouped +/// aggregate (the SQL heavy-hitter shape and PromQL `topk(k, sum by ..)`) it +/// is that measure, the last column after the group keys, whichever Stage 1 +/// candidate realizes it: the aggregate itself, or a `SummaryAgg` read by +/// `FinalizeExactAccumulator` or `SummaryEstimate`, under the `Project` +/// restoring its names (a renaming) and a value `HAVING`'s `Filter`. Over +/// per-series rows (PromQL `topk by (..) (k, rows)`) it is the sample value. +pub(crate) fn ranked_column(child: &OperatorNode) -> Result { + let mut node = child; + let reduction = loop { + match &node.operator { + // Only a renaming keeps the measure last. + Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) + if cols.len() == child.schema.fields.len() + && cols + .iter() + .enumerate() + .all(|(i, item)| item.expr == ScalarExpr::Column(i)) => + { + node = child + } + Operator::NonASAP(NonASAPOp::Filter { child, .. }) + | Operator::ASAP( + ASAPOp::FinalizeExactAccumulator { child } + | ASAPOp::SummaryEstimate { + summary_input: child, + .. + }, + ) => node = child, + Operator::ASAP(ASAPOp::SummaryAgg { reduction, .. }) => break Some(reduction), + Operator::NonASAP(NonASAPOp::Aggregate { + reduction, + measures, + .. + }) if measures.len() == 1 => break Some(reduction), + _ => break None, + } + }; + match reduction { + Some(Reduction::Reduce(_)) => Ok(child.schema.fields.len() - 1), + _ => resolve_column_ref(&ColumnRef::SampleValue, &child.schema), + } +} + +/// [`ranked_column`] as a reference: the sample value when it is that +/// column, else the column by name. +pub(crate) fn ranked_value(child: &OperatorNode) -> Result<(ColumnId, ColumnRef), ResolveError> { + let ranked = ranked_column(child)?; + let column = match resolve_column_ref(&ColumnRef::SampleValue, &child.schema) { + Ok(value) if value == ranked => ColumnRef::SampleValue, + _ => column_ref(&child.schema.fields[ranked]), + }; + Ok((ranked, column)) +} + pub(crate) fn column_ref(column: &Field) -> ColumnRef { match &column.table { Some(t) => ColumnRef::Qualified { diff --git a/crates/asap-aware-mapping/src/stage2_physical_asap_optimization.rs b/crates/asap-aware-mapping/src/stage2_physical_asap_optimization.rs index 67c21dd9d..180550b5a 100644 --- a/crates/asap-aware-mapping/src/stage2_physical_asap_optimization.rs +++ b/crates/asap-aware-mapping/src/stage2_physical_asap_optimization.rs @@ -31,13 +31,12 @@ use asap_types::ir::physical_export::{ compile_physical_asap_workload_with_node_ids, PhysicalASAPDAG, }; use asap_types::ir::{ - apply_materialization_timings, ASAPOp, MaterializationAssignment, NonASAPOp, Operator, - OperatorNode, ProjectItem, QueryRoot, ScalarExpr, SchemaDerivationError, SortKey, TimingMemo, + apply_materialization_timings, MaterializationAssignment, NonASAPOp, Operator, OperatorNode, + ProjectItem, QueryRoot, ScalarExpr, SchemaDerivationError, SortKey, TimingMemo, }; use asap_types::post_asap::{ExecutionDataStateError, Materialization}; use asap_types::pre_asap::column_resolution::resolve_column_ref; use asap_types::pre_asap::expr_ir::ColumnRef; -use asap_types::pre_asap::schema::ColumnId; use asap_types::pre_asap::{AggIntent, Reduction}; use thiserror::Error; @@ -176,7 +175,8 @@ fn implement_topk( return Err(unsupported("HAVING")); } let child = memo[&Rc::as_ptr(child)].clone(); - let value = ranked_column(&child)?; + let value = crate::realization::ranked_column(&child) + .map_err(|e| unsupported(&format!("no ranked column: {e}")))?; let sorted = Rc::new( OperatorNode::new(Operator::NonASAP(NonASAPOp::Sort { keys: vec![SortKey { @@ -228,52 +228,6 @@ fn implement_topk( )) } -/// The column exact top-k ranks `child` by. Over a one-measure grouped -/// aggregate (the SQL heavy-hitter shape and PromQL `topk(k, sum by ..)`) it -/// is that measure, the last column after the group keys, whichever Stage 1 -/// candidate realizes it: the aggregate itself, or a `SummaryAgg` read by -/// `FinalizeExactAccumulator` or `SummaryEstimate`, under the `Project` -/// restoring its names (a renaming) and a value `HAVING`'s `Filter`. Over per-series rows -/// (PromQL `topk by (..) (k, rows)`) it is the sample value. -fn ranked_column(child: &OperatorNode) -> Result { - let mut node = child; - let reduction = loop { - match &node.operator { - // Only a renaming keeps the measure last. - Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) - if cols.len() == child.schema.fields.len() - && cols - .iter() - .enumerate() - .all(|(i, item)| item.expr == ScalarExpr::Column(i)) => - { - node = child - } - Operator::NonASAP(NonASAPOp::Filter { child, .. }) - | Operator::ASAP( - ASAPOp::FinalizeExactAccumulator { child } - | ASAPOp::SummaryEstimate { - summary_input: child, - .. - }, - ) => node = child, - Operator::ASAP(ASAPOp::SummaryAgg { reduction, .. }) => break Some(reduction), - Operator::NonASAP(NonASAPOp::Aggregate { - reduction, - measures, - .. - }) if measures.len() == 1 => break Some(reduction), - _ => break None, - } - }; - match reduction { - Some(Reduction::Reduce(_)) => Ok(child.schema.fields.len() - 1), - _ => resolve_column_ref(&ColumnRef::SampleValue, &child.schema).map_err(|e| { - PhysicalASAPOptimizationError::ExactTopK(format!("no ranked column: {e}")) - }), - } -} - /// `node` over its implemented children; the same `Rc` when none changed. fn rebuilt( node: &Rc, diff --git a/crates/integration-tests/tests/per_computation_candidates_sql.rs b/crates/integration-tests/tests/per_computation_candidates_sql.rs index b2c17d253..f1a82490c 100644 --- a/crates/integration-tests/tests/per_computation_candidates_sql.rs +++ b/crates/integration-tests/tests/per_computation_candidates_sql.rs @@ -1,11 +1,16 @@ -//! SQL `HAVING` → Pass 1 per-computation candidates: where each `HAVING` is applied. +//! SQL → Pass 1 per-computation candidates: where each `HAVING` is applied, +//! the guarantees composed over estimates, and what a Top-K heap sketch +//! ranks. use asap_aware_mapping::per_computation_candidates::{ - generate_per_computation_candidates, ComputationTarget, HavingPlacement, + compose_logical_candidate, generate_per_computation_candidates, ComputationCandidate, + ComputationTarget, HavingPlacement, }; use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_types::ir::QueryRoot; +use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; +use asap_types::post_asap::{FieldDataType, SketchAlgorithm, SummaryInputExpr}; use asap_types::pre_asap::schema::{DataType, Field, Schema}; +use asap_types::pre_asap::ColumnRef; use asap_types::types::AccuracyTarget; async fn targets(sql: &str) -> Vec { @@ -177,3 +182,103 @@ async fn a_sql_fold_over_estimates_leaves_its_row_count_unbound() { assert!(!delta.contains("input_row_count"), "{delta}"); } } + +/// The item and weight columns of every heap sketch build in each composed +/// candidate of `sql` over `flows(key: key_type, bytes)` that picks a heap +/// sketch for its top-k. +async fn heap_updates(key: &str, key_type: DataType, sql: &str) -> Vec<(String, String)> { + let catalog = SqlCatalog::new().with_table( + "flows", + Schema::new(vec![ + Field::plain(key, key_type, false), + Field::plain("bytes", DataType::Float64, false), + ]), + ); + let accuracy = AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.01, + }; + let root = lower_sql(sql, &catalog, accuracy).await.unwrap(); + let inventory = + generate_per_computation_candidates(vec![(0, QueryRoot::Operator(root))]).unwrap(); + let name = |expr: &SummaryInputExpr| match expr { + SummaryInputExpr::Column(ColumnRef::Named(name)) => name.clone(), + SummaryInputExpr::Tuple(items) => match items.as_slice() { + [SummaryInputExpr::Column(ColumnRef::Named(name))] => name.clone(), + other => panic!("one item column: {other:?}"), + }, + other => panic!("a named column: {other:?}"), + }; + let mut updates = Vec::new(); + for (t, target) in inventory.targets.iter().enumerate() { + for (c, candidate) in target.choices.candidates.iter().enumerate() { + if !matches!( + candidate, + ComputationCandidate::Summary( + SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap + ) + ) { + continue; + } + let mut choice = vec![0; inventory.targets.len()]; + choice[t] = c; + let composed = compose_logical_candidate(&inventory, &choice) + .unwrap_or_else(|e| panic!("{candidate:?} composes: {e}")); + let QueryRoot::Operator(root) = &composed[0].1 else { + panic!("operator root") + }; + for node in OperatorNode::reachable(root) { + if let Some(ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(..), + input, + .. + }) = node.asap() + { + updates.push((name(input.item.as_ref().unwrap()), name(&input.weight))); + } + } + } + } + updates +} + +const COUNT_RANKED_TOPK: &str = + "SELECT KEY, count(*) FROM flows GROUP BY KEY ORDER BY count(*) DESC LIMIT 10"; + +/// An Int64 key is not mistaken for the ranked value: both heap sketches +/// compose, keyed by the key and weighted by the count. +#[tokio::test] +async fn topk_heap_over_an_int64_key_is_weighted_by_the_count() { + let updates = heap_updates( + "port", + DataType::Int64, + &COUNT_RANKED_TOPK.replace("KEY", "port"), + ) + .await; + assert_eq!(updates.len(), 2, "{updates:?}"); + assert!( + updates + .iter() + .all(|(item, weight)| item == "port" && weight.starts_with("count(")), + "{updates:?}" + ); +} + +/// A key named `value` is not the ranked value: the heap sketches are +/// weighted by the count, not by the key. +#[tokio::test] +async fn topk_heap_over_a_key_named_value_is_weighted_by_the_count() { + let updates = heap_updates( + "value", + DataType::Utf8, + &COUNT_RANKED_TOPK.replace("KEY", "value"), + ) + .await; + assert_eq!(updates.len(), 2, "{updates:?}"); + assert!( + updates + .iter() + .all(|(item, weight)| item == "value" && weight.starts_with("count(")), + "{updates:?}" + ); +} From 0612c3cd9f3fc707530762a2ee1748e7ce92ecb0 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:32:30 +0000 Subject: [PATCH 02/15] feat(logical-asap-optimization): Pass 2 identical-expression rule as Stage 3 candidates Stage 1 Pass 2 now applies the identical-expression rule of Algorithm 2. For each composed Pass 1 candidate it finds the sharing groups (two or more nodes with identical input and computation semantics that are legal to share) and adds one candidate per non-empty subset of the groups, keeping the independent candidate. A subset that leaves out a group that the other shared groups already merge gives the same candidate as the larger subset, so it is listed once. Nothing is capped. The variants are ordinary candidates for Stage 2 and Stage 3. PromQL rows that carry the full series identity now declare (identity, timestamp) as a unique key, which makes scans and range selectors legal to share. Plan selection treats totals that differ only by floating-point rounding as a tie and keeps the earlier candidate. A shared NotMaterialized input is charged once per consumer, so it ties with the independent plan. Example 1 gets 9 logical candidates (3 Pass 1 x (1 + 2 Pass 2)) and Example 3a gets 972. stage_pipeline's --max-combinations still counts Pass 1 combinations; its document lists every Stage 1 candidate. The Example 1 and scalar-queries viewer fixtures are regenerated. --- .../asap-aware-mapping/src/asap_aware_cse.rs | 343 + crates/asap-aware-mapping/src/lib.rs | 4 + crates/asap-aware-mapping/src/pass/mod.rs | 4 +- .../src/pass/stage1_to_stage3.rs | 91 +- .../src/stage3_plan_selection.rs | 28 +- crates/devtools/src/bin/stage_pipeline.rs | 14 +- crates/devtools/src/planning_stages.rs | 6 +- crates/devtools/tests/stage_pipeline.rs | 16 +- .../tests/planner_layering_common/mod.rs | 4 +- .../tests/planner_layering_example1.rs | 20 +- .../tests/planner_layering_example3.rs | 3 +- crates/planner/tests/summary_sharing.rs | 9 +- crates/types/src/ir/schema_support.rs | 7 + .../examples/planner-layering-example1.json | 16589 ++++++++++++++-- tools/dag-viewer/examples/scalar-queries.json | 154 +- 15 files changed, 16068 insertions(+), 1224 deletions(-) create mode 100644 crates/asap-aware-mapping/src/asap_aware_cse.rs diff --git a/crates/asap-aware-mapping/src/asap_aware_cse.rs b/crates/asap-aware-mapping/src/asap_aware_cse.rs new file mode 100644 index 000000000..4945e93e5 --- /dev/null +++ b/crates/asap-aware-mapping/src/asap_aware_cse.rs @@ -0,0 +1,343 @@ +//! Stage 1 (logical ASAP-aware optimization), Pass 2: ASAP-aware +//! common-subexpression elimination, as defined in +//! `docs/design_docs/proposals/planner-layering.md` (Algorithm 2). +//! +//! Only the identical-expression rule is implemented. A *sharing group* of a +//! whole-workload candidate is a set of two or more of its nodes with +//! identical input and computation semantics that are legal to share: the +//! `ir::cse` interning merges only ASAP nodes and nodes with a unique key. +//! [`identical_expression_variants`] returns one candidate per non-empty +//! subset of the groups (Algorithm 2, line 31): each group in the subset +//! becomes one node serving every consumer, and every other group stays +//! separate. The caller keeps the independent candidate beside them, so plan +//! selection compares all of them; nothing is capped. +//! +//! Sharing a node shares the sub-DAG below it. A subset that leaves out a +//! group whose nodes all lie below the nodes of a group it shares gives the +//! same candidate as the subset with that group; such a candidate is +//! returned once, under the subset of the groups it actually shares. +//! +//! The summary-capability and window-composition rules are not implemented. + +use std::collections::HashMap; +#[cfg(test)] +use std::collections::HashSet; +use std::rc::Rc; + +use asap_types::ir::cse::share_common_sub_dags; +use asap_types::ir::{OperatorNode, QueryRoot}; + +/// A Pass 2 candidate: the candidate's roots with the sharing groups +/// `shared_groups` (indices into [`sharing_groups`]) shared. +#[derive(Debug, Clone)] +pub struct IdenticalExpressionVariant { + pub shared_groups: Vec, + pub roots: Vec, +} + +/// One sharing group: the identical nodes, in the order a children-first +/// walk of the roots meets them. +#[derive(Debug, Clone)] +pub struct SharingGroup { + pub nodes: Vec>, +} + +/// The sharing groups of the candidate with query roots `roots`, children +/// before parents. +pub fn sharing_groups(roots: &[QueryRoot]) -> Vec { + let operators: Vec> = roots.iter().flat_map(root_operators).cloned().collect(); + let interned = share_common_sub_dags(operators.iter().cloned().enumerate().collect()); + // Each node's interned representative: interning rebuilds every node + // over its children in order, so the two DAGs are walked side by side. + let mut representative: HashMap<*const OperatorNode, *const OperatorNode> = HashMap::new(); + let mut order = Vec::new(); + for (original, (_, merged)) in operators.iter().zip(&interned) { + pair(original, merged, &mut representative, &mut order); + } + let mut members: HashMap<*const OperatorNode, Vec>> = HashMap::new(); + let mut first_seen = Vec::new(); + for node in order { + let key = representative[&Rc::as_ptr(&node)]; + let entry = members.entry(key).or_default(); + if entry.is_empty() { + first_seen.push(key); + } + entry.push(node); + } + first_seen + .into_iter() + .filter_map(|key| { + let nodes = members.remove(&key)?; + (nodes.len() >= 2).then_some(SharingGroup { nodes }) + }) + .collect() +} + +/// Record `original`'s representative `merged` and recurse into the +/// children, then list `original` (children first, each node once). +fn pair( + original: &Rc, + merged: &Rc, + representative: &mut HashMap<*const OperatorNode, *const OperatorNode>, + order: &mut Vec>, +) { + if representative.contains_key(&Rc::as_ptr(original)) { + return; + } + representative.insert(Rc::as_ptr(original), Rc::as_ptr(merged)); + for (child, merged_child) in original.children().into_iter().zip(merged.children()) { + pair(child, merged_child, representative, order); + } + order.push(Rc::clone(original)); +} + +/// Every Pass 2 candidate of the identical-expression rule for the +/// candidate with query roots `roots`: one per non-empty subset of its +/// [`sharing_groups`] that gives a distinct candidate, in binary-counter +/// order of the subsets (the first group varying fastest). Empty when +/// nothing is identical. +pub fn identical_expression_variants(roots: &[QueryRoot]) -> Vec { + let groups = sharing_groups(roots); + let group_of: HashMap<*const OperatorNode, usize> = groups + .iter() + .enumerate() + .flat_map(|(index, group)| group.nodes.iter().map(move |n| (Rc::as_ptr(n), index))) + .collect(); + let mut variants = Vec::new(); + let mut subset = vec![false; groups.len()]; + while next_subset(&mut subset) { + let mut sharing = Sharing { + subset: &subset, + group_of: &group_of, + rebuilt: HashMap::new(), + shared: HashMap::new(), + }; + let shared_roots: Vec = roots.iter().map(|r| sharing.root(r)).collect(); + // The groups this candidate actually shares: every node of the group + // that is still read became one node. A node below a shared node + // other than the one kept is no longer read. + let merged: Vec = groups + .iter() + .map(|group| { + let mut nodes = group.nodes.iter().filter_map(|node| { + let (_, rebuilt) = sharing.rebuilt.get(&Rc::as_ptr(node))?; + Some(Rc::as_ptr(rebuilt)) + }); + let first = nodes.next(); + nodes.all(|node| Some(node) == first) + }) + .collect(); + if merged == subset { + variants.push(IdenticalExpressionVariant { + shared_groups: (0..groups.len()).filter(|&g| subset[g]).collect(), + roots: shared_roots, + }); + } + } + variants +} + +/// Binary increment of `subset`, the first entry the lowest bit; `false` +/// once every subset has been visited. +fn next_subset(subset: &mut [bool]) -> bool { + for bit in subset.iter_mut() { + *bit = !*bit; + if *bit { + return true; + } + } + false +} + +/// One application of the rule: the groups in `subset` shared. +struct Sharing<'a> { + subset: &'a [bool], + group_of: &'a HashMap<*const OperatorNode, usize>, + /// Original node → (original, rebuilt). The original is kept alive so + /// its address cannot be reused while it is a key. + rebuilt: HashMap<*const OperatorNode, (Rc, Rc)>, + /// Group → the one node every member of a shared group becomes. + shared: HashMap>, +} + +impl Sharing<'_> { + fn root(&mut self, root: &QueryRoot) -> QueryRoot { + match root { + QueryRoot::Operator(node) => QueryRoot::Operator(self.node(node)), + QueryRoot::Scalar(expr) => { + QueryRoot::Scalar(expr.map_operator_refs(&mut |node| self.node(node))) + } + } + } + + fn node(&mut self, node: &Rc) -> Rc { + if let Some((_, rebuilt)) = self.rebuilt.get(&Rc::as_ptr(node)) { + return Rc::clone(rebuilt); + } + let group = self + .group_of + .get(&Rc::as_ptr(node)) + .copied() + .filter(|&g| self.subset[g]); + let rebuilt = match group.and_then(|g| self.shared.get(&g)) { + Some(shared) => Rc::clone(shared), + None => { + let rebuilt = self.over_rebuilt_children(node); + if let Some(g) = group { + self.shared.insert(g, Rc::clone(&rebuilt)); + } + rebuilt + } + }; + self.rebuilt + .insert(Rc::as_ptr(node), (Rc::clone(node), Rc::clone(&rebuilt))); + rebuilt + } + + /// `node` over its rebuilt children; the same `Rc` when none changed. + /// Every child is replaced by an identical node, so `node`'s schema, + /// guarantee, timing and materialization stay valid. + fn over_rebuilt_children(&mut self, node: &Rc) -> Rc { + let mut changed = false; + let operator = node.operator.map_children(|child| { + let rebuilt = self.node(child); + changed |= !Rc::ptr_eq(&rebuilt, child); + rebuilt + }); + if !changed { + return Rc::clone(node); + } + Rc::new( + OperatorNode::with_schema(operator, node.schema.clone()) + .with_guarantee(node.guarantee.clone()) + .with_timing(node.timing) + .with_materialization(node.materialization), + ) + } +} + +fn root_operators(root: &QueryRoot) -> Vec<&Rc> { + match root { + QueryRoot::Operator(node) => vec![node], + QueryRoot::Scalar(expr) => expr.operator_refs(), + } +} + +/// Distinct nodes (by identity) reachable from `roots`. +#[cfg(test)] +fn distinct_nodes(roots: &[QueryRoot]) -> usize { + roots + .iter() + .flat_map(root_operators) + .flat_map(OperatorNode::reachable) + .map(|node| Rc::as_ptr(&node)) + .collect::>() + .len() +} + +/// Nodes reachable from both `a` and `b`. +#[cfg(test)] +fn common_nodes(a: &QueryRoot, b: &QueryRoot) -> HashSet<*const OperatorNode> { + let reach = |root: &QueryRoot| -> HashSet<*const OperatorNode> { + root_operators(root) + .into_iter() + .flat_map(OperatorNode::reachable) + .map(|node| Rc::as_ptr(&node)) + .collect() + }; + reach(a).intersection(&reach(b)).copied().collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::lower_promql; + use asap_types::ir::schema_support::with_promql_series_identity; + use asap_types::types::AccuracyTarget; + + fn roots(queries: &[&str]) -> Vec { + queries + .iter() + .map(|q| { + let root = lower_promql(q, AccuracyTarget::Exact); + QueryRoot::Operator(with_promql_series_identity(&root).unwrap()) + }) + .collect() + } + + fn kinds(group: &SharingGroup) -> &'static str { + group.nodes[0].operator.kind_name() + } + + /// Two queries over the same range selector have two sharing groups, + /// the scans and the time ranges. Sharing the time range shares the + /// scan below it, so there are two variants: the scan alone shared, and + /// both shared. Each query's value is unchanged. + #[test] + fn identical_range_selector_gives_one_variant_per_distinct_subset() { + let roots = roots(&[ + "sum by (job) (rate(m[1m]))", + "topk by (job) (10, sum_over_time(m[1m]))", + ]); + let groups = sharing_groups(&roots); + assert_eq!( + groups.iter().map(kinds).collect::>(), + ["Scan", "TimeRange"] + ); + let variants = identical_expression_variants(&roots); + assert_eq!( + variants + .iter() + .map(|v| v.shared_groups.clone()) + .collect::>(), + [vec![0], vec![0, 1]] + ); + for variant in &variants { + assert_eq!(variant.roots, roots, "values are unchanged"); + assert!(distinct_nodes(&variant.roots) < distinct_nodes(&roots)); + } + assert_eq!( + common_nodes(&variants[0].roots[0], &variants[0].roots[1]).len(), + 1 + ); + assert_eq!( + common_nodes(&variants[1].roots[0], &variants[1].roots[1]).len(), + 2 + ); + } + + /// Two pairs of queries over different metrics: each pair has a scan + /// group and an instant-selector group, with two distinct ways to share + /// (the scan, or the scan and the selector) or none, so 3 × 3 − 1 = 8 + /// Pass 2 candidates; each pair is shared alone or with the other. + #[test] + fn independent_groups_give_every_non_empty_subset() { + let roots = roots(&["sum(a)", "max(a)", "sum(b)", "max(b)"]); + assert_eq!( + sharing_groups(&roots).iter().map(kinds).collect::>(), + ["Scan", "TimeRange", "Scan", "TimeRange"] + ); + let variants = identical_expression_variants(&roots); + assert_eq!(variants.len(), 8); + let shares = |v: &IdenticalExpressionVariant| { + ( + !common_nodes(&v.roots[0], &v.roots[1]).is_empty(), + !common_nodes(&v.roots[2], &v.roots[3]).is_empty(), + ) + }; + assert_eq!( + variants.iter().map(shares).collect::>(), + HashSet::from([(true, false), (false, true), (true, true)]) + ); + assert!(variants.iter().all(|v| v.roots == roots)); + } + + /// Queries with nothing in common have no Pass 2 candidate. + #[test] + fn nothing_identical_gives_no_variant() { + assert!( + identical_expression_variants(&roots(&["sum(rate(a[1m]))", "sum(rate(b[1m]))"])) + .is_empty() + ); + } +} diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index ab508f2eb..41323238f 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -214,6 +214,10 @@ pub mod maintained_population; /// assigned: that is a Stage 2 materialization decision. pub mod per_computation_candidates; +/// Stage 1, Pass 2: ASAP-aware common-subexpression elimination (the +/// identical-expression rule). +pub mod asap_aware_cse; + /// #509 Stage 2, physical ASAP-aware optimization (MVP): materialization /// (every sub-DAG `NotMaterialized`) and physical operator implementation, /// producing the CandidatePhysicalASAPDAGs. diff --git a/crates/asap-aware-mapping/src/pass/mod.rs b/crates/asap-aware-mapping/src/pass/mod.rs index ee9f8c582..de7e9ac02 100644 --- a/crates/asap-aware-mapping/src/pass/mod.rs +++ b/crates/asap-aware-mapping/src/pass/mod.rs @@ -35,8 +35,8 @@ use crate::per_computation_candidates::PerComputationCandidateError; use crate::stage3_plan_selection::{Selection, SelectionError}; pub use stage1_to_stage3::{ - run_stage1_to_stage3, run_stage1_to_stage3_from_pass1, LogicalCandidate, PrunedCandidate, - Stage1ToStage3Pass, Stage1ToStage3Run, + run_stage1_to_stage3, run_stage1_to_stage3_from_pass1, IdenticalExpressionCandidate, + LogicalCandidate, PrunedCandidate, Stage1ToStage3Pass, Stage1ToStage3Run, }; static DEFAULT_COST_MODEL: DefaultCostModel = DefaultCostModel; diff --git a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs index d03e40e7b..e1ec2141e 100644 --- a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs +++ b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs @@ -9,13 +9,17 @@ //! 1. Stage 1, logical ASAP-aware optimization: Pass 1 per-computation candidate //! generation, then composition of one whole-workload candidate per choice //! of a per-computation candidate for every target (the Cartesian product, -//! in mixed-radix order, the last target varying fastest); -//! 2. Stage 2, physical ASAP-aware optimization of every composed candidate +//! in mixed-radix order, the last target varying fastest), each followed +//! by its Pass 2 candidates from the identical-expression rule +//! ([`identical_expression_variants`]: one per subset of its sharing +//! groups; the independent candidate is kept); +//! 2. Stage 2, physical ASAP-aware optimization of every Stage 1 candidate //! ([`optimize_physical_asap`]); //! 3. Stage 3, plan selection over the physical candidates ([`select_plan`]). //! //! Stage 0, the query-language frontends, runs before it (the `asap-planner` -//! facade lowers the workload). Pass 2 (ASAP-aware CSE) is not planned yet. +//! facade lowers the workload). Pass 2's summary-capability and +//! window-composition rules are not planned yet. //! //! Every combination is enumerated and plan selection prices every //! candidate; nothing is capped. A search that avoids building every @@ -30,6 +34,7 @@ use asap_types::types::AccuracyTarget; use asap_types::workload::{DataWorkload, QueryLanguage}; use super::{OptimizationInput, OptimizationPass, OptimizeError, PlanOutput, QueryPlan}; +use crate::asap_aware_cse::identical_expression_variants; use crate::per_computation_candidates::{ compose_logical_candidate, generate_per_computation_candidates, ComputationCandidate, PerComputationCandidateError, PerComputationCandidates, @@ -52,6 +57,17 @@ pub struct LogicalCandidate { pub roots: Result, PerComputationCandidateError>, } +/// A Stage 1 candidate Pass 2's identical-expression rule added: `id` is +/// `from_logical` with the nodes of each group in `shared` (named by +/// operator kind, children first) made one node. It has `from_logical`'s +/// `choice`. +#[derive(Debug, Clone, PartialEq)] +pub struct IdenticalExpressionCandidate { + pub id: String, + pub from_logical: String, + pub shared: Vec, +} + /// A composed logical candidate that physical ASAP-aware optimization could /// not implement, with the reason. #[derive(Debug, Clone, PartialEq)] @@ -70,11 +86,16 @@ pub struct Stage1ToStage3Run { /// Stage 1, Pass 1: the per-computation candidates, with the roots they /// read. pub stage1_pass1_per_computation: PerComputationCandidates, - /// Stage 1: the number of whole-workload candidates, the size of the - /// Cartesian product of the targets' candidates, saturating at - /// `usize::MAX`. + /// Stage 1, Pass 1: the number of composed whole-workload candidates, + /// the size of the Cartesian product of the targets' candidates, + /// saturating at `usize::MAX`. Pass 2 adds to them. pub combinations: usize, - /// Stage 1: every whole-workload logical candidate, in enumeration order. + /// Stage 1, Pass 2: the candidates the identical-expression rule added, + /// in enumeration order. + pub stage1_pass2_identical_expression: Vec, + /// Stage 1: every whole-workload logical candidate, in enumeration + /// order: each composed Pass 1 candidate, followed by its Pass 2 + /// candidates. pub stage1_logical_asap_optimization: Vec, /// Stage 2: one physical candidate (`P` from `L`) per Stage 1 /// candidate that composed and was implemented. @@ -141,19 +162,21 @@ pub fn run_stage1_to_stage3_from_pass1( ) -> Stage1ToStage3Run { let combinations = per_computation.combinations(); let query_of_target = query_of_target(&per_computation); - let mut logical = Vec::new(); + let mut logical: Vec = Vec::new(); let mut physical = Vec::new(); let mut pruned = Vec::new(); - let mut choice = vec![0; per_computation.targets.len()]; - for index in 0usize.. { - let id = format!("L{}", index + 1); - let display_name = display_name(&per_computation, &query_of_target, &choice); - let roots = compose_logical_candidate(&per_computation, &choice) - .map(|roots| roots.into_iter().map(|(_, root)| root).collect::>()); + let mut pass2 = Vec::new(); + // Stage 2 for one Stage 1 candidate, numbered after the ones before it. + let mut add = |logical: &mut Vec, + display_name: String, + choice: &[usize], + roots: Result, PerComputationCandidateError>| { + let number = logical.len() + 1; + let id = format!("L{number}"); if let Ok(roots) = &roots { match optimize_physical_asap(&id, roots) { Ok(mut candidate) => { - candidate.id = format!("P{}", index + 1); + candidate.id = format!("P{number}"); candidate.display_name = display_name.clone(); physical.push(candidate); } @@ -165,11 +188,44 @@ pub fn run_stage1_to_stage3_from_pass1( } } logical.push(LogicalCandidate { - id, + id: id.clone(), display_name, - choice: choice.clone(), + choice: choice.to_vec(), roots, }); + id + }; + let mut choice = vec![0; per_computation.targets.len()]; + loop { + let display_name = display_name(&per_computation, &query_of_target, &choice); + let roots = compose_logical_candidate(&per_computation, &choice) + .map(|roots| roots.into_iter().map(|(_, root)| root).collect::>()); + let (groups, variants) = match &roots { + Ok(roots) => ( + crate::asap_aware_cse::sharing_groups(roots), + identical_expression_variants(roots), + ), + Err(_) => (Vec::new(), Vec::new()), + }; + let from_logical = add(&mut logical, display_name.clone(), &choice, roots); + for variant in variants { + let shared: Vec = variant + .shared_groups + .iter() + .map(|&g| groups[g].nodes[0].operator.kind_name().to_string()) + .collect(); + let id = add( + &mut logical, + format!("{display_name} · shared {}", shared.join(", ")), + &choice, + Ok(variant.roots), + ); + pass2.push(IdenticalExpressionCandidate { + id, + from_logical: from_logical.clone(), + shared, + }); + } // Mixed-radix increment: the last target varies fastest; a carry // out of the first target ends the enumeration. let mut carry = true; @@ -189,6 +245,7 @@ pub fn run_stage1_to_stage3_from_pass1( Stage1ToStage3Run { stage1_pass1_per_computation: per_computation, combinations, + stage1_pass2_identical_expression: pass2, stage1_logical_asap_optimization: logical, stage2_physical_asap_optimization: physical, stage2_pruned: pruned, diff --git a/crates/asap-aware-mapping/src/stage3_plan_selection.rs b/crates/asap-aware-mapping/src/stage3_plan_selection.rs index 0dc836e71..17f4ac276 100644 --- a/crates/asap-aware-mapping/src/stage3_plan_selection.rs +++ b/crates/asap-aware-mapping/src/stage3_plan_selection.rs @@ -179,7 +179,7 @@ pub fn select_plan( node, error, })?; - if best.is_none_or(|(_, total)| cost.total < total) { + if best.is_none_or(|(_, total)| cheaper(cost.total, total)) { best = Some((&candidate.id, cost.total)); } costs.insert(candidate.id.clone(), cost); @@ -190,7 +190,7 @@ pub fn select_plan( for candidate in cands { if candidate.id != selected && costs.contains_key(&candidate.id) { let total = costs[&candidate.id].total; - let reason = if total == best_total { + let reason = if !cheaper(best_total, total) { format!( "tied with the selected candidate {selected} at {total:.3} {COST_UNIT}; \ lost on enumeration order" @@ -212,6 +212,13 @@ pub fn select_plan( }) } +/// Whether `cost` is cheaper than `best`. Totals that differ only by +/// floating-point rounding (summing the same charges in another grouping) +/// tie, and a tie keeps the earlier candidate. +fn cheaper(cost: f64, best: f64) -> bool { + cost < best - best.abs() * 1e-9 +} + /// The first query whose output is not shown to meet its target, as a /// reason. An output meets its target when its theoretical root guarantee /// (composed in Stage 1 with the built-in model) satisfies it, or when the @@ -1116,6 +1123,20 @@ mod tests { } } + /// Totals that differ only by rounding tie, so the earlier candidate is + /// kept; a real difference is cheaper. + #[test] + fn totals_differing_by_rounding_tie() { + // One unit in the last place apart, as the same charges summed in + // another grouping can be. + let separate = 0.336006_f64; + let together = f64::from_bits(separate.to_bits() - 1); + assert!(!cheaper(separate, together)); + assert!(!cheaper(together, separate)); + assert!(cheaper(0.3, 0.4)); + assert!(!cheaper(0.4, 0.3)); + } + /// For a query with a target, Count-Min over weights not proven /// non-negative is invalid. #[test] @@ -1939,7 +1960,8 @@ mod tests { .filter(|c| sketches(c) == [SketchAlgorithm::DDSketch, SketchAlgorithm::DDSketch]) .map(|c| c.id.clone()) .collect(); - assert_eq!(ids.len(), 1); + // The Pass 1 candidate and its Pass 2 candidates sharing the input. + assert!(!ids.is_empty()); assert!(ids.iter().all(|id| !without.costs.contains_key(id))); let (_, with) = run(PlanningModels::builtin().with_evidence(&BoundedQuantiles)); assert!(ids.iter().all(|id| with.costs.contains_key(id)), "{with:?}"); diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index 450439256..fbd325e37 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -1,7 +1,7 @@ // cargo run -p asap-devtools --bin stage_pipeline -- \ // --example planner-layering-1 --out planner-layering-example1.json // (also planner-layering-3a and planner-layering-3b: #509 Example 3, -// Patterns A and B; 3a has 243 candidates, all priced, 64 shown by default) +// Patterns A and B; 3a has 972 candidates, all priced, 64 shown by default) // cargo run -p asap-devtools --bin stage_pipeline -- \ // --promql "topk by (job) (10, rate(x[1m]))" --epsilon 0.01 --delta 0.001 --out run.json // cargo run -p asap-devtools --bin stage_pipeline -- \ @@ -14,7 +14,8 @@ // - stage0_query_language_frontends: the frontends' workload DAG, one root per query; // - stage1_logical_asap_optimization: Pass 1 whole-workload candidates, one per // choice of a per-computation candidate for every target (the Cartesian -// product), in enumeration order. Every candidate is optimized and +// product), each followed by its Pass 2 identical-expression candidates +// (one per subset of its sharing groups), in enumeration order. Every candidate is optimized and // priced; the document shows the first `--max-displayed` (default 64) // and the selected one, with `combinations` (how many exist) and // `capped` (whether some are not shown). A candidate that cannot be @@ -34,8 +35,9 @@ // When Pass 1 fails, the document still has the Stage 0 DAG; Stage 1 has // `error` and no candidates, and nothing is selected. // -// A workload with more than `--max-combinations` (default 10^4) -// whole-workload candidates is refused before Stage 1 enumerates any: the +// A workload with more than `--max-combinations` (default 10^4) Pass 1 +// whole-workload combinations is refused before Stage 1 enumerates any (Pass +// 2 adds candidates to each, which is known only once it is composed): the // run exits with status 3, the count and the limit, and writes no document. // Planning never caps or truncates the candidates it ranks; only the display // does (`--max-displayed`). @@ -77,8 +79,8 @@ const USAGE: &str = [--name ]... [--table ...] [--epsilon --delta ] [--interval-ms ])) \ [--max-combinations ] [--max-displayed ] --out "; -/// The default `--max-combinations`: the most whole-workload candidates a -/// run enumerates. +/// The default `--max-combinations`: the most Pass 1 whole-workload +/// combinations a run enumerates. const MAX_COMBINATIONS: usize = 10_000; /// Why a run wrote no document. diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index e2b8450f9..01616d3ef 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -94,12 +94,14 @@ impl From for String { } } -/// Logical ASAP-aware optimization (Pass 1 and composition; no Pass 2), +/// Logical ASAP-aware optimization (Pass 1, composition and Pass 2's +/// identical-expression rule), /// physical ASAP-aware optimization and plan selection over `roots`, the /// lowered entries of `workload`: the library's planner pipeline /// ([`run_stage1_to_stage3_from_pass1`]) with the built-in models. Logical /// candidates are every choice of a per-computation candidate for every -/// target, the last target varying fastest. Every candidate is enumerated +/// target, the last target varying fastest, each followed by its Pass 2 +/// candidates. Every candidate is enumerated /// and priced, so the selection is the cheapest of all; a caller that shows /// fewer cuts only its display. With `max_combinations`, a workload with more /// whole-workload candidates is refused after Pass 1, before any is diff --git a/crates/devtools/tests/stage_pipeline.rs b/crates/devtools/tests/stage_pipeline.rs index 0a5a584c1..4e4fa24c6 100644 --- a/crates/devtools/tests/stage_pipeline.rs +++ b/crates/devtools/tests/stage_pipeline.rs @@ -139,8 +139,9 @@ fn display_cap_is_recorded_and_keeps_the_selection() { let document = generate(&["--max-displayed", "1"]); let logical = &document["stage1_logical_asap_optimization"]; assert_eq!(logical["capped"], true); - assert!(logical["candidates"].as_array().unwrap().len() < 3); - assert_eq!(logical["combinations"], 3); + assert!(logical["candidates"].as_array().unwrap().len() < 9); + // 3 Pass 1 candidates, each with 2 Pass 2 candidates. + assert_eq!(logical["combinations"], 9); assert_eq!( document["stage3_plan_selection"]["selected"], full["stage3_plan_selection"]["selected"] @@ -185,7 +186,8 @@ fn uncomposable_candidate_is_recorded_with_its_error() { } } -/// Example 3a (243 candidates) selects what the planning stages select over +/// Example 3a (243 Pass 1 candidates, each with 3 Pass 2 candidates) +/// selects what the planning stages select over /// every candidate, which the acceptance tests also run: the document's /// display cap does not change the selection, and the selected plan is /// displayed with its cost. @@ -193,7 +195,7 @@ fn uncomposable_candidate_is_recorded_with_its_error() { fn example3a_selects_the_cheapest_of_all_candidates() { let workload = pattern_a(PatternARecurrence::OnceAdHoc, DataArrival::Mixed); let run = run_planning_stages(&workload, lower_promql_roots(&workload).unwrap(), None).unwrap(); - assert_eq!(run.stage1_logical_asap_optimization.len(), 243); + assert_eq!(run.stage1_logical_asap_optimization.len(), 972); let expected = run.stage3_plan_selection.unwrap().selected; let document = generate_example("planner-layering-3a", &[]); @@ -306,11 +308,13 @@ fn workload_above_the_combination_limit_is_rejected() { ), "{stderr}" ); - // At the limit, it plans. + // At the limit, it plans. The limit counts Pass 1's combinations; the + // document lists every Stage 1 candidate, each Pass 1 candidate with its + // 2 Pass 2 candidates. let document = generate(&["--max-combinations", "3"]); assert_eq!( document["stage1_logical_asap_optimization"]["combinations"], - 3 + 9 ); } diff --git a/crates/integration-tests/tests/planner_layering_common/mod.rs b/crates/integration-tests/tests/planner_layering_common/mod.rs index 663eb57c5..94f82428a 100644 --- a/crates/integration-tests/tests/planner_layering_common/mod.rs +++ b/crates/integration-tests/tests/planner_layering_common/mod.rs @@ -444,7 +444,9 @@ pub fn assert_selects_cheapest_valid(run: &Run) { .iter() .filter(|p| !invalid.contains_key(p.id.as_str())) { - assert!(best <= run.cost(&p.id).unwrap(), "{} is cheaper", p.id); + // Totals within rounding of each other tie (plan selection's rule). + let cost = run.cost(&p.id).unwrap(); + assert!(best <= cost + cost.abs() * 1e-9, "{} is cheaper", p.id); } } diff --git a/crates/integration-tests/tests/planner_layering_example1.rs b/crates/integration-tests/tests/planner_layering_example1.rs index eb36de598..4c984cb37 100644 --- a/crates/integration-tests/tests/planner_layering_example1.rs +++ b/crates/integration-tests/tests/planner_layering_example1.rs @@ -5,10 +5,12 @@ //! the real stages. Tests whose expectation needs a feature not in the stack //! yet are `#[ignore]`d with the PR or issue that delivers it. //! -//! Scope: logical ASAP-aware optimization = Pass 1 only (no Pass 2); -//! physical ASAP-aware optimization = operator implementation with every -//! sub-DAG `NotMaterialized`. Counts are 1 → 3 → 3 → 1, a subset of the -//! doc's 1 → 54 → 156 → 1. +//! Scope: logical ASAP-aware optimization = Pass 1 and Pass 2's +//! identical-expression rule (no window composition); physical ASAP-aware +//! optimization = operator implementation with every sub-DAG +//! `NotMaterialized`. Pass 1 gives 3 candidates; Pass 2 adds, for each, one +//! sharing the scan and one sharing the scan and the range selector: 9 → 9 → +//! 1, a subset of the doc's 1 → 54 → 156 → 1. mod planner_layering_common; @@ -93,8 +95,8 @@ fn classify(dag: &impl ExportedDag, query_roots: &[NodeId]) -> (Q2Option, bool) (option, !c1.is_disjoint(&c2)) } -/// The candidate with Q2 `option` whose queries share no node (#588 adds a -/// shared-input variant of each). +/// The candidate with Q2 `option` whose queries share no node (Pass 2 adds +/// shared-input variants of each). fn physical_with(run: &Run, option: Q2Option) -> &Physical { let mut found = run .physical @@ -156,8 +158,8 @@ fn stage0_query_language_frontends_queries_do_not_share_nodes() { // ── Logical ASAP-aware optimization ────────────────────────────────────── /// Pass 1 gives 1 × 3 = 3 candidates without a shared input: Q2's top-k is -/// exact, Count-Min + heap or CountSketch + heap. (#588 adds a shared-input -/// variant of each.) +/// exact, Count-Min + heap or CountSketch + heap. (Pass 2 adds shared-input +/// variants of each.) #[test] fn stage1_logical_asap_optimization_has_three_pass1_candidates() { let run = pipeline(); @@ -243,7 +245,6 @@ fn stage1_logical_asap_optimization_q2_offers_hydra() { /// Pass 2's identical-expression rule adds a shared-input variant of every Q2 option and keeps the independent one. #[test] -#[ignore = "needs the Pass 2 identical-expression rule (#588)"] fn stage1_logical_asap_optimization_keeps_independent_and_shared_variants() { let run = pipeline(); let found: Vec<_> = run @@ -603,7 +604,6 @@ fn stage3_plan_selection_selected_plan_compiles_in_the_physical_planner() { /// Sharing the input never costs more than reading it separately. #[test] -#[ignore = "needs the Pass 2 identical-expression rule (#588): there are no shared-input variants to compare"] fn stage3_plan_selection_shared_input_is_not_costlier() { let run = pipeline(); let by_combo: BTreeMap<_, _> = run diff --git a/crates/integration-tests/tests/planner_layering_example3.rs b/crates/integration-tests/tests/planner_layering_example3.rs index 06450e771..cb0a5f023 100644 --- a/crates/integration-tests/tests/planner_layering_example3.rs +++ b/crates/integration-tests/tests/planner_layering_example3.rs @@ -159,7 +159,6 @@ fn stage1_logical_asap_optimization_a_keeps_five_independent_klls() { /// The identical-expression rule shares only raw input (scan, range, shift), never a quantile or summary, and keeps the unshared variant. #[test] -#[ignore = "needs the Pass 2 identical-expression rule (#588)"] fn stage1_logical_asap_optimization_a_identical_expression_rule_shares_only_raw_input() { let run = run_a(); assert!(run.logical.iter().any(|c| c.shared_input())); @@ -272,7 +271,7 @@ fn stage3_plan_selection_a_charges_each_node_per_consuming_query() { /// Sharing the scan never costs more than scanning separately, for the same per-computation choices. #[test] -#[ignore = "needs the Pass 2 identical-expression rule (#588) and plan selection row estimates for time-shifted scans and narrower time ranges (#632)"] +#[ignore = "needs plan selection row estimates for time-shifted scans and narrower time ranges (#632)"] fn stage3_plan_selection_a_shared_scan_is_not_costlier() { let run = run_a(); let key = |c: &Logical| { diff --git a/crates/planner/tests/summary_sharing.rs b/crates/planner/tests/summary_sharing.rs index 24206273c..e8d47b126 100644 --- a/crates/planner/tests/summary_sharing.rs +++ b/crates/planner/tests/summary_sharing.rs @@ -494,15 +494,16 @@ async fn combined_vector_quantiles_keep_unselected_summary_candidates() { } /// Front ends build structurally identical references as separate nodes. -/// Until Pass 2 merges them (#580), each is planned on its own; the plan -/// holds one plan per distinct reference and satisfies the plan contract. +/// Pass 2's identical-expression rule merges them, and the merged candidate +/// computes the reference once per query, so it is selected: the plan holds +/// one plan per distinct reference and satisfies the plan contract. #[tokio::test] async fn identical_scalar_references_satisfy_the_plan_contract() { for (query, distinct) in [ - ("scalar(rate(x[5m])) - scalar(rate(x[5m]))", 2), + ("scalar(rate(x[5m])) - scalar(rate(x[5m]))", 1), ( "scalar(sum(rate(x[5m]))) / scalar(sum(rate(x[5m]))) + scalar(count(x))", - 3, + 2, ), ] { let output = plan_promql(&[(query, 0.01)]).await; diff --git a/crates/types/src/ir/schema_support.rs b/crates/types/src/ir/schema_support.rs index 6b86cfae1..c3ec9931f 100644 --- a/crates/types/src/ir/schema_support.rs +++ b/crates/types/src/ir/schema_support.rs @@ -61,6 +61,13 @@ pub fn with_promql_series_identity( false, )); schema.closed = true; + // A series has at most one sample per timestamp, so the full + // identity and the timestamp key each row. Pass 2's + // identical-expression rule reads this key: only a keyed + // node is legal to share (`ir::cse`). + if let Some(time) = schema.time_index { + schema.add_unique_key(vec![schema.fields.len() - 1, time]); + } } Operator::NonASAP(NonASAPOp::Sort { partition_by, .. }) if partition_by.is_without() => diff --git a/tools/dag-viewer/examples/planner-layering-example1.json b/tools/dag-viewer/examples/planner-layering-example1.json index 2f79af830..f1680bb7e 100644 --- a/tools/dag-viewer/examples/planner-layering-example1.json +++ b/tools/dag-viewer/examples/planner-layering-example1.json @@ -47,7 +47,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "source": { "TimeSeries": { @@ -95,7 +100,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -152,7 +162,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -215,7 +230,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -319,7 +339,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "source": { "TimeSeries": { @@ -367,7 +392,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -424,7 +454,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -488,7 +523,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -626,7 +666,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "source": { "TimeSeries": { @@ -674,7 +719,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -745,7 +795,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -818,7 +873,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -884,7 +944,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -1065,7 +1130,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "source": { "TimeSeries": { @@ -1113,7 +1183,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -1184,7 +1259,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -1257,7 +1337,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -1323,7 +1408,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -1475,7 +1565,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "source": { "TimeSeries": { @@ -1523,7 +1618,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -1594,7 +1694,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -1667,7 +1772,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -1733,7 +1843,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -1856,116 +1971,6 @@ }, "timing": null }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact over exact inputs" - } - ] - }, - "materialization": "NotMaterialized", - "operator": { - "NonASAP": { - "Scan": { - "predicates": [], - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "source": { - "TimeSeries": { - "metric": "http_requests_total" - } - } - } - } - }, - "result_kind": "instant_vector", - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "timing": null - }, { "guarantee": { "bound": { @@ -1986,7 +1991,7 @@ "operator": { "NonASAP": { "TimeRange": { - "child": 6, + "child": 0, "kind": "range", "range": { "nanos": 0, @@ -2033,7 +2038,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -2043,7 +2053,7 @@ "operator": { "ASAP": { "SummaryAgg": { - "child": 7, + "child": 6, "family": { "ExactAggregate": [ "Sum", @@ -2106,7 +2116,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -2130,7 +2145,7 @@ "operator": { "ASAP": { "FinalizeExactAccumulator": { - "child": 8 + "child": 7 } } }, @@ -2172,166 +2187,56 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, { - "guarantee": null, - "materialization": "NotMaterialized", - "operator": { - "ASAP": { - "SummaryAgg": { - "child": 9, - "family": { - "Sketch": [ - { - "algorithm": "CmsWithHeap", - "category": "TopK", - "params": { - "CmsWithHeap": { - "depth": 7, - "heap_size": 100, - "width": 272 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "filter": null, - "grouping": "PerSubpopulationInstance", - "input": { - "item": { - "Column": { - "Named": "$promql_series_identity" - } - }, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" - } - }, - "reduction": { - "Reduce": [ - 2 - ] - } - } - } - }, - "result_kind": "state", - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Sketch": [ - { - "algorithm": "CmsWithHeap", - "category": "TopK", - "params": { - "CmsWithHeap": { - "depth": 7, - "heap_size": 100, - "width": 272 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "name": "state", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "timing": null - }, - { - "guarantee": null, - "materialization": "NotMaterialized", - "operator": { - "ASAP": { - "SummaryEstimate": { - "query": { - "TopK": { - "k": 10 - } - }, - "summary_input": 10 - } - } - }, - "result_kind": "instant_vector", - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ { - "dtype": { - "Plain": "utf8" - }, - "name": "topk", - "nullable": false, - "table": null + "kind": "exact", + "reason": "exact over exact inputs" } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] ] }, - "timing": null - }, - { - "guarantee": null, "materialization": "NotMaterialized", "operator": { "NonASAP": { - "Project": { - "child": 11, - "cols": [ - { - "alias": "job", - "expr": { - "Column": 0 - } - }, + "Aggregate": { + "child": 8, + "filters": [], + "having": null, + "measures": [ { - "alias": "topk_10", - "expr": { - "Column": 1 - } + "accuracy": { + "EpsilonDelta": { + "delta": 0.001, + "epsilon": 0.01 + } + }, + "k": 10, + "kind": "top_k" } ], - "qualifier": null + "output_names": [], + "reduction": { + "Reduce": [ + 2 + ] + } } } }, @@ -2371,11 +2276,11 @@ "Operator": 5 }, { - "Operator": 12 + "Operator": 9 } ] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap (Sum acc)", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 exact (Sum acc) · shared Scan", "id": "L2" }, { @@ -2439,7 +2344,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "source": { "TimeSeries": { @@ -2487,7 +2397,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -2558,7 +2473,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -2631,7 +2551,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -2697,7 +2622,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -2821,74 +2751,34 @@ "timing": null }, { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact over exact inputs" - } - ] - }, + "guarantee": null, "materialization": "NotMaterialized", "operator": { - "NonASAP": { - "Scan": { - "predicates": [], - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] }, - "source": { - "TimeSeries": { - "metric": "http_requests_total" - } - } - } + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } } }, - "result_kind": "instant_vector", + "result_kind": "state", "schema": { "closed": true, "fields": [ @@ -2902,9 +2792,12 @@ }, { "dtype": { - "Plain": "float64" + "ExactAggregate": [ + "Sum", + "Sum" + ] }, - "name": "value", + "name": "state", "nullable": false, "table": null }, @@ -2926,7 +2819,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -2942,24 +2840,19 @@ "provenance": [ { "kind": "exact", - "reason": "exact over exact inputs" + "reason": "exact accumulator over exact input" } ] }, "materialization": "NotMaterialized", "operator": { - "NonASAP": { - "TimeRange": { - "child": 6, - "kind": "range", - "range": { - "nanos": 0, - "secs": 60 - } + "ASAP": { + "FinalizeExactAccumulator": { + "child": 6 } } }, - "result_kind": "range_vector", + "result_kind": "instant_vector", "schema": { "closed": true, "fields": [ @@ -2997,39 +2890,179 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, { - "guarantee": null, + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, "materialization": "NotMaterialized", "operator": { - "ASAP": { - "SummaryAgg": { + "NonASAP": { + "Aggregate": { "child": 7, - "family": { - "ExactAggregate": [ - "Sum", - "Sum" + "filters": [], + "having": null, + "measures": [ + { + "accuracy": { + "EpsilonDelta": { + "delta": 0.001, + "epsilon": 0.01 + } + }, + "k": 10, + "kind": "top_k" + } + ], + "output_names": [], + "reduction": { + "Reduce": [ + 2 ] + } + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" }, - "filter": null, - "grouping": "PerSubpopulationInstance", - "input": { - "item": null, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" - } + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" }, - "reduction": "PerEntity" + "name": "topk_10", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + } + ], + "roots": [ + { + "Operator": 5 + }, + { + "Operator": 8 + } + ] + }, + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 exact (Sum acc) · shared Scan, TimeRange", + "id": "L3" + }, + { + "dag": { + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } } } }, - "result_kind": "state", + "result_kind": "instant_vector", "schema": { "closed": true, "fields": [ @@ -3043,12 +3076,9 @@ }, { "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" - ] + "Plain": "float64" }, - "name": "state", + "name": "value", "nullable": false, "table": null }, @@ -3070,7 +3100,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -3086,19 +3121,24 @@ "provenance": [ { "kind": "exact", - "reason": "exact accumulator over exact input" + "reason": "exact over exact inputs" } ] }, "materialization": "NotMaterialized", "operator": { - "ASAP": { - "FinalizeExactAccumulator": { - "child": 8 + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } } } }, - "result_kind": "instant_vector", + "result_kind": "range_vector", "schema": { "closed": true, "fields": [ @@ -3136,7 +3176,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -3146,31 +3191,166 @@ "operator": { "ASAP": { "SummaryAgg": { - "child": 9, + "child": 1, "family": { - "Sketch": [ - { - "algorithm": "CountSketchWithHeap", - "category": "TopK", - "params": { - "CountSketchWithHeap": { - "depth": 125, - "heap_size": 100, - "width": 30000 - } - } - }, - "PerSubpopulationInstance" + "ExactAggregate": [ + "Rate", + "Rate" ] }, "filter": null, "grouping": "PerSubpopulationInstance", "input": { - "item": { - "Column": { - "Named": "$promql_series_identity" - } + "item": null, + "weight": { + "Column": "SampleValue" }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 2 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 3, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, "weight": { "Column": "SampleValue" }, @@ -3200,19 +3380,9 @@ }, { "dtype": { - "Sketch": [ - { - "algorithm": "CountSketchWithHeap", - "category": "TopK", - "params": { - "CountSketchWithHeap": { - "depth": 125, - "heap_size": 100, - "width": 30000 - } - } - }, - "PerSubpopulationInstance" + "ExactAggregate": [ + "Sum", + "Sum" ] }, "name": "state", @@ -3230,17 +3400,26 @@ "timing": null }, { - "guarantee": null, + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, "materialization": "NotMaterialized", "operator": { "ASAP": { - "SummaryEstimate": { - "query": { - "TopK": { - "k": 10 - } - }, - "summary_input": 10 + "FinalizeExactAccumulator": { + "child": 4 } } }, @@ -3258,9 +3437,9 @@ }, { "dtype": { - "Plain": "utf8" + "Plain": "float64" }, - "name": "topk", + "name": "sum", "nullable": false, "table": null } @@ -3275,32 +3454,14114 @@ "timing": null }, { - "guarantee": null, + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, "materialization": "NotMaterialized", "operator": { "NonASAP": { - "Project": { - "child": 11, - "cols": [ - { - "alias": "job", - "expr": { - "Column": 0 - } - }, - { - "alias": "topk_10", - "expr": { - "Column": 1 - } - } - ], - "qualifier": null - } - } - }, - "result_kind": "instant_vector", - "schema": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "TimeRange": { + "child": 6, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + }, + "result_kind": "range_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 7, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 8 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 9, + "family": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 10 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Project": { + "child": 11, + "cols": [ + { + "alias": "job", + "expr": { + "Column": 0 + } + }, + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk_10", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + } + ], + "roots": [ + { + "Operator": 5 + }, + { + "Operator": 12 + } + ] + }, + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap (Sum acc)", + "id": "L4" + }, + { + "dag": { + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + }, + "result_kind": "range_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 2 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 3, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 4 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + }, + "result_kind": "range_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 6, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 7 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 8, + "family": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 9 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Project": { + "child": 10, + "cols": [ + { + "alias": "job", + "expr": { + "Column": 0 + } + }, + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk_10", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + } + ], + "roots": [ + { + "Operator": 5 + }, + { + "Operator": 11 + } + ] + }, + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap (Sum acc) · shared Scan", + "id": "L5" + }, + { + "dag": { + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + }, + "result_kind": "range_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 2 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 3, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 4 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 6 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 7, + "family": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 8 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Project": { + "child": 9, + "cols": [ + { + "alias": "job", + "expr": { + "Column": 0 + } + }, + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk_10", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + } + ], + "roots": [ + { + "Operator": 5 + }, + { + "Operator": 10 + } + ] + }, + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap (Sum acc) · shared Scan, TimeRange", + "id": "L6" + }, + { + "dag": { + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + }, + "result_kind": "range_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 2 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 3, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 4 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "TimeRange": { + "child": 6, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + }, + "result_kind": "range_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 7, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 8 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 9, + "family": { + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 10 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Project": { + "child": 11, + "cols": [ + { + "alias": "job", + "expr": { + "Column": 0 + } + }, + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk_10", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + } + ], + "roots": [ + { + "Operator": 5 + }, + { + "Operator": 12 + } + ] + }, + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap (Sum acc)", + "id": "L7" + }, + { + "dag": { + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + }, + "result_kind": "range_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 2 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 3, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 4 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + }, + "result_kind": "range_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 6, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 7 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 8, + "family": { + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 9 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Project": { + "child": 10, + "cols": [ + { + "alias": "job", + "expr": { + "Column": 0 + } + }, + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk_10", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + } + ], + "roots": [ + { + "Operator": 5 + }, + { + "Operator": 11 + } + ] + }, + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap (Sum acc) · shared Scan", + "id": "L8" + }, + { + "dag": { + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + }, + "result_kind": "range_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 2 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 3, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 4 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 6 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryAgg": { + "child": 7, + "family": { + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + }, + "result_kind": "state", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "ASAP": { + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 8 + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + }, + { + "guarantee": null, + "materialization": "NotMaterialized", + "operator": { + "NonASAP": { + "Project": { + "child": 9, + "cols": [ + { + "alias": "job", + "expr": { + "Column": 0 + } + }, + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } + } + }, + "result_kind": "instant_vector", + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk_10", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "timing": null + } + ], + "roots": [ + { + "Operator": 5 + }, + { + "Operator": 10 + } + ] + }, + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap (Sum acc) · shared Scan, TimeRange", + "id": "L9" + } + ], + "capped": false, + "combinations": 9 + }, + "stage2_physical_asap_optimization": { + "candidates": [ + { + "dag": { + "edges": [ + { + "consumer": 1, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 0, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 2, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 3, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 2, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 4, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 3, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 5, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 4, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 7, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 6, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 8, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 7, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 9, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 8, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 10, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 9, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 11, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 10, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 12, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 11, + "role": "Input", + "window": "NotApplicable" + } + ], + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 0, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 1, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 2, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 3, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 2 + } + } + } + }, + { + "guarantee": null, + "id": 4, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 3, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 5, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 4 + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 6, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 7, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 6, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 8, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 7, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 9, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 8 + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 10, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Sort": { + "child": 9, + "keys": [ + { + "ascending": false, + "expr": { + "Column": 1 + }, + "nulls_first": false + } + ], + "partition_by": [ + 2 + ] + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 11, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Limit": { + "child": 10, + "n": 10, + "offset": 0, + "partition_by": [ + 2 + ] + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 12, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "topk_10", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 11, + "cols": [ + { + "alias": "job", + "expr": { + "Column": 2 + } + }, + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } + } + } + } + ], + "roots": [ + 5, + 12 + ], + "scalar_roots": [] + }, + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 exact (Sum acc)", + "from_logical": "L1", + "id": "P1" + }, + { + "dag": { + "edges": [ + { + "consumer": 1, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 0, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 2, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 3, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 2, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 4, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 3, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 5, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 4, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 6, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 0, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 7, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 6, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 8, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 7, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 9, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 8, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 10, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 9, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 11, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 10, + "role": "Input", + "window": "NotApplicable" + } + ], + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 0, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 1, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 2, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 3, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 2 + } + } + } + }, + { + "guarantee": null, + "id": 4, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 3, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 5, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 4 + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 6, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 7, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 6, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 8, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 7 + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 9, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Sort": { + "child": 8, + "keys": [ + { + "ascending": false, + "expr": { + "Column": 1 + }, + "nulls_first": false + } + ], + "partition_by": [ + 2 + ] + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 10, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Limit": { + "child": 9, + "n": 10, + "offset": 0, + "partition_by": [ + 2 + ] + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 11, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "topk_10", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 10, + "cols": [ + { + "alias": "job", + "expr": { + "Column": 2 + } + }, + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } + } + } + } + ], + "roots": [ + 5, + 11 + ], + "scalar_roots": [] + }, + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 exact (Sum acc) · shared Scan", + "from_logical": "L2", + "id": "P2" + }, + { + "dag": { + "edges": [ + { + "consumer": 1, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 0, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 2, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 3, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 2, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 4, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 3, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 5, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 4, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 6, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 7, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 6, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 8, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 7, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 9, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 8, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 10, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 9, + "role": "Input", + "window": "NotApplicable" + } + ], + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 0, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 1, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 2, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 3, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 2 + } + } + } + }, + { + "guarantee": null, + "id": 4, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 3, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 5, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 4 + } + } + } + }, + { + "guarantee": null, + "id": 6, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 7, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 6 + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 8, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Sort": { + "child": 7, + "keys": [ + { + "ascending": false, + "expr": { + "Column": 1 + }, + "nulls_first": false + } + ], + "partition_by": [ + 2 + ] + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 9, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Limit": { + "child": 8, + "n": 10, + "offset": 0, + "partition_by": [ + 2 + ] + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 10, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "topk_10", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 9, + "cols": [ + { + "alias": "job", + "expr": { + "Column": 2 + } + }, + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } + } + } + } + ], + "roots": [ + 5, + 10 + ], + "scalar_roots": [] + }, + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 exact (Sum acc) · shared Scan, TimeRange", + "from_logical": "L3", + "id": "P3" + }, + { + "dag": { + "edges": [ + { + "consumer": 1, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 0, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 2, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 3, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 2, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 4, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 3, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 5, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 4, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 7, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 6, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 8, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 7, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 9, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 8, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 10, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 9, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 11, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 10, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 12, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 11, + "role": "Input", + "window": "NotApplicable" + } + ], + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 0, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 1, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 2, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 3, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 2 + } + } + } + }, + { + "guarantee": null, + "id": 4, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 3, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 5, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 4 + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 6, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 7, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 6, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 8, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 7, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 9, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 8 + } + } + } + }, + { + "guarantee": null, + "id": 10, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 9, + "family": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + } + }, + { + "guarantee": null, + "id": 11, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 10 + } + } + } + }, + { + "guarantee": null, + "id": 12, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk_10", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 11, + "cols": [ + { + "alias": "job", + "expr": { + "Column": 0 + } + }, + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } + } + } + } + ], + "roots": [ + 5, + 12 + ], + "scalar_roots": [] + }, + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap (Sum acc)", + "from_logical": "L4", + "id": "P4" + }, + { + "dag": { + "edges": [ + { + "consumer": 1, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 0, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 2, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 3, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 2, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 4, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 3, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 5, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 4, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 6, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 0, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 7, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 6, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 8, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 7, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 9, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 8, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 10, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 9, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 11, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 10, + "role": "Input", + "window": "NotApplicable" + } + ], + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 0, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 1, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 2, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 3, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 2 + } + } + } + }, + { + "guarantee": null, + "id": 4, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 3, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 5, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 4 + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 6, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 7, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 6, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 8, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 7 + } + } + } + }, + { + "guarantee": null, + "id": 9, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 8, + "family": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + } + }, + { + "guarantee": null, + "id": 10, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 9 + } + } + } + }, + { + "guarantee": null, + "id": 11, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk_10", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 10, + "cols": [ + { + "alias": "job", + "expr": { + "Column": 0 + } + }, + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } + } + } + } + ], + "roots": [ + 5, + 11 + ], + "scalar_roots": [] + }, + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap (Sum acc) · shared Scan", + "from_logical": "L5", + "id": "P5" + }, + { + "dag": { + "edges": [ + { + "consumer": 1, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 0, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 2, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 3, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 2, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 4, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 3, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 5, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 4, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 6, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 1, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 7, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 6, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 8, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "producer": 7, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 9, + "data_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 8, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 10, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 9, + "role": "Input", + "window": "NotApplicable" + } + ], + "nodes": [ + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 0, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 1, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } + } + } + } + }, + { + "guarantee": null, + "id": 2, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 3, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 2 + } + } + } + }, + { + "guarantee": null, + "id": 4, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 3, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 5, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 4 + } + } + } + }, + { + "guarantee": null, + "id": 6, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 1, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact accumulator over exact input" + } + ] + }, + "id": 7, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 6 + } + } + } + }, + { + "guarantee": null, + "id": 8, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 7, + "family": { + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 2 + ] + } + } + } + } + }, + { + "guarantee": null, + "id": 9, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 8 + } + } + } + }, + { + "guarantee": null, + "id": 10, + "materialization": "NotMaterialized", + "output_schema": { "closed": true, "fields": [ { @@ -3327,27 +17588,44 @@ ] ] }, - "timing": null + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "NonASAP": { + "Project": { + "child": 9, + "cols": [ + { + "alias": "job", + "expr": { + "Column": 0 + } + }, + { + "alias": "topk_10", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + } + } + } } ], "roots": [ - { - "Operator": 5 - }, - { - "Operator": 12 - } - ] + 5, + 10 + ], + "scalar_roots": [] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap (Sum acc)", - "id": "L3" - } - ], - "capped": false, - "combinations": 3 - }, - "stage2_physical_asap_optimization": { - "candidates": [ + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap (Sum acc) · shared Scan, TimeRange", + "from_logical": "L6", + "id": "P6" + }, { "dag": { "edges": [ @@ -3395,7 +17673,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "producer": 0, "role": "Input", @@ -3445,7 +17728,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "producer": 1, "role": "Input", @@ -3498,7 +17786,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "producer": 2, "role": "Input", @@ -3548,7 +17841,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "producer": 3, "role": "Input", @@ -3639,7 +17937,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "producer": 6, "role": "Input", @@ -3689,7 +17992,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "producer": 7, "role": "Input", @@ -3742,7 +18050,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "producer": 8, "role": "Input", @@ -3792,7 +18105,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "producer": 9, "role": "Input", @@ -3801,29 +18119,13 @@ { "consumer": 11, "data_state": { - "primitive": "Raw", + "primitive": "SummaryState", "timing": "query_time" }, "grouping": "NotApplicable", "intermediate_schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, { "dtype": { "Plain": "utf8" @@ -3834,15 +18136,32 @@ }, { "dtype": { - "Plain": "utf8" + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } + }, + "PerSubpopulationInstance" + ] }, - "name": "$promql_series_identity", + "name": "state", "nullable": false, "table": null } ], - "time_index": 0, - "unique_keys": [] + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] }, "producer": 10, "role": "Input", @@ -3858,22 +18177,6 @@ "intermediate_schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, { "dtype": { "Plain": "utf8" @@ -3886,13 +18189,17 @@ "dtype": { "Plain": "utf8" }, - "name": "$promql_series_identity", + "name": "topk", "nullable": false, "table": null } ], - "time_index": 0, - "unique_keys": [] + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] }, "producer": 11, "role": "Input", @@ -3955,7 +18262,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "Raw", @@ -4002,7 +18314,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "source": { "TimeSeries": { @@ -4068,7 +18385,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "Raw", @@ -4131,7 +18453,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "SummaryState", @@ -4218,7 +18545,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "Raw", @@ -4412,7 +18744,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "Raw", @@ -4459,7 +18796,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "source": { "TimeSeries": { @@ -4525,7 +18867,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "Raw", @@ -4588,7 +18935,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "SummaryState", @@ -4675,7 +19027,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "Raw", @@ -4690,42 +19047,12 @@ } }, { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact accumulator over exact input" - } - ] - }, + "guarantee": null, "id": 10, "materialization": "NotMaterialized", "output_schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, { "dtype": { "Plain": "utf8" @@ -4736,77 +19063,88 @@ }, { "dtype": { - "Plain": "utf8" + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } + }, + "PerSubpopulationInstance" + ] }, - "name": "$promql_series_identity", + "name": "state", "nullable": false, "table": null } ], - "time_index": 0, - "unique_keys": [] + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] }, "output_state": { - "primitive": "Raw", + "primitive": "SummaryState", "timing": "query_time" }, "payload": { - "NonASAP": { - "Sort": { + "ASAP": { + "SummaryAgg": { "child": 9, - "keys": [ - { - "ascending": false, - "expr": { - "Column": 1 + "family": { + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } }, - "nulls_first": false + "PerSubpopulationInstance" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" } - ], - "partition_by": [ - 2 - ] + }, + "reduction": { + "Reduce": [ + 2 + ] + } } } } }, { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact over exact inputs" - } - ] - }, + "guarantee": null, "id": 11, "materialization": "NotMaterialized", "output_schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, { "dtype": { "Plain": "utf8" @@ -4819,47 +19157,37 @@ "dtype": { "Plain": "utf8" }, - "name": "$promql_series_identity", + "name": "topk", "nullable": false, "table": null } ], - "time_index": 0, - "unique_keys": [] + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] }, "output_state": { "primitive": "Raw", "timing": "query_time" }, "payload": { - "NonASAP": { - "Limit": { - "child": 10, - "n": 10, - "offset": 0, - "partition_by": [ - 2 - ] + "ASAP": { + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 10 } } } }, { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact over exact inputs" - } - ] - }, + "guarantee": null, "id": 12, "materialization": "NotMaterialized", "output_schema": { @@ -4875,7 +19203,7 @@ }, { "dtype": { - "Plain": "float64" + "Plain": "utf8" }, "name": "topk_10", "nullable": false, @@ -4883,7 +19211,11 @@ } ], "time_index": null, - "unique_keys": [] + "unique_keys": [ + [ + 0 + ] + ] }, "output_state": { "primitive": "Raw", @@ -4897,7 +19229,7 @@ { "alias": "job", "expr": { - "Column": 2 + "Column": 0 } }, { @@ -4919,9 +19251,9 @@ ], "scalar_roots": [] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 exact (Sum acc)", - "from_logical": "L1", - "id": "P1" + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap (Sum acc)", + "from_logical": "L7", + "id": "P7" }, { "dag": { @@ -4970,7 +19302,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "producer": 0, "role": "Input", @@ -5020,7 +19357,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "producer": 1, "role": "Input", @@ -5073,7 +19415,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "producer": 2, "role": "Input", @@ -5123,7 +19470,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "producer": 3, "role": "Input", @@ -5171,7 +19523,7 @@ "window": "NotApplicable" }, { - "consumer": 7, + "consumer": 6, "data_state": { "primitive": "Raw", "timing": "query_time" @@ -5214,14 +19566,19 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, - "producer": 6, + "producer": 0, "role": "Input", "window": "NotApplicable" }, { - "consumer": 8, + "consumer": 7, "data_state": { "primitive": "Raw", "timing": "query_time" @@ -5264,14 +19621,19 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, - "producer": 7, + "producer": 6, "role": "Input", "window": "NotApplicable" }, { - "consumer": 9, + "consumer": 8, "data_state": { "primitive": "SummaryState", "timing": "query_time" @@ -5317,14 +19679,19 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, - "producer": 8, + "producer": 7, "role": "Input", "window": "NotApplicable" }, { - "consumer": 10, + "consumer": 9, "data_state": { "primitive": "Raw", "timing": "query_time" @@ -5367,14 +19734,19 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, - "producer": 9, + "producer": 8, "role": "Input", "window": "NotApplicable" }, { - "consumer": 11, + "consumer": 10, "data_state": { "primitive": "SummaryState", "timing": "query_time" @@ -5395,13 +19767,13 @@ "dtype": { "Sketch": [ { - "algorithm": "CmsWithHeap", + "algorithm": "CountSketchWithHeap", "category": "TopK", "params": { - "CmsWithHeap": { - "depth": 7, + "CountSketchWithHeap": { + "depth": 125, "heap_size": 100, - "width": 272 + "width": 30000 } } }, @@ -5420,12 +19792,12 @@ ] ] }, - "producer": 10, + "producer": 9, "role": "Input", "window": "NotApplicable" }, { - "consumer": 12, + "consumer": 11, "data_state": { "primitive": "Raw", "timing": "query_time" @@ -5458,7 +19830,7 @@ ] ] }, - "producer": 11, + "producer": 10, "role": "Input", "window": "NotApplicable" } @@ -5519,7 +19891,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "Raw", @@ -5566,7 +19943,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "source": { "TimeSeries": { @@ -5632,7 +20014,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "Raw", @@ -5695,7 +20082,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "SummaryState", @@ -5782,7 +20174,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "Raw", @@ -5976,121 +20373,13 @@ } ], "time_index": 0, - "unique_keys": [] - }, - "output_state": { - "primitive": "Raw", - "timing": "query_time" - }, - "payload": { - "NonASAP": { - "Scan": { - "predicates": [], - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "source": { - "TimeSeries": { - "metric": "http_requests_total" - } - } - } - } - } - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact over exact inputs" - } + "unique_keys": [ + [ + 3, + 0 + ] ] }, - "id": 7, - "materialization": "NotMaterialized", - "output_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, "output_state": { "primitive": "Raw", "timing": "query_time" @@ -6098,7 +20387,7 @@ "payload": { "NonASAP": { "TimeRange": { - "child": 6, + "child": 0, "kind": "range", "range": { "nanos": 0, @@ -6110,7 +20399,7 @@ }, { "guarantee": null, - "id": 8, + "id": 7, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -6152,7 +20441,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "SummaryState", @@ -6161,7 +20455,7 @@ "payload": { "ASAP": { "SummaryAgg": { - "child": 7, + "child": 6, "family": { "ExactAggregate": [ "Sum", @@ -6200,7 +20494,7 @@ } ] }, - "id": 9, + "id": 8, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -6239,7 +20533,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "Raw", @@ -6248,14 +20547,14 @@ "payload": { "ASAP": { "FinalizeExactAccumulator": { - "child": 8 + "child": 7 } } } }, { "guarantee": null, - "id": 10, + "id": 9, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -6272,13 +20571,13 @@ "dtype": { "Sketch": [ { - "algorithm": "CmsWithHeap", + "algorithm": "CountSketchWithHeap", "category": "TopK", "params": { - "CmsWithHeap": { - "depth": 7, + "CountSketchWithHeap": { + "depth": 125, "heap_size": 100, - "width": 272 + "width": 30000 } } }, @@ -6304,17 +20603,17 @@ "payload": { "ASAP": { "SummaryAgg": { - "child": 9, + "child": 8, "family": { "Sketch": [ { - "algorithm": "CmsWithHeap", + "algorithm": "CountSketchWithHeap", "category": "TopK", "params": { - "CmsWithHeap": { - "depth": 7, + "CountSketchWithHeap": { + "depth": 125, "heap_size": 100, - "width": 272 + "width": 30000 } } }, @@ -6347,7 +20646,7 @@ }, { "guarantee": null, - "id": 11, + "id": 10, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -6388,14 +20687,14 @@ "k": 10 } }, - "summary_input": 10 + "summary_input": 9 } } } }, { "guarantee": null, - "id": 12, + "id": 11, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -6431,7 +20730,7 @@ "payload": { "NonASAP": { "Project": { - "child": 11, + "child": 10, "cols": [ { "alias": "job", @@ -6454,13 +20753,13 @@ ], "roots": [ 5, - 12 + 11 ], "scalar_roots": [] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap (Sum acc)", - "from_logical": "L2", - "id": "P2" + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap (Sum acc) · shared Scan", + "from_logical": "L8", + "id": "P8" }, { "dag": { @@ -6509,7 +20808,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "producer": 0, "role": "Input", @@ -6559,7 +20863,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "producer": 1, "role": "Input", @@ -6612,7 +20921,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "producer": 2, "role": "Input", @@ -6662,7 +20976,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "producer": 3, "role": "Input", @@ -6710,57 +21029,7 @@ "window": "NotApplicable" }, { - "consumer": 7, - "data_state": { - "primitive": "Raw", - "timing": "query_time" - }, - "grouping": "NotApplicable", - "intermediate_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "producer": 6, - "role": "Input", - "window": "NotApplicable" - }, - { - "consumer": 8, + "consumer": 6, "data_state": { "primitive": "Raw", "timing": "query_time" @@ -6803,14 +21072,19 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, - "producer": 7, + "producer": 1, "role": "Input", "window": "NotApplicable" }, { - "consumer": 9, + "consumer": 7, "data_state": { "primitive": "SummaryState", "timing": "query_time" @@ -6856,14 +21130,19 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, - "producer": 8, + "producer": 6, "role": "Input", "window": "NotApplicable" }, { - "consumer": 10, + "consumer": 8, "data_state": { "primitive": "Raw", "timing": "query_time" @@ -6906,14 +21185,19 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, - "producer": 9, + "producer": 7, "role": "Input", "window": "NotApplicable" }, { - "consumer": 11, + "consumer": 9, "data_state": { "primitive": "SummaryState", "timing": "query_time" @@ -6959,12 +21243,12 @@ ] ] }, - "producer": 10, + "producer": 8, "role": "Input", "window": "NotApplicable" }, { - "consumer": 12, + "consumer": 10, "data_state": { "primitive": "Raw", "timing": "query_time" @@ -6997,7 +21281,7 @@ ] ] }, - "producer": 11, + "producer": 9, "role": "Input", "window": "NotApplicable" } @@ -7058,7 +21342,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "Raw", @@ -7105,7 +21394,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "source": { "TimeSeries": { @@ -7171,7 +21465,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "Raw", @@ -7234,7 +21533,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "SummaryState", @@ -7321,129 +21625,9 @@ } ], "time_index": 0, - "unique_keys": [] - }, - "output_state": { - "primitive": "Raw", - "timing": "query_time" - }, - "payload": { - "ASAP": { - "FinalizeExactAccumulator": { - "child": 2 - } - } - } - }, - { - "guarantee": null, - "id": 4, - "materialization": "NotMaterialized", - "output_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" - ] - }, - "name": "state", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "output_state": { - "primitive": "SummaryState", - "timing": "query_time" - }, - "payload": { - "ASAP": { - "SummaryAgg": { - "child": 3, - "family": { - "ExactAggregate": [ - "Sum", - "Sum" - ] - }, - "filter": null, - "grouping": "PerSubpopulationInstance", - "input": { - "item": null, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" - } - }, - "reduction": { - "Reduce": [ - 2 - ] - } - } - } - } - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact accumulator over exact input" - } - ] - }, - "id": 5, - "materialization": "NotMaterialized", - "output_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "sum", - "nullable": false, - "table": null - } - ], - "time_index": null, "unique_keys": [ [ + 3, 0 ] ] @@ -7455,48 +21639,18 @@ "payload": { "ASAP": { "FinalizeExactAccumulator": { - "child": 4 + "child": 2 } } } }, { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact over exact inputs" - } - ] - }, - "id": 6, + "guarantee": null, + "id": 4, "materialization": "NotMaterialized", "output_schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, { "dtype": { "Plain": "utf8" @@ -7506,68 +21660,53 @@ "table": null }, { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] - }, - "output_state": { - "primitive": "Raw", - "timing": "query_time" - }, - "payload": { - "NonASAP": { - "Scan": { - "predicates": [], - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [] + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] }, - "source": { - "TimeSeries": { - "metric": "http_requests_total" + "name": "state", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "SummaryState", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "SummaryAgg": { + "child": 3, + "family": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" } + }, + "reduction": { + "Reduce": [ + 2 + ] } } } @@ -7585,31 +21724,15 @@ "provenance": [ { "kind": "exact", - "reason": "exact over exact inputs" + "reason": "exact accumulator over exact input" } ] }, - "id": 7, + "id": 5, "materialization": "NotMaterialized", "output_schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, { "dtype": { "Plain": "utf8" @@ -7620,36 +21743,35 @@ }, { "dtype": { - "Plain": "utf8" + "Plain": "float64" }, - "name": "$promql_series_identity", + "name": "sum", "nullable": false, "table": null } ], - "time_index": 0, - "unique_keys": [] + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] }, "output_state": { "primitive": "Raw", "timing": "query_time" }, "payload": { - "NonASAP": { - "TimeRange": { - "child": 6, - "kind": "range", - "range": { - "nanos": 0, - "secs": 60 - } + "ASAP": { + "FinalizeExactAccumulator": { + "child": 4 } } } }, { "guarantee": null, - "id": 8, + "id": 6, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -7691,7 +21813,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "SummaryState", @@ -7700,7 +21827,7 @@ "payload": { "ASAP": { "SummaryAgg": { - "child": 7, + "child": 1, "family": { "ExactAggregate": [ "Sum", @@ -7739,7 +21866,7 @@ } ] }, - "id": 9, + "id": 7, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -7778,7 +21905,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "Raw", @@ -7787,14 +21919,14 @@ "payload": { "ASAP": { "FinalizeExactAccumulator": { - "child": 8 + "child": 6 } } } }, { "guarantee": null, - "id": 10, + "id": 8, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -7843,7 +21975,7 @@ "payload": { "ASAP": { "SummaryAgg": { - "child": 9, + "child": 7, "family": { "Sketch": [ { @@ -7886,7 +22018,7 @@ }, { "guarantee": null, - "id": 11, + "id": 9, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -7927,14 +22059,14 @@ "k": 10 } }, - "summary_input": 10 + "summary_input": 8 } } } }, { "guarantee": null, - "id": 12, + "id": 10, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -7970,7 +22102,7 @@ "payload": { "NonASAP": { "Project": { - "child": 11, + "child": 9, "cols": [ { "alias": "job", @@ -7993,13 +22125,13 @@ ], "roots": [ 5, - 12 + 10 ], "scalar_roots": [] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap (Sum acc)", - "from_logical": "L3", - "id": "P3" + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap (Sum acc) · shared Scan, TimeRange", + "from_logical": "L9", + "id": "P9" } ], "pruned": [] @@ -8077,18 +22209,177 @@ "source": "analytical-cost-v1 (illustrative statistics)", "total": 81.05020880000001, "unit": "cpu_ms_per_workload_evaluation" + }, + "P2": { + "per_node": { + "0": { + "cost": 46.4, + "detail": "scan 4000000 samples", + "multiplier": 2 + }, + "1": { + "cost": 4.0, + "detail": "time range 60s: pass 4000000 rows", + "multiplier": 1 + }, + "10": { + "cost": 0.001, + "detail": "limit to 1000 rows", + "multiplier": 1 + }, + "11": { + "cost": 0.001, + "detail": "operator: pass 1000 rows", + "multiplier": 1 + }, + "2": { + "cost": 4.0, + "detail": "build exact Rate accumulator into 1000000 states: 4000000 rows x depth 1", + "multiplier": 1 + }, + "3": { + "cost": 1.0, + "detail": "finalize 1000000 accumulators", + "multiplier": 1 + }, + "4": { + "cost": 1.0, + "detail": "build exact Sum accumulator into 100 states: 1000000 rows x depth 1", + "multiplier": 1 + }, + "5": { + "cost": 0.00009999999999999999, + "detail": "finalize 100 accumulators", + "multiplier": 1 + }, + "6": { + "cost": 4.0, + "detail": "time range 60s: pass 4000000 rows", + "multiplier": 1 + }, + "7": { + "cost": 4.0, + "detail": "build exact Sum accumulator into 1000000 states: 4000000 rows x depth 1", + "multiplier": 1 + }, + "8": { + "cost": 1.0, + "detail": "finalize 1000000 accumulators", + "multiplier": 1 + }, + "9": { + "cost": 14.0, + "detail": "sort 1000000 rows in 100 partitions", + "multiplier": 1 + } + }, + "source": "analytical-cost-v1 (illustrative statistics)", + "total": 79.40210000000002, + "unit": "cpu_ms_per_workload_evaluation" + }, + "P3": { + "per_node": { + "0": { + "cost": 46.4, + "detail": "scan 4000000 samples", + "multiplier": 2 + }, + "1": { + "cost": 8.0, + "detail": "time range 60s: pass 4000000 rows", + "multiplier": 2 + }, + "10": { + "cost": 0.001, + "detail": "operator: pass 1000 rows", + "multiplier": 1 + }, + "2": { + "cost": 4.0, + "detail": "build exact Rate accumulator into 1000000 states: 4000000 rows x depth 1", + "multiplier": 1 + }, + "3": { + "cost": 1.0, + "detail": "finalize 1000000 accumulators", + "multiplier": 1 + }, + "4": { + "cost": 1.0, + "detail": "build exact Sum accumulator into 100 states: 1000000 rows x depth 1", + "multiplier": 1 + }, + "5": { + "cost": 0.00009999999999999999, + "detail": "finalize 100 accumulators", + "multiplier": 1 + }, + "6": { + "cost": 4.0, + "detail": "build exact Sum accumulator into 1000000 states: 4000000 rows x depth 1", + "multiplier": 1 + }, + "7": { + "cost": 1.0, + "detail": "finalize 1000000 accumulators", + "multiplier": 1 + }, + "8": { + "cost": 14.0, + "detail": "sort 1000000 rows in 100 partitions", + "multiplier": 1 + }, + "9": { + "cost": 0.001, + "detail": "limit to 1000 rows", + "multiplier": 1 + } + }, + "source": "analytical-cost-v1 (illustrative statistics)", + "total": 79.40210000000002, + "unit": "cpu_ms_per_workload_evaluation" } }, "rejected": [ { - "id": "P2", + "id": "P4", "reason": "q2: CmsWithHeap needs non-negative update weights, and these are not proven non-negative", "valid": false }, { - "id": "P3", + "id": "P5", + "reason": "q2: CmsWithHeap needs non-negative update weights, and these are not proven non-negative", + "valid": false + }, + { + "id": "P6", + "reason": "q2: CmsWithHeap needs non-negative update weights, and these are not proven non-negative", + "valid": false + }, + { + "id": "P7", + "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target EpsilonDelta { epsilon: 0.01, delta: 0.001 }", + "valid": false + }, + { + "id": "P8", + "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target EpsilonDelta { epsilon: 0.01, delta: 0.001 }", + "valid": false + }, + { + "id": "P9", "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target EpsilonDelta { epsilon: 0.01, delta: 0.001 }", "valid": false + }, + { + "id": "P2", + "reason": "costlier: 79.402 vs 79.402 cpu_ms_per_workload_evaluation", + "valid": true + }, + { + "id": "P3", + "reason": "costlier: 79.402 vs 79.402 cpu_ms_per_workload_evaluation", + "valid": true } ], "selected": "P1" diff --git a/tools/dag-viewer/examples/scalar-queries.json b/tools/dag-viewer/examples/scalar-queries.json index 71fd1adf4..0ba6d84ff 100644 --- a/tools/dag-viewer/examples/scalar-queries.json +++ b/tools/dag-viewer/examples/scalar-queries.json @@ -186,7 +186,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 2, + 0 + ] + ] }, "source": { "TimeSeries": { @@ -226,7 +231,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 2, + 0 + ] + ] }, "timing": null }, @@ -275,7 +285,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 2, + 0 + ] + ] }, "timing": null }, @@ -365,7 +380,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "source": { "TimeSeries": { @@ -413,7 +433,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -470,7 +495,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -837,7 +867,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 2, + 0 + ] + ] }, "source": { "TimeSeries": { @@ -877,7 +912,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 2, + 0 + ] + ] }, "timing": null }, @@ -940,7 +980,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 2, + 0 + ] + ] }, "timing": null }, @@ -1095,7 +1140,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "source": { "TimeSeries": { @@ -1143,7 +1193,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -1214,7 +1269,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "timing": null }, @@ -1507,7 +1567,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 2, + 0 + ] + ] }, "producer": 4, "role": "Input", @@ -1549,7 +1614,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 2, + 0 + ] + ] }, "producer": 5, "role": "Input", @@ -1628,7 +1698,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "producer": 8, "role": "Input", @@ -1678,7 +1753,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "producer": 9, "role": "Input", @@ -2012,7 +2092,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 2, + 0 + ] + ] }, "output_state": { "primitive": "Raw", @@ -2051,7 +2136,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 2, + 0 + ] + ] }, "source": { "TimeSeries": { @@ -2109,7 +2199,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 2, + 0 + ] + ] }, "output_state": { "primitive": "Raw", @@ -2282,7 +2377,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "Raw", @@ -2329,7 +2429,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "source": { "TimeSeries": { @@ -2395,7 +2500,12 @@ } ], "time_index": 0, - "unique_keys": [] + "unique_keys": [ + [ + 3, + 0 + ] + ] }, "output_state": { "primitive": "Raw", From e0893932fed03fade00be36f4c45784eceaf4043 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:36:28 +0000 Subject: [PATCH 03/15] refactor(logical-asap-optimization): move the keyed-additive heap rule into the realization helpers Pass 1's whole-expression heap reuses the legacy search's keyed-additive rule, so the rule, its result types and schema_column_ref move from replacement.rs into realization.rs, which both use. Pure move. --- crates/asap-aware-mapping/src/realization.rs | 166 ++++++++++++++++++- crates/asap-aware-mapping/src/replacement.rs | 161 +----------------- 2 files changed, 168 insertions(+), 159 deletions(-) diff --git a/crates/asap-aware-mapping/src/realization.rs b/crates/asap-aware-mapping/src/realization.rs index 44661ba75..0ab6b942e 100644 --- a/crates/asap-aware-mapping/src/realization.rs +++ b/crates/asap-aware-mapping/src/realization.rs @@ -1,19 +1,24 @@ //! Summary realization helpers shared by Pass 1 (`per_computation_candidates`) //! and the legacy replacement search: which sketch kinds can serve an intent, //! the intent's accuracy target and `(ε, δ)` budget, default sketch sizing, -//! and the column/tuple a summary consumes. +//! the column/tuple a summary consumes, the column a top-k ranks, and the +//! keyed-additive rule for a heap sketch that reads a larger sub-DAG. use asap_types::ir::{ ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr, SchemaDerivationError, }; use asap_types::post_asap::{AccuracyError, ExactOperationSchemaError, ExecutionDataStateError}; -use asap_types::post_asap::{Field, Schema, SketchAlgorithm, SketchParams, SummaryInputExpr}; +use asap_types::post_asap::{ + EntityIdentity, Field, FieldDataType, NonNegativeWeightProof, Schema, SketchAlgorithm, + SketchParams, SummaryInputExpr, SummaryUpdate, WeightDomain, +}; use asap_types::pre_asap::agg_intent::AggIntent; use asap_types::pre_asap::column_resolution::{resolve_column_ref, ResolveError}; use asap_types::pre_asap::expr_ir::ColumnRef; use asap_types::pre_asap::schema::ColumnId; use asap_types::pre_asap::Reduction; use asap_types::types::AccuracyTarget; +use std::rc::Rc; use thiserror::Error; /// Confidence δ assumed when the target carries only an ε @@ -259,3 +264,160 @@ pub enum RealizationError { #[error("exact operator schema derivation failed: {0}")] ExactOperationSchema(#[from] ExactOperationSchemaError), } + +/// The physical input consumed by one summary realization. Most summaries +/// consume the logical aggregate's immediate child and summarize its declared +/// input value. Composite realizations can instead consume a larger +/// logical sub-DAG and bind a different key or value. +pub(crate) struct PhysicalSummaryInput { + pub(crate) child: Rc, + pub(crate) input: SummaryUpdate, +} + +pub(crate) enum PhysicalSummaryInputRuleResult { + NotApplicable, + Realized(PhysicalSummaryInput), + Unsupported(&'static str), +} + +/// Realize the composite heavy-hitter realization for +/// `TopK(Count GROUP BY key)`. The heap sketch consumes the raw keyed stream; +/// it does not consume an independently materialized Count result. +pub(crate) fn realize_keyed_additive_summary_input( + intent: &AggIntent, + family: &FieldDataType, + output_reduction: &Reduction, + child: &Rc, +) -> PhysicalSummaryInputRuleResult { + if !matches!(intent, AggIntent::TopK { .. }) { + return PhysicalSummaryInputRuleResult::NotApplicable; + } + let FieldDataType::Sketch(kind, _) = family else { + return PhysicalSummaryInputRuleResult::NotApplicable; + }; + let heap_algorithm = kind.algorithm(); + if !matches!( + heap_algorithm, + SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap + ) { + return PhysicalSummaryInputRuleResult::NotApplicable; + } + let Some(NonASAPOp::Aggregate { + reduction, + measures, + filters, + having: None, + child: raw_child, + .. + }) = child.non_asap() + else { + return PhysicalSummaryInputRuleResult::NotApplicable; + }; + // The heap sketch consumes every raw row; a filtered measure counts only + // some of them (#466). + if asap_types::ir::non_asap::any_measure_filtered(filters) { + return PhysicalSummaryInputRuleResult::NotApplicable; + } + let counter_input = matches!(measures.as_slice(), [AggIntent::Sum { .. }]) + && matches!(raw_child.non_asap(), Some(NonASAPOp::Aggregate { measures, .. }) + if matches!(measures.as_slice(), [AggIntent::Rate | AggIntent::Increase])); + let weight = match measures.as_slice() { + [AggIntent::Count { .. }] => SummaryInputExpr::Constant(1.0), + [AggIntent::Sum { .. }] if counter_input => { + SummaryInputExpr::Column(ColumnRef::SampleValue) + } + [AggIntent::Sum { col }] => SummaryInputExpr::Column(match col { + None => ColumnRef::SampleValue, + Some(index) => match schema_column_ref(raw_child, *index) { + Some(column) => column, + None => { + return PhysicalSummaryInputRuleResult::Unsupported( + "sum-ranked Top-K value column is outside the raw input schema", + ) + } + }, + }), + _ => return PhysicalSummaryInputRuleResult::NotApplicable, + }; + let weight_domain = match measures.as_slice() { + [AggIntent::Count { .. }] => WeightDomain::NonNegative { + proof: NonNegativeWeightProof::UnitCount, + }, + [AggIntent::Sum { .. }] if counter_input => WeightDomain::NonNegative { + proof: NonNegativeWeightProof::ResetAwareCounterDerivative, + }, + _ => WeightDomain::UnknownOrSigned, + }; + if matches!(heap_algorithm, SketchAlgorithm::CmsWithHeap) + && !matches!(weight_domain, WeightDomain::NonNegative { .. }) + { + return PhysicalSummaryInputRuleResult::Unsupported( + "value-weighted CMS requires non-negative update evidence; use CountSketch for arbitrary values", + ); + } + let subpopulation_columns = match output_reduction { + Reduction::PerEntity => vec![], + Reduction::Reduce(keys) => keys + .iter() + .filter_map(|index| schema_column_ref(child, *index)) + .collect(), + }; + let item = match reduction { + Reduction::PerEntity => SummaryInputExpr::EntityIdentity(EntityIdentity::PromqlLabelSet { + excluding: subpopulation_columns, + }), + Reduction::Reduce(keys) if !keys.is_without() && !keys.is_empty() => { + let Some(columns) = keys + .iter() + .map(|index| schema_column_ref(raw_child, *index)) + .collect::>>() + else { + return PhysicalSummaryInputRuleResult::Unsupported( + "ranked item column is outside the raw input schema", + ); + }; + let item_columns: Vec<_> = columns + .into_iter() + .filter(|column| !subpopulation_columns.contains(column)) + .collect(); + match item_columns.as_slice() { + [] => { + return PhysicalSummaryInputRuleResult::Unsupported( + "subpopulation columns consume the complete ranked item identity", + ) + } + [column] => SummaryInputExpr::Column(column.clone()), + _ => SummaryInputExpr::Tuple( + item_columns + .into_iter() + .map(SummaryInputExpr::Column) + .collect(), + ), + } + } + Reduction::Reduce(_) => { + return PhysicalSummaryInputRuleResult::Unsupported( + "an empty or without grouping does not identify ranked items", + ) + } + }; + PhysicalSummaryInputRuleResult::Realized(PhysicalSummaryInput { + child: Rc::clone(raw_child), + input: SummaryUpdate { + item: Some(item), + weight, + weight_domain, + }, + }) +} + +pub(crate) fn schema_column_ref(child: &OperatorNode, index: usize) -> Option { + let column = child.schema.fields.get(index)?; + Some(match &column.table { + Some(table) => ColumnRef::Qualified { + table: table.clone(), + name: column.name.clone(), + }, + None => ColumnRef::Named(column.name.clone()), + }) +} diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index 547fe44e9..2c556d524 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -343,6 +343,10 @@ use crate::accuracy::estimators::{ cms::{cms_depth, cms_width}, saturating_ceil, }; +use crate::realization::{ + realize_keyed_additive_summary_input, schema_column_ref, PhysicalSummaryInput, + PhysicalSummaryInputRuleResult, +}; use asap_types::ir::non_asap::any_measure_filtered; use asap_types::pre_asap::resolve_column_ref; use std::cell::RefCell; @@ -2668,21 +2672,6 @@ fn summary_family(realization: Realization) -> Option<(FieldDataType, bool)> { }) } -/// The physical input consumed by one summary realization. Most summaries -/// consume the logical aggregate's immediate child and summarize its declared -/// input value. Composite realizations can instead consume a larger -/// logical sub-DAG and bind a different key or value. -struct PhysicalSummaryInput { - child: Rc, - input: SummaryUpdate, -} - -enum PhysicalSummaryInputRuleResult { - NotApplicable, - Realized(PhysicalSummaryInput), - Unsupported(&'static str), -} - type PhysicalSummaryInputRule = fn(&AggIntent, &FieldDataType, &Reduction, &Rc) -> PhysicalSummaryInputRuleResult; @@ -3423,148 +3412,6 @@ fn realize_current_series_summary_input( }) } -/// Realize the composite heavy-hitter realization for -/// `TopK(Count GROUP BY key)`. The heap sketch consumes the raw keyed stream; -/// it does not consume an independently materialized Count result. -fn realize_keyed_additive_summary_input( - intent: &AggIntent, - family: &FieldDataType, - output_reduction: &Reduction, - child: &Rc, -) -> PhysicalSummaryInputRuleResult { - if !matches!(intent, AggIntent::TopK { .. }) { - return PhysicalSummaryInputRuleResult::NotApplicable; - } - let FieldDataType::Sketch(kind, _) = family else { - return PhysicalSummaryInputRuleResult::NotApplicable; - }; - let heap_algorithm = kind.algorithm(); - if !matches!( - heap_algorithm, - SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap - ) { - return PhysicalSummaryInputRuleResult::NotApplicable; - } - let Some(NonASAPOp::Aggregate { - reduction, - measures, - filters, - having: None, - child: raw_child, - .. - }) = child.non_asap() - else { - return PhysicalSummaryInputRuleResult::NotApplicable; - }; - // The heap sketch consumes every raw row; a filtered measure counts only - // some of them (#466). - if asap_types::ir::non_asap::any_measure_filtered(filters) { - return PhysicalSummaryInputRuleResult::NotApplicable; - } - let counter_input = matches!(measures.as_slice(), [AggIntent::Sum { .. }]) - && matches!(raw_child.non_asap(), Some(NonASAPOp::Aggregate { measures, .. }) - if matches!(measures.as_slice(), [AggIntent::Rate | AggIntent::Increase])); - let weight = match measures.as_slice() { - [AggIntent::Count { .. }] => SummaryInputExpr::Constant(1.0), - [AggIntent::Sum { .. }] if counter_input => { - SummaryInputExpr::Column(ColumnRef::SampleValue) - } - [AggIntent::Sum { col }] => SummaryInputExpr::Column(match col { - None => ColumnRef::SampleValue, - Some(index) => match schema_column_ref(raw_child, *index) { - Some(column) => column, - None => { - return PhysicalSummaryInputRuleResult::Unsupported( - "sum-ranked Top-K value column is outside the raw input schema", - ) - } - }, - }), - _ => return PhysicalSummaryInputRuleResult::NotApplicable, - }; - let weight_domain = match measures.as_slice() { - [AggIntent::Count { .. }] => WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, - [AggIntent::Sum { .. }] if counter_input => WeightDomain::NonNegative { - proof: NonNegativeWeightProof::ResetAwareCounterDerivative, - }, - _ => WeightDomain::UnknownOrSigned, - }; - if matches!(heap_algorithm, SketchAlgorithm::CmsWithHeap) - && !matches!(weight_domain, WeightDomain::NonNegative { .. }) - { - return PhysicalSummaryInputRuleResult::Unsupported( - "value-weighted CMS requires non-negative update evidence; use CountSketch for arbitrary values", - ); - } - let subpopulation_columns = match output_reduction { - Reduction::PerEntity => vec![], - Reduction::Reduce(keys) => keys - .iter() - .filter_map(|index| schema_column_ref(child, *index)) - .collect(), - }; - let item = match reduction { - Reduction::PerEntity => SummaryInputExpr::EntityIdentity(EntityIdentity::PromqlLabelSet { - excluding: subpopulation_columns, - }), - Reduction::Reduce(keys) if !keys.is_without() && !keys.is_empty() => { - let Some(columns) = keys - .iter() - .map(|index| schema_column_ref(raw_child, *index)) - .collect::>>() - else { - return PhysicalSummaryInputRuleResult::Unsupported( - "ranked item column is outside the raw input schema", - ); - }; - let item_columns: Vec<_> = columns - .into_iter() - .filter(|column| !subpopulation_columns.contains(column)) - .collect(); - match item_columns.as_slice() { - [] => { - return PhysicalSummaryInputRuleResult::Unsupported( - "subpopulation columns consume the complete ranked item identity", - ) - } - [column] => SummaryInputExpr::Column(column.clone()), - _ => SummaryInputExpr::Tuple( - item_columns - .into_iter() - .map(SummaryInputExpr::Column) - .collect(), - ), - } - } - Reduction::Reduce(_) => { - return PhysicalSummaryInputRuleResult::Unsupported( - "an empty or without grouping does not identify ranked items", - ) - } - }; - PhysicalSummaryInputRuleResult::Realized(PhysicalSummaryInput { - child: Rc::clone(raw_child), - input: SummaryUpdate { - item: Some(item), - weight, - weight_domain, - }, - }) -} - -fn schema_column_ref(child: &OperatorNode, index: usize) -> Option { - let column = child.schema.fields.get(index)?; - Some(match &column.table { - Some(table) => ColumnRef::Qualified { - table: table.clone(), - name: column.name.clone(), - }, - None => ColumnRef::Named(column.name.clone()), - }) -} - /// The guarantee of the value a `family` node produces over `child` — /// [`AccuracyModel::propagate`] under the [`CompositionOperator`] this family /// applies to its child's values — checked against `intent`'s own From 7761c0a5cb5a6393613fb797a2870761a6b75934 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:21:37 +0000 Subject: [PATCH 04/15] feat(logical-asap-optimization): whole-expression top-k heaps and counter-sample weights Pass 1 gains a whole-expression candidate kind. A top-k over a per-item sum or count that is itself a target (one consumer, no filter or HAVING, realizable by the keyed-additive rule) has its Count-Min and CountSketch heap candidates replaced by whole-expression ones: one heap over the inner aggregate's input, keyed by the ranked item and weighted by the summed value. It absorbs the inner target, which is then not computed. The outer top-k's partition keys are moved onto the input the heap reads. Example 1's Q2 gets exactly three candidates: exact, Count-Min + heap and CountSketch + heap. Enumeration skips choices that differ only in an absorbed target, and PerComputationCandidates::combinations counts the same set. Count-Min needs non-negative weights. Pass 1 does not read the data workload: when a heap's weights are raw samples of metric M, or plain sums of them, it records the open requirement WeightDomain::SamplesOfMetric{M}. Rate, other arithmetic and other functions end it, and no type is inferred from a `_total` name. DataWorkload gains metric_types (declared Prometheus types). Plan selection admits Count-Min over such weights only when M is declared a counter; an unconfirmed requirement counts as unmet. The planner pipeline's selected plan carries the confirmed requirement as the proof NonNegativeWeightProof::CounterSamples, since the executor reads no data workload. Example 1 declares http_requests_total a counter, so its Count-Min heap is now rejected only because a top-k output has no guarantee; without the declaration it is rejected on its weights. Raw-sample precompute now checks the whole-expression heaps. The Example 1 viewer fixture is regenerated. --- .../src/pass/stage1_to_stage3.rs | 71 +- .../src/per_computation_candidates.rs | 551 ++- .../src/stage3_plan_selection.rs | 176 +- .../tests/weighted_topk_binding.rs | 48 +- .../devtools/src/planner_layering_examples.rs | 12 +- .../tests/planner_layering_example1.rs | 46 +- .../tests/planner_layering_example2.rs | 1 + .../tests/precompute_raw_samples.rs | 34 +- .../tests/promql_numeric_regressions.rs | 2 - .../tests/stage1_logical_asap_optimization.rs | 8 +- crates/types/src/post_asap/sketch.rs | 34 + crates/types/src/workload.rs | 17 + .../examples/planner-layering-example1.json | 4346 ++++------------- 13 files changed, 1857 insertions(+), 3489 deletions(-) diff --git a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs index e1ec2141e..054b46380 100644 --- a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs +++ b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs @@ -197,6 +197,12 @@ pub fn run_stage1_to_stage3_from_pass1( }; let mut choice = vec![0; per_computation.targets.len()]; loop { + if per_computation.absorbed_choice_varies(&choice) { + if next_choice(&per_computation, &mut choice) { + continue; + } + break; + } let display_name = display_name(&per_computation, &query_of_target, &choice); let roots = compose_logical_candidate(&per_computation, &choice) .map(|roots| roots.into_iter().map(|(_, root)| root).collect::>()); @@ -226,18 +232,7 @@ pub fn run_stage1_to_stage3_from_pass1( shared, }); } - // Mixed-radix increment: the last target varies fastest; a carry - // out of the first target ends the enumeration. - let mut carry = true; - for (digit, target) in choice.iter_mut().zip(&per_computation.targets).rev() { - *digit += 1; - if *digit < target.choices.candidates.len() { - carry = false; - break; - } - *digit = 0; - } - if carry { + if !next_choice(&per_computation, &mut choice) { break; } } @@ -253,6 +248,19 @@ pub fn run_stage1_to_stage3_from_pass1( } } +/// Mixed-radix increment of `choice`: the last target varies fastest. +/// `false` when a carry out of the first target ends the enumeration. +fn next_choice(inventory: &PerComputationCandidates, choice: &mut [usize]) -> bool { + for (digit, target) in choice.iter_mut().zip(&inventory.targets).rev() { + *digit += 1; + if *digit < target.choices.candidates.len() { + return true; + } + *digit = 0; + } + false +} + /// `query_of_target[t]`: the index of the query root that first reads target /// `t` (the first root, in input order, from which the target is reachable). /// [`display_name`] lists each target's choice under that query. @@ -291,16 +299,30 @@ fn display_name( query_of_target: &[usize], choice: &[usize], ) -> String { + // Targets a chosen whole-expression candidate absorbs are not computed. + let absorbed: Vec = inventory + .targets + .iter() + .zip(choice) + .filter(|(target, &index)| { + matches!( + target.choices.candidates[index], + ComputationCandidate::WholeExpression(_) + ) + }) + .filter_map(|(target, _)| target.absorbs) + .collect(); (0..inventory.roots.len()) .map(|query| { let mut sketches = Vec::new(); let mut accumulators = Vec::new(); - for (target, (&target_query, &index)) in inventory + for (t, (target, (&target_query, &index))) in inventory .targets .iter() .zip(query_of_target.iter().zip(choice)) + .enumerate() { - if target_query != query { + if target_query != query || absorbed.contains(&t) { continue; } match &target.choices.candidates[index] { @@ -311,11 +333,14 @@ fn display_name( ComputationCandidate::AvgAsSumCount => { accumulators.push("Sum/Count acc".into()) } - ComputationCandidate::Summary(algorithm) => sketches.push(match algorithm { - SketchAlgorithm::CmsWithHeap => "CMS+heap".to_string(), - SketchAlgorithm::CountSketchWithHeap => "CountSketch+heap".to_string(), - other => format!("{other:?}"), - }), + ComputationCandidate::Summary(algorithm) + | ComputationCandidate::WholeExpression(algorithm) => { + sketches.push(match algorithm { + SketchAlgorithm::CmsWithHeap => "CMS+heap".to_string(), + SketchAlgorithm::CountSketchWithHeap => "CountSketch+heap".to_string(), + other => format!("{other:?}"), + }) + } } } let mut text = format!("Q{} ", query + 1); @@ -338,7 +363,8 @@ fn display_name( /// identity as a column (the row representation per-series state needs at /// runtime; a root with no such representation is planned over its labels /// alone), through [`run_stage1_to_stage3`] with every entry's accuracy target. The -/// output is the selected physical candidate's roots. +/// output is the selected physical candidate's roots, with the open weight +/// requirements plan selection confirmed made proofs. #[derive(Debug, Default, Clone, Copy)] pub struct Stage1ToStage3Pass; @@ -386,8 +412,11 @@ impl OptimizationPass for Stage1ToStage3Pass { .into_iter() .find(|p| p.id == selection.selected) .expect("the selected candidate is a physical candidate"); + // The executor reads no data workload: answer every open weight + // requirement plan selection confirmed. + let roots = crate::stage3_plan_selection::with_confirmed_weights(&selected.roots, &data); let mut output = PlanOutput::new(Vec::new()); - for (entry_index, root) in selected.roots.iter().enumerate() { + for (entry_index, root) in roots.iter().enumerate() { match root { QueryRoot::Operator(node) => output.plans.push(QueryPlan { entry_index, diff --git a/crates/asap-aware-mapping/src/per_computation_candidates.rs b/crates/asap-aware-mapping/src/per_computation_candidates.rs index 7e223f9ed..4fb8a9083 100644 --- a/crates/asap-aware-mapping/src/per_computation_candidates.rs +++ b/crates/asap-aware-mapping/src/per_computation_candidates.rs @@ -121,6 +121,14 @@ pub enum ComputationCandidate { Summary(SketchAlgorithm), /// `avg` rewritten as exact `sum` and `count` accumulator states. AvgAsSumCount, + /// A whole-expression top-k: one heap summary (Count-Min or CountSketch + /// with a heap) over the input of the per-item `sum` or `count` beneath + /// the top-k, keyed by the ranked item and weighted by the summed value. + /// It absorbs the target beneath ([`ComputationTarget::absorbs`]): that + /// aggregate is not computed. Where it applies it replaces the heap of + /// the same family over the exact per-item result. Unsized, like + /// `Summary`. + WholeExpression(SketchAlgorithm), } /// Why Pass 1 proved a candidate unable to realize its target. @@ -207,6 +215,17 @@ pub struct ComputationTarget { /// The aggregate's own `having`, then one entry per consumer `Filter` /// directly above the aggregate. pub having: Vec, + /// The target directly beneath this one that its + /// [`ComputationCandidate::WholeExpression`] candidates absorb. A + /// whole-workload choice of such a candidate takes the absorbed target's + /// first candidate: the target is not computed, so its own choice does + /// not matter. + pub absorbs: Option, + /// The metric whose raw samples, or plain sums of them, are every value + /// the aggregate reads ([`samples_of_metric`]); `None` otherwise. A + /// heap-sketch weight read from them carries the open requirement + /// [`WeightDomain::SamplesOfMetric`], which plan selection confirms. + pub samples_of: Option, } /// Compact Pass 1 inventory; roots and nested producer dependencies are retained. @@ -218,13 +237,49 @@ pub struct PerComputationCandidates { impl PerComputationCandidates { /// The number of whole-workload candidates: the size of the Cartesian - /// product of the targets' candidates, saturating at `usize::MAX`. + /// product of the targets' candidates, without the choices that pick a + /// whole-expression candidate and anything but the first candidate of + /// the target it absorbs (the same candidate as with the first), + /// saturating at `usize::MAX`. An absorbed target has one absorber, and + /// an absorbing target is never absorbed, so each pair counts as + /// `(n − w) · m + w` for an absorber with `n` candidates, `w` of them + /// whole-expression, over an absorbed target with `m`. pub fn combinations(&self) -> usize { + let absorbed: HashSet = self.targets.iter().filter_map(|t| t.absorbs).collect(); self.targets .iter() - .map(|target| target.choices.candidates.len()) + .enumerate() + .filter(|(index, _)| !absorbed.contains(index)) + .map(|(_, target)| { + let n = target.choices.candidates.len(); + let Some(inner) = target.absorbs else { + return n; + }; + let whole = target + .choices + .candidates + .iter() + .filter(|c| matches!(c, ComputationCandidate::WholeExpression(_))) + .count(); + (n - whole) + .saturating_mul(self.targets[inner].choices.candidates.len()) + .saturating_add(whole) + }) .fold(1usize, usize::saturating_mul) } + + /// Whether `choice` picks a whole-expression candidate and a candidate + /// other than the first for the target it absorbs: the same + /// whole-workload candidate as the one with the first, so enumeration + /// skips it. + pub fn absorbed_choice_varies(&self, choice: &[usize]) -> bool { + self.targets.iter().zip(choice).any(|(target, &index)| { + matches!( + target.choices.candidates[index], + ComputationCandidate::WholeExpression(_) + ) && target.absorbs.is_some_and(|inner| choice[inner] != 0) + }) + } } #[derive(Debug, Error)] @@ -460,12 +515,200 @@ pub fn generate_per_computation_candidates( choices, filter: filters.get(measure).cloned().flatten(), having: entries.clone(), + absorbs: None, + samples_of: node.non_asap().and_then(|op| match op { + NonASAPOp::Aggregate { child, .. } => samples_of_metric(child), + _ => None, + }), }); } } + add_whole_expression_candidates(&mut targets, &consumers); Ok(PerComputationCandidates { roots, targets }) } +/// Whole-expression top-k (#509 Example 1's Q2): a top-k over a per-item +/// `sum` or `count` that is itself a target has its heap candidates +/// (Count-Min and CountSketch with a heap) replaced by +/// [`ComputationCandidate::WholeExpression`] ones of the same families, +/// which absorb the inner target. Applied only when the inner aggregate has +/// no other consumer, so absorbing it removes its work, when neither has a +/// filter or `HAVING`, and when the keyed-additive rule realizes it +/// ([`whole_expression_input`]). +fn add_whole_expression_candidates( + targets: &mut [ComputationTarget], + consumers: &HashMap<*const OperatorNode, Vec>>>, +) { + for t in 0..targets.len() { + let target = &targets[t]; + let Some(NonASAPOp::Aggregate { child, .. }) = target.target.non_asap() else { + continue; + }; + let Some(inner) = targets + .iter() + .position(|other| Rc::ptr_eq(&other.target, child)) + else { + continue; + }; + if consumers.get(&Rc::as_ptr(child)).map(Vec::len) != Some(1) + || target.filter.is_some() + || !target.having.is_empty() + || targets[inner].filter.is_some() + || !targets[inner].having.is_empty() + || whole_expression_input(&target.target).is_none() + { + continue; + } + let target = &mut targets[t]; + let mut replaced = false; + for candidate in &mut target.choices.candidates { + if let ComputationCandidate::Summary( + algorithm @ (SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap), + ) = candidate + { + *candidate = ComputationCandidate::WholeExpression(algorithm.clone()); + replaced = true; + } + } + if replaced { + target.absorbs = Some(inner); + } + } +} + +/// The input, update and partitions of a whole-expression top-k over +/// `target`'s inner aggregate, by the keyed-additive rule, or `None` when it +/// does not apply. Rows that carry the full series identity rank it as a +/// column, as [`summary_update`] does. The partitions are `target`'s, moved +/// onto the input the heap reads ([`partitions_over`]). +fn whole_expression_input( + target: &OperatorNode, +) -> Option<(Rc, SummaryUpdate, Reduction)> { + use crate::realization::{ + realize_keyed_additive_summary_input, PhysicalSummaryInputRuleResult, + }; + let Some(NonASAPOp::Aggregate { + child, + reduction, + measures, + filters, + having: None, + .. + }) = target.non_asap() + else { + return None; + }; + let [intent @ AggIntent::TopK { .. }] = measures.as_slice() else { + return None; + }; + if asap_types::ir::non_asap::any_measure_filtered(filters) { + return None; + } + // The rule reads only the family's algorithm, and rejects Count-Min over + // weights it cannot prove non-negative. Ask with CountSketch so one + // update serves both heap families; plan selection decides whether + // Count-Min is admissible, as for every other Count-Min candidate. + let family = FieldDataType::Sketch( + SketchKind::new( + SketchAlgorithm::CountSketchWithHeap, + asap_types::post_asap::SketchParams::CountSketchWithHeap { + width: 1, + depth: 1, + heap_size: 1, + }, + ), + GroupingStrategy::default(), + ); + let PhysicalSummaryInputRuleResult::Realized(realized) = + realize_keyed_additive_summary_input(intent, &family, reduction, child) + else { + return None; + }; + let mut input = realized.input; + let per_series = matches!( + child.non_asap(), + Some(NonASAPOp::Aggregate { + reduction: Reduction::PerEntity, + .. + }) + ); + if per_series && realized.child.schema.has_promql_series_identity() { + input.item = Some(SummaryInputExpr::Column(ColumnRef::Named( + asap_types::pre_asap::schema::PROMQL_SERIES_IDENTITY.into(), + ))); + } else if realized.child.schema.closed { + // An encoded label set needs an open PromQL schema. + return None; + } + let partitions = partitions_over(reduction, &child.schema, &realized.child.schema)?; + Some((realized.child, input, partitions)) +} + +/// `reduction`'s keys, which index `inner` (the absorbed aggregate's +/// output), as indices into `input` (the rows the heap reads), matched by +/// column. `None` when a key is missing or ambiguous in `input`, or the keys +/// are given by `without`: `input` carries columns `inner` dropped. +fn partitions_over(reduction: &Reduction, inner: &Schema, input: &Schema) -> Option { + let Reduction::Reduce(keys) = reduction else { + return Some(Reduction::PerEntity); + }; + if keys.is_without() { + return None; + } + let column = + |schema: &Schema, index: usize| crate::realization::column_ref(&schema.fields[index]); + let keys = keys + .keys() + .iter() + .map(|&index| { + inner.fields.get(index)?; + let key = column(inner, index); + let mut matches = (0..input.fields.len()).filter(|&i| column(input, i) == key); + match (matches.next(), matches.next()) { + (Some(i), None) => Some(i), + _ => None, + } + }) + .collect::>()?; + Some(Reduction::by(keys)) +} + +/// The metric whose raw samples, or plain sums of them, are every value +/// `node` outputs, or `None`. Only a time-series scan, a time range over it +/// and a plain `sum` keep that; any other operator (rate and arithmetic +/// included) ends it. The metric's type is not read here: whether its +/// samples are non-negative is plan selection's question +/// ([`WeightDomain::SamplesOfMetric`]). +fn samples_of_metric(node: &OperatorNode) -> Option { + match node.non_asap()? { + NonASAPOp::Scan { + source: Source::TimeSeries { metric }, + .. + } => Some(metric.clone()), + NonASAPOp::TimeRange { child, .. } => samples_of_metric(child), + NonASAPOp::Aggregate { + measures, child, .. + } if matches!(measures.as_slice(), [AggIntent::Sum { col: None }]) => { + samples_of_metric(child) + } + _ => None, + } +} + +/// `update` with the open requirement that its weights are samples of +/// `metric` ([`WeightDomain::SamplesOfMetric`]), when it is a keyed heap +/// update whose weights are not otherwise proven. +fn with_samples_requirement(mut update: SummaryUpdate, metric: Option<&str>) -> SummaryUpdate { + if let Some(metric) = metric { + if update.item.is_some() && update.weight_domain == WeightDomain::UnknownOrSigned { + update.weight_domain = WeightDomain::SamplesOfMetric { + metric: metric.to_string(), + }; + } + } + update +} + /// Build one whole-workload candidate (end of Pass 1): `choice[i]` indexes /// `inventory.targets[i].choices.candidates`. An aggregate with a chosen /// candidate other than `Exact` is replaced measure by measure: an exact @@ -1438,12 +1681,18 @@ fn substitute( .iter() .enumerate() .map(|(measure, intent)| { - let candidate = targets + let (candidate, samples_of) = targets .iter() .find(|(target, _)| target.measure == measure) - .map_or(&ComputationCandidate::Exact, |(_, candidate)| *candidate); + .map_or( + (&ComputationCandidate::Exact, None), + |(target, candidate)| (*candidate, target.samples_of.as_deref()), + ); let filter = filters.get(measure).and_then(Option::as_ref).map(rewritten); - substitute_measure(target, intent, candidate, filter, &input) + if let ComputationCandidate::WholeExpression(algorithm) = candidate { + return whole_expression(target, intent, algorithm, samples_of, memo); + } + substitute_measure(target, intent, candidate, filter, &input, samples_of) }) .collect::, _>>()?; let mut output = match <[_; 1]>::try_from(parts) { @@ -1500,6 +1749,35 @@ fn with_names_of( ))?) } +/// A whole-expression top-k for `target` ([`whole_expression_input`]): one +/// heap summary of `algorithm` over the absorbed aggregate's rewritten +/// input, partitioned as `target`. +fn whole_expression( + target: &OperatorNode, + intent: &AggIntent, + algorithm: &SketchAlgorithm, + samples_of: Option<&str>, + memo: &Memo, +) -> Result, PerComputationCandidateError> { + let (input, update, partitions) = whole_expression_input(target).ok_or( + PerComputationCandidateError::Unsupported("whole-expression top-k input"), + )?; + let state = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { + child: memo[&Rc::as_ptr(&input)].clone(), + input: with_samples_requirement(update, samples_of), + family: FieldDataType::Sketch(sized(algorithm, intent), GroupingStrategy::default()), + reduction: partitions, + grouping: GroupingStrategy::default(), + filter: None, + }))?; + Ok(OperatorNode::new_shared(Operator::ASAP( + ASAPOp::SummaryEstimate { + summary_input: state, + query: statistic(intent)?, + }, + ))?) +} + /// One measure of `target` over `input`: its group keys, then its value. fn substitute_measure( target: &OperatorNode, @@ -1507,6 +1785,7 @@ fn substitute_measure( candidate: &ComputationCandidate, filter: Option, input: &Rc, + samples_of: Option<&str>, ) -> Result, PerComputationCandidateError> { let Some(NonASAPOp::Aggregate { reduction, .. }) = target.non_asap() else { unreachable!("substitute checked the aggregate") @@ -1515,7 +1794,10 @@ fn substitute_measure( Ok::<_, PerComputationCandidateError>(OperatorNode::new_shared(Operator::ASAP( ASAPOp::SummaryAgg { child: input.clone(), - input: summary_update(intent, &family, reduction, &input.schema, read)?, + input: with_samples_requirement( + summary_update(intent, &family, reduction, &input.schema, read)?, + samples_of, + ), family, reduction: reduction.clone(), grouping: GroupingStrategy::default(), @@ -1556,6 +1838,11 @@ fn substitute_measure( query: statistic(intent)?, }))? } + ComputationCandidate::WholeExpression(_) => { + return Err(PerComputationCandidateError::Unsupported( + "whole-expression top-k is substituted with its aggregate", + )) + } ComputationCandidate::AvgAsSumCount => { let AggIntent::Avg { col } = intent else { return Err(PerComputationCandidateError::Unsupported( @@ -2431,4 +2718,256 @@ mod tests { ); } } + + fn inventory(query: &str) -> PerComputationCandidates { + let root = crate::test_support::lower_promql( + query, + AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.001, + }, + ); + // Per-series state ranks each series' identity, which the rows carry + // as a column; a query over aggregated rows is planned over labels. + let root = + asap_types::ir::schema_support::with_promql_series_identity(&root).unwrap_or(root); + generate_per_computation_candidates(vec![(0, QueryRoot::Operator(root))]).unwrap() + } + + /// Every candidate of target `t` composed with every other target at its + /// first candidate, with the candidate. + fn composed_with( + inventory: &PerComputationCandidates, + t: usize, + ) -> Vec<(ComputationCandidate, Rc)> { + (0..inventory.targets[t].choices.candidates.len()) + .map(|c| { + let mut choice = vec![0; inventory.targets.len()]; + choice[t] = c; + let composed = compose_logical_candidate(inventory, &choice).unwrap(); + let QueryRoot::Operator(root) = &composed[0].1 else { + panic!("operator root") + }; + ( + inventory.targets[t].choices.candidates[c].clone(), + root.clone(), + ) + }) + .collect() + } + + fn heap_states(root: &Rc) -> Vec> { + OperatorNode::reachable(root) + .into_iter() + .filter(|node| { + matches!(node.asap(), Some(ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(kind, _), .. + }) if matches!( + kind.algorithm(), + SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap + )) + }) + .collect() + } + + fn topk_target(inventory: &PerComputationCandidates) -> usize { + inventory + .targets + .iter() + .position(|t| { + matches!( + t.target.non_asap(), + Some(NonASAPOp::Aggregate { measures, .. }) + if matches!(measures.as_slice(), [AggIntent::TopK { .. }]) + ) + }) + .expect("a top-k target") + } + + /// Example 1's Q2: the top-k over a per-series `sum_over_time` gets + /// exactly three candidates, exact and a whole-expression Count-Min and + /// CountSketch heap, which replace the heaps over the exact sum. They + /// absorb the inner sum: composed, no aggregate is computed and the heap + /// reads the range selector's raw samples. + #[test] + fn whole_expression_topk_replaces_the_heap_over_the_exact_sum() { + let inventory = inventory("topk by (job) (10, sum_over_time(m[1m]))"); + let topk = topk_target(&inventory); + let target = &inventory.targets[topk]; + assert_eq!( + target.choices.candidates, + [ + ComputationCandidate::Exact, + ComputationCandidate::WholeExpression(SketchAlgorithm::CmsWithHeap), + ComputationCandidate::WholeExpression(SketchAlgorithm::CountSketchWithHeap), + ] + ); + let inner = target.absorbs.expect("absorbs the inner sum"); + assert_eq!(inventory.targets[inner].choices.candidates.len(), 1); + assert_eq!(inventory.combinations(), 3); + for (candidate, root) in composed_with(&inventory, topk) { + if candidate == ComputationCandidate::Exact { + continue; + } + assert!( + !OperatorNode::reachable(&root).iter().any(|n| matches!( + n.non_asap(), + Some(NonASAPOp::Aggregate { .. }) + ) || matches!( + n.asap(), + Some(ASAPOp::FinalizeExactAccumulator { .. }) + )), + "{candidate:?}: the inner sum is not computed" + ); + let [state] = &heap_states(&root)[..] else { + panic!("one heap") + }; + let Some(ASAPOp::SummaryAgg { child, .. }) = state.asap() else { + unreachable!() + }; + assert!(matches!( + child.non_asap(), + Some(NonASAPOp::TimeRange { .. }) + )); + } + } + + /// The whole-expression heap reads the inner sum's input, so the outer + /// top-k's partition keys (indices into the sum's output) are moved onto + /// that input by column. + #[test] + fn whole_expression_topk_partitions_index_the_heap_input() { + let root = crate::test_support::lower_promql( + "topk by (job) (2, sum by (service, job) (rate(m[1m])))", + AccuracyTarget::Epsilon(0.1), + ); + let inventory = + generate_per_computation_candidates(vec![(0, QueryRoot::Operator(root))]).unwrap(); + let topk = topk_target(&inventory); + let mut heaps = 0; + for (candidate, root) in composed_with(&inventory, topk) { + if !matches!(candidate, ComputationCandidate::WholeExpression(_)) { + continue; + } + for state in heap_states(&root) { + let Some(ASAPOp::SummaryAgg { + child, reduction, .. + }) = state.asap() + else { + unreachable!() + }; + let names: Vec<_> = reduction + .group_keys() + .expect("partitioned by job") + .keys() + .iter() + .map(|&index| child.schema.fields[index].name.as_str()) + .collect(); + assert_eq!(names, ["job"]); + heaps += 1; + } + } + assert_eq!( + heaps, 2, + "a whole-expression Count-Min and CountSketch heap" + ); + } + + /// The weight domain of every heap-sketch update in every candidate of + /// `query`'s top-k. + fn heap_weight_domains(query: &str) -> Vec { + let inventory = inventory(query); + let topk = topk_target(&inventory); + composed_with(&inventory, topk) + .iter() + .flat_map(|(_, root)| heap_states(root)) + .map(|state| match state.asap() { + Some(ASAPOp::SummaryAgg { input, .. }) => input.weight_domain.clone(), + _ => unreachable!(), + }) + .collect() + } + + /// Pass 1 does not read the metric's type: heap weights that are raw + /// samples of `m` carry the open requirement that `m` is a counter, under + /// any name (`m_total` included), for plan selection to confirm. + #[test] + fn heap_weights_over_samples_record_an_open_requirement() { + for (query, metric) in [ + ("topk by (job) (10, sum_over_time(m[1m]))", "m"), + ("topk by (job) (10, sum_over_time(m_total[1m]))", "m_total"), + ] { + let domains = heap_weight_domains(query); + assert_eq!(domains.len(), 2, "{query}"); + assert!( + domains.iter().all(|d| *d + == WeightDomain::SamplesOfMetric { + metric: metric.into() + }), + "{query}: {domains:?}" + ); + } + } + + /// Only raw samples and plain sums of them: arithmetic, `rate` and other + /// functions end the requirement. + #[test] + fn samples_of_metric_ends_at_any_other_operator() { + for (query, metric) in [ + ("sum_over_time(m[1m])", Some("m")), + ("sum by (job) (sum_over_time(m[1m]))", Some("m")), + ("sum_over_time(m[1m]) - 100", None), + ("-sum_over_time(m[1m])", None), + ("rate(m[1m])", None), + ("max_over_time(m[1m])", None), + ] { + let root = crate::test_support::lower_promql(query, AccuracyTarget::Exact); + assert_eq!(samples_of_metric(&root).as_deref(), metric, "{query}"); + } + } + + /// `combinations` counts each choice once: picking a whole-expression + /// candidate with any but the first candidate of the target it absorbs + /// is the same candidate. An absorbed target with two candidates (an + /// approximate inner count) gives (1 exact + 0 summaries) · 2 + 2 + /// whole-expression = 4, not 3 · 2. + #[test] + fn combinations_skip_choices_a_whole_expression_candidate_absorbs() { + let inventory = inventory("topk by (job) (10, count_over_time(m[1m]))"); + let topk = topk_target(&inventory); + let target = &inventory.targets[topk]; + let Some(inner) = target.absorbs else { + panic!("absorbs the count: {:?}", target.choices.candidates) + }; + let n = target.choices.candidates.len(); + let whole = target + .choices + .candidates + .iter() + .filter(|c| matches!(c, ComputationCandidate::WholeExpression(_))) + .count(); + let m = inventory.targets[inner].choices.candidates.len(); + let mut enumerated = 0; + let mut choice = vec![0; inventory.targets.len()]; + loop { + if !inventory.absorbed_choice_varies(&choice) { + enumerated += 1; + } + let mut carry = true; + for (digit, target) in choice.iter_mut().zip(&inventory.targets).rev() { + *digit += 1; + if *digit < target.choices.candidates.len() { + carry = false; + break; + } + *digit = 0; + } + if carry { + break; + } + } + assert!(whole > 0 && m > 1, "{n} {whole} {m}"); + assert_eq!(inventory.combinations(), (n - whole) * m + whole); + assert_eq!(inventory.combinations(), enumerated); + } } diff --git a/crates/asap-aware-mapping/src/stage3_plan_selection.rs b/crates/asap-aware-mapping/src/stage3_plan_selection.rs index 17f4ac276..e99f2d23c 100644 --- a/crates/asap-aware-mapping/src/stage3_plan_selection.rs +++ b/crates/asap-aware-mapping/src/stage3_plan_selection.rs @@ -59,7 +59,7 @@ use asap_types::post_asap::{ }; use asap_types::pre_asap::{AggIntent, DataType, Reduction, ScalarValue, Schema}; use asap_types::types::AccuracyTarget; -use asap_types::workload::DataWorkload; +use asap_types::workload::{DataWorkload, MetricType}; use thiserror::Error; use crate::accuracy::composition::{ @@ -228,7 +228,9 @@ fn cheaper(cost: f64, best: f64) -> bool { /// or ratio of certified estimates can be unbounded. The root guarantee's /// symbolic statistics are bound first ([`bound_root_guarantee`]); one still /// unknown leaves the target unmet. Count-Min is admitted only over update -/// weights proven non-negative, which its bound assumes. +/// weights proven non-negative, which its bound assumes: an open +/// [`WeightDomain::SamplesOfMetric`] requirement is met when `data` declares +/// the metric a counter ([`confirmed_weights`]), and unmet otherwise. fn accuracy_violation( candidate: &PhysicalCandidate, targets: &[Option], @@ -247,11 +249,20 @@ fn accuracy_violation( }; if matches!(family, FieldDataType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap)) - && !matches!(input.weight_domain, WeightDomain::NonNegative { .. }) + && !matches!( + confirmed_weights(&input.weight_domain, data), + WeightDomain::NonNegative { .. } + ) { + let unconfirmed = match &input.weight_domain { + WeightDomain::SamplesOfMetric { metric } => { + format!(" (samples of `{metric}`, which is not declared a counter)") + } + _ => String::new(), + }; return Some(format!( "q{query}: {} needs non-negative update weights, and these are not proven \ - non-negative", + non-negative{unconfirmed}", family_name(family) )); } @@ -283,6 +294,64 @@ fn accuracy_violation( None } +/// `domain` with an open [`WeightDomain::SamplesOfMetric`] requirement +/// answered from `data`'s declared metric types: proven when the metric is +/// declared a counter, unknown otherwise. No type is inferred from a name. +pub fn confirmed_weights(domain: &WeightDomain, data: &DataWorkload) -> WeightDomain { + domain.confirmed(|metric| data.metric_types.get(metric) == Some(&MetricType::Counter)) +} + +/// `roots` with every open [`WeightDomain::SamplesOfMetric`] requirement +/// replaced by its answer from `data` ([`confirmed_weights`]): the form of a +/// selected plan the executor runs, which reads no data workload. The same +/// `Rc` where nothing changes. +pub fn with_confirmed_weights(roots: &[QueryRoot], data: &DataWorkload) -> Vec { + fn confirm( + node: &Rc, + data: &DataWorkload, + memo: &mut HashMap<*const OperatorNode, Rc>, + ) -> Rc { + if let Some(done) = memo.get(&Rc::as_ptr(node)) { + return Rc::clone(done); + } + let mut changed = false; + let mut operator = node.operator.map_children(|child| { + let rebuilt = confirm(child, data, memo); + changed |= !Rc::ptr_eq(&rebuilt, child); + rebuilt + }); + if let Operator::ASAP(ASAPOp::SummaryAgg { input, .. }) = &mut operator { + let confirmed = confirmed_weights(&input.weight_domain, data); + if confirmed != input.weight_domain { + input.weight_domain = confirmed; + changed = true; + } + } + let rebuilt = if changed { + Rc::new( + OperatorNode::with_schema(operator, node.schema.clone()) + .with_guarantee(node.guarantee.clone()) + .with_timing(node.timing) + .with_materialization(node.materialization), + ) + } else { + Rc::clone(node) + }; + memo.insert(Rc::as_ptr(node), Rc::clone(&rebuilt)); + rebuilt + } + let mut memo = HashMap::new(); + roots + .iter() + .map(|root| match root { + QueryRoot::Operator(n) => QueryRoot::Operator(confirm(n, data, &mut memo)), + QueryRoot::Scalar(expr) => { + QueryRoot::Scalar(expr.map_operator_refs(&mut |n| confirm(n, data, &mut memo))) + } + }) + .collect() +} + /// The operator nodes a query root reads. fn root_operators(root: &QueryRoot) -> Vec<&Rc> { match root { @@ -1045,7 +1114,7 @@ mod tests { } } - /// Exact (P1), CMS+heap (P2) and CountSketch+heap (P3) realizations of + /// Exact (P1), whole-expression CMS+heap (P2) and CountSketch+heap (P3) realizations of /// one approximate top-k query, in Pass 1 catalog order. fn candidates() -> Vec { let root = lower_promql( @@ -1157,6 +1226,103 @@ mod tests { assert!(p2.reason.contains("non-negative"), "{}", p2.reason); } + /// Count-Min over raw samples of `m` is admitted only when the data + /// workload declares `m` a counter: a gauge, an undeclared metric or a + /// counter under another name leaves the requirement unmet. Admitted, + /// the top-k still needs evidence about its output. + #[test] + fn count_min_over_samples_needs_a_declared_counter() { + use asap_types::workload::MetricType; + let target = AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.001, + }; + let with_types = |types: &[(&str, MetricType)]| DataWorkload { + metric_types: types.iter().map(|(m, t)| (m.to_string(), *t)).collect(), + ..data() + }; + for (types, admitted) in [ + (vec![("m", MetricType::Counter)], true), + (vec![("m", MetricType::Gauge)], false), + (vec![("m_total", MetricType::Counter)], false), + (vec![], false), + ] { + let data = with_types(&types); + let selection = select_plan( + &candidates(), + &[Some(target.clone())], + &data, + PlanningModels::builtin(), + ) + .unwrap(); + let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); + assert!(!p2.valid); + assert_eq!( + !p2.reason.contains("non-negative"), + admitted, + "{types:?}: {}", + p2.reason + ); + if admitted { + assert!(p2.reason.contains("no guarantee"), "{}", p2.reason); + } else { + assert!( + p2.reason.contains("not declared a counter"), + "{}", + p2.reason + ); + } + } + } + + /// The selected plan's open requirement becomes a proof when confirmed + /// and stays unproven otherwise; the executor reads no data workload. + #[test] + fn confirmed_weights_rewrite_only_the_summary_update() { + use asap_types::post_asap::NonNegativeWeightProof; + use asap_types::workload::MetricType; + let p2 = &candidates()[1]; + let domains = |roots: &[QueryRoot]| -> Vec { + roots + .iter() + .flat_map(root_operators) + .flat_map(OperatorNode::reachable) + .filter_map(|n| match n.asap() { + Some(ASAPOp::SummaryAgg { input, .. }) => Some(input.weight_domain.clone()), + _ => None, + }) + .collect() + }; + assert_eq!( + domains(&p2.roots), + [WeightDomain::SamplesOfMetric { metric: "m".into() }] + ); + let counter = DataWorkload { + metric_types: [("m".to_string(), MetricType::Counter)].into(), + ..data() + }; + let confirmed = with_confirmed_weights(&p2.roots, &counter); + assert_eq!( + domains(&confirmed), + [WeightDomain::NonNegative { + proof: NonNegativeWeightProof::CounterSamples + }] + ); + assert_eq!( + domains(&with_confirmed_weights(&p2.roots, &data())), + [WeightDomain::UnknownOrSigned] + ); + // Nothing open: the same roots. + let exact = &candidates()[0]; + let QueryRoot::Operator(before) = &exact.roots[0] else { + panic!() + }; + let QueryRoot::Operator(after) = &with_confirmed_weights(&exact.roots, &counter)[0] else { + panic!() + }; + assert!(Rc::ptr_eq(before, after)); + } + /// A query without a target constrains nothing: the Count-Min /// candidate is admitted and priced. #[test] diff --git a/crates/asap-physical-operators/tests/weighted_topk_binding.rs b/crates/asap-physical-operators/tests/weighted_topk_binding.rs index f89a4b74e..9016ba502 100644 --- a/crates/asap-physical-operators/tests/weighted_topk_binding.rs +++ b/crates/asap-physical-operators/tests/weighted_topk_binding.rs @@ -13,8 +13,8 @@ use planner_types::ir::ASAPOp; use planner_types::{post_asap::*, pre_asap::DataType, types::AccuracyTarget}; use std::{collections::BTreeMap, rc::Rc, sync::Arc}; #[test] -#[ignore = "Pass 1 builds the heap over the grouped Sum's output; a whole-expression heap \ - that ranks the rates and absorbs the Sum is #589"] +#[ignore = "the whole-expression heap's SummaryEstimate{TopK} output is not the keyed-evaluation \ + shape the physical planner builds (#579)"] fn planner_weighted_topk_binds_at_either_deployment_phase() { assert_weighted_binding(SketchAlgorithm::CmsWithHeap); assert_weighted_binding(SketchAlgorithm::CountSketchWithHeap); @@ -138,23 +138,39 @@ fn assert_weighted_binding(algorithm: SketchAlgorithm) { } output }); - assert_eq!(output.len(), 4); - let mut scores = output + // Two items per `job`: the outer `by(job)` partitions the ranking. + let schema = &dag + .nodes .iter() - .map(|row| { - row.iter() - .find_map(|v| { - if let Value::Float64(v) = v { - Some(*v) - } else { - None - } - }) - .unwrap() + .find(|node| node.id == dag.roots[0]) + .unwrap() + .output_schema; + let column = |name: &str| schema.fields.iter().position(|f| f.name == name).unwrap(); + let (job, value) = (column("job"), column("value")); + let mut ranked = output + .iter() + .map(|row| match (&row[job], &row[value]) { + (Value::Utf8(job), Value::Float64(score)) => (job.to_string(), *score), + other => panic!("unexpected (job, value) {other:?}"), }) .collect::>(); - scores.sort_by(f64::total_cmp); - assert_eq!(scores, vec![0.3125, 0.375, 80., 100.]); + ranked.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.total_cmp(&b.1))); + let expected = [ + ("api", 0.3125), + ("api", 0.375), + ("batch", 80.), + ("batch", 100.), + ]; + assert_eq!( + ranked, + expected.map(|(job, score)| (job.to_string(), score)), + "{}", + if placed == dag { + "query time" + } else { + "ingestion time" + } + ); } } diff --git a/crates/devtools/src/planner_layering_examples.rs b/crates/devtools/src/planner_layering_examples.rs index 2ea585d3b..b9f38a639 100644 --- a/crates/devtools/src/planner_layering_examples.rs +++ b/crates/devtools/src/planner_layering_examples.rs @@ -5,8 +5,8 @@ use asap_types::types::AccuracyTarget; use asap_types::workload::{ AccuracyRequirement, BatchEntry, DataArrival, DataDistribution, DataWorkload, DurationMs, - Evidence, EvidenceSource, LatencyRequirement, PlanningWorkload, Predictability, Query, - QueryLanguage, QueryRequirements, QueryTimeScope, QueryWorkload, Rate, RepeatedDemand, + Evidence, EvidenceSource, LatencyRequirement, MetricType, PlanningWorkload, Predictability, + Query, QueryLanguage, QueryRequirements, QueryTimeScope, QueryWorkload, Rate, RepeatedDemand, RepeatingEntry, RepetitionInterval, TimeSelection, TimestampMs, }; @@ -39,6 +39,7 @@ pub fn shared_data_workload(arrival: DataArrival) -> DataWorkload { stream_norms: Evidence::default(), max_distinct_per_evaluation: Evidence::default(), arrival, + metric_types: Default::default(), } } @@ -46,6 +47,8 @@ pub const EXAMPLE1_Q1: &str = "sum by (job) (rate(http_requests_total[1m]))"; pub const EXAMPLE1_Q2: &str = "topk by (job) (10, sum_over_time(http_requests_total[1m]))"; /// Example 1: two dashboard panels, every 10 s over the last minute. +/// `http_requests_total` is declared a counter: its samples are never +/// negative. pub fn example1() -> PlanningWorkload { let panel = |query: &str, accuracy, response_latency| RepeatingEntry { query: Query(query.into()), @@ -81,7 +84,10 @@ pub fn example1() -> PlanningWorkload { ), ]), }, - data_workload: Some(shared_data_workload(DataArrival::ContinuouslyIngesting)), + data_workload: Some(DataWorkload { + metric_types: [("http_requests_total".to_string(), MetricType::Counter)].into(), + ..shared_data_workload(DataArrival::ContinuouslyIngesting) + }), } } diff --git a/crates/integration-tests/tests/planner_layering_example1.rs b/crates/integration-tests/tests/planner_layering_example1.rs index 4c984cb37..b69e46df8 100644 --- a/crates/integration-tests/tests/planner_layering_example1.rs +++ b/crates/integration-tests/tests/planner_layering_example1.rs @@ -191,11 +191,17 @@ fn stage1_logical_asap_optimization_q1_is_always_exact() { } } -/// Every Pass 1 target's exact option is its exact accumulator: Q1's rate and sum and Q2's sum_over_time. +/// Every Pass 1 target's exact option is its exact accumulator: Q1's rate +/// and sum and Q2's sum_over_time. Q2's heaps are whole-expression: they +/// absorb its sum_over_time, which is then not computed. #[test] fn stage1_logical_asap_optimization_exact_targets_use_accumulators() { let run = pipeline(); for c in &run.logical { + let expected = match classify(&c.dag, &c.query_roots).0 { + Q2Option::Exact => 3, + _ => 2, + }; let accumulators = c .dag .nodes @@ -207,7 +213,7 @@ fn stage1_logical_asap_optimization_exact_targets_use_accumulators() { ) }) .count(); - assert_eq!(accumulators, 3, "{}", c.id); + assert_eq!(accumulators, expected, "{}", c.id); } } @@ -443,9 +449,10 @@ fn compile_in_runtime(p: &Physical) -> Result<(), String> { /// Runtime capability check (added by the implementer, not part of the /// spec): the physical planner compiles every physical candidate. #[test] -#[ignore = "runtime gaps: CMS+heap needs a non-negative weight contract (#593); \ - CountSketch+heap's SummaryEstimate{TopK} output is not the keyed-evaluation \ - shape the physical planner builds (#579)"] +#[ignore = "runtime gaps: a physical candidate's CMS+heap weights are an open requirement \ + that only the selected plan's confirmation answers; the heaps' \ + SummaryEstimate{TopK} output is not the keyed-evaluation shape the physical \ + planner builds (#579)"] fn stage2_physical_asap_optimization_every_candidate_compiles_in_the_physical_planner() { let run = pipeline(); let failures: Vec<_> = run @@ -464,9 +471,10 @@ fn stage3_plan_selection_selects_one_and_explains_the_rest() { assert_selects_one_and_explains_the_rest(&pipeline()); } -/// The exact Q2 is selected. Count-Min + heap is invalid because its weights -/// are not proven non-negative; CountSketch + heap is invalid because its -/// top-k output has no guarantee and the deployment has no evidence for it. +/// The exact Q2 is selected. Both heaps are invalid because a top-k output +/// has no guarantee and the deployment has no evidence for it. Count-Min's +/// weights are raw samples of `http_requests_total`, which the workload +/// declares a counter, so they are non-negative. #[test] fn stage3_plan_selection_selects_exact_q2_and_rejects_both_heaps() { let run = pipeline(); @@ -486,7 +494,7 @@ fn stage3_plan_selection_selects_exact_q2_and_rejects_both_heaps() { } let cms = &physical_with(&run, Q2Option::CountMinHeapPerJob).id; assert!( - invalid[cms.as_str()].contains("non-negative"), + invalid[cms.as_str()].contains("no guarantee"), "{invalid:?}" ); let count_sketch = &physical_with(&run, Q2Option::CountSketchHeapPerJob).id; @@ -496,6 +504,26 @@ fn stage3_plan_selection_selects_exact_q2_and_rejects_both_heaps() { ); } +/// Without the counter declaration, Count-Min's weights are not proven +/// non-negative: plan selection reads no type from the `_total` name. +#[test] +fn stage3_plan_selection_count_min_needs_the_counter_declaration() { + let mut workload = example1(); + workload + .data_workload + .as_mut() + .unwrap() + .metric_types + .clear(); + let run = run_promql(&workload); + let invalid = run.invalid(); + let cms = &physical_with(&run, Q2Option::CountMinHeapPerJob).id; + assert!( + invalid[cms.as_str()].contains("non-negative"), + "{invalid:?}" + ); +} + /// The selected plan is the cheapest valid candidate for the whole workload. #[test] fn stage3_plan_selection_selects_cheapest_valid() { diff --git a/crates/integration-tests/tests/planner_layering_example2.rs b/crates/integration-tests/tests/planner_layering_example2.rs index 3fcaa8247..0abd45fac 100644 --- a/crates/integration-tests/tests/planner_layering_example2.rs +++ b/crates/integration-tests/tests/planner_layering_example2.rs @@ -85,6 +85,7 @@ fn data_workload(ingestion_interval: bool) -> DataWorkload { distribution: declared(DataDistribution::Zipf), stream_norms: Default::default(), max_distinct_per_evaluation: Default::default(), + metric_types: Default::default(), } } diff --git a/crates/integration-tests/tests/precompute_raw_samples.rs b/crates/integration-tests/tests/precompute_raw_samples.rs index c783fc9d1..7e9505793 100644 --- a/crates/integration-tests/tests/precompute_raw_samples.rs +++ b/crates/integration-tests/tests/precompute_raw_samples.rs @@ -49,10 +49,28 @@ fn canonical(labels: &Series) -> Series { .collect() } -/// Every logical candidate the planner pipeline composes for `query`. +/// Every logical candidate the planner pipeline composes for `query`, with +/// `m` declared a counter: plan selection's answer to a heap's open +/// requirement that its weights are counter samples, which the executor +/// needs before it builds a Count-Min heap. fn candidates(query: &str, accuracy: AccuracyTarget) -> Vec> { + use asap_aware_mapping::stage3_plan_selection::with_confirmed_weights; + use asap_types::ir::QueryRoot; + use asap_types::workload::{DataWorkload, MetricType}; let root = lower_promql(query, accuracy.clone()).expect("lowering failed"); + let data = DataWorkload { + metric_types: [("m".to_string(), MetricType::Counter)].into(), + ..Default::default() + }; logical_candidates(root, accuracy) + .into_iter() + .map(|candidate| { + match &with_confirmed_weights(&[QueryRoot::Operator(candidate)], &data)[..] { + [QueryRoot::Operator(confirmed)] => Rc::clone(confirmed), + _ => unreachable!("one operator root"), + } + }) + .collect() } /// Raw-input summary nodes: `(dag, raw source id, summary id)`. @@ -420,10 +438,18 @@ fn raw_sample_summaries_compile_and_match_their_kernels() { ); } for family in [ - "Sum", "Count", "Min", "Max", "Rate", "Increase", "Kll", "DDSketch", + "Sum", + "Count", + "Min", + "Max", + "Rate", + "Increase", + "Kll", + "DDSketch", "Hll", - // A heap over raw samples is the whole-expression heap (#589); Pass 1 - // builds a top-k heap over the ranked aggregate's output. + // The whole-expression heaps over raw samples. + "CmsWithHeap", + "CountSketchWithHeap", ] { assert!( families.contains(family), diff --git a/crates/integration-tests/tests/promql_numeric_regressions.rs b/crates/integration-tests/tests/promql_numeric_regressions.rs index d28144faa..15dbe1567 100644 --- a/crates/integration-tests/tests/promql_numeric_regressions.rs +++ b/crates/integration-tests/tests/promql_numeric_regressions.rs @@ -172,8 +172,6 @@ fn quantile_over_temporal_average_keeps_a_legal_candidate() { } #[test] -#[ignore = "Pass 1 builds the top-k heap over the counts the aggregate outputs, weighted by \ - them; a heap over raw samples with unit weights is the whole-expression heap (#589)"] fn sketch_counts_use_unit_weights_and_signed_sums_keep_value_weights() { use asap_types::post_asap::{NonNegativeWeightProof, SketchAlgorithm, WeightDomain}; for is_count in [true, false] { diff --git a/crates/integration-tests/tests/stage1_logical_asap_optimization.rs b/crates/integration-tests/tests/stage1_logical_asap_optimization.rs index cef004033..b56c0851f 100644 --- a/crates/integration-tests/tests/stage1_logical_asap_optimization.rs +++ b/crates/integration-tests/tests/stage1_logical_asap_optimization.rs @@ -231,8 +231,8 @@ fn topk_estimates_carry_no_frequency_bound() { assert!(cms.guarantee.as_ref().is_some_and(|g| !g.is_exact())); } -/// Top-K over a closed PromQL child without series identity ranks the -/// child's labels outside the partition. +/// Top-K over a grouped sum is a whole-expression heap: it absorbs the +/// sum and ranks its group keys, read from the rows the sum reads. #[test] fn topk_over_an_aggregate_ranks_its_labels() { let estimates = composed_estimates(promql("topk(3, sum by (job) (rate(m[5m])))")); @@ -240,9 +240,7 @@ fn topk_over_an_aggregate_ranks_its_labels() { for estimate in &estimates { assert_eq!( update_of(estimate).1.item, - Some(SummaryInputExpr::Tuple(vec![SummaryInputExpr::Column( - ColumnRef::Named("job".into()) - )])) + Some(SummaryInputExpr::Column(ColumnRef::Named("job".into()))) ); } } diff --git a/crates/types/src/post_asap/sketch.rs b/crates/types/src/post_asap/sketch.rs index b0a338c12..ef3b96a36 100644 --- a/crates/types/src/post_asap/sketch.rs +++ b/crates/types/src/post_asap/sketch.rs @@ -663,6 +663,34 @@ pub enum WeightDomain { NonNegative { proof: NonNegativeWeightProof, }, + /// An open requirement, not a proof: the weights are raw samples of + /// `metric`, or plain sums of them (no rate, no other arithmetic), so they + /// are non-negative exactly when `metric` is a counter. Logical + /// optimization records it without reading the data workload; plan + /// selection confirms it from the declared metric types + /// ([`DataWorkload::metric_types`](crate::workload::DataWorkload::metric_types)). + /// Unconfirmed, it counts as not proven. + SamplesOfMetric { + metric: String, + }, +} + +impl WeightDomain { + /// The domain once `is_counter` has answered an open + /// [`SamplesOfMetric`](Self::SamplesOfMetric) requirement: proven + /// ([`NonNegativeWeightProof::CounterSamples`]) when the metric is a + /// counter, else unknown. Any other domain is returned unchanged. + pub fn confirmed(&self, is_counter: impl Fn(&str) -> bool) -> WeightDomain { + match self { + WeightDomain::SamplesOfMetric { metric } if is_counter(metric) => { + WeightDomain::NonNegative { + proof: NonNegativeWeightProof::CounterSamples, + } + } + WeightDomain::SamplesOfMetric { .. } => WeightDomain::UnknownOrSigned, + other => other.clone(), + } + } } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -672,6 +700,12 @@ pub enum NonNegativeWeightProof { /// PromQL counter reset correction produces a non-negative increase; rate /// divides that increase by a positive duration. ResetAwareCounterDerivative, + /// Raw samples of a metric declared a counter + /// ([`MetricType::Counter`](crate::workload::MetricType::Counter)), or + /// plain sums of them: a counter sample is never negative. Plan selection + /// gives it on confirming a + /// [`WeightDomain::SamplesOfMetric`] requirement. + CounterSamples, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/types/src/workload.rs b/crates/types/src/workload.rs index 03c464667..140865e0f 100644 --- a/crates/types/src/workload.rs +++ b/crates/types/src/workload.rs @@ -557,6 +557,12 @@ pub struct DataWorkload { /// without a bound leaves that probability unknown. #[serde(default)] pub max_distinct_per_evaluation: Evidence>, + + /// Declared type of each metric, by metric name (Prometheus `# TYPE` + /// metadata). The planner never infers a type from a name such as + /// `_total`; an undeclared metric's samples may take any value. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub metric_types: BTreeMap, } /// Upper bounds on the norms of the frequency (weight) vector any one @@ -572,6 +578,17 @@ pub struct StreamNorms { pub l2: Option, } +/// The Prometheus type of a metric's samples. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MetricType { + /// Starts at zero and only increases, or resets to zero: every sample is + /// non-negative. + Counter, + /// May take any value. + Gauge, +} + // ── Top-level workload ──────────────────────────────────────────────────────── /// `query_batch` and `repeating_queries` may both be present. [`Self::entries`] diff --git a/tools/dag-viewer/examples/planner-layering-example1.json b/tools/dag-viewer/examples/planner-layering-example1.json index f1680bb7e..f6d7045f4 100644 --- a/tools/dag-viewer/examples/planner-layering-example1.json +++ b/tools/dag-viewer/examples/planner-layering-example1.json @@ -3656,155 +3656,6 @@ "ASAP": { "SummaryAgg": { "child": 7, - "family": { - "ExactAggregate": [ - "Sum", - "Sum" - ] - }, - "filter": null, - "grouping": "PerSubpopulationInstance", - "input": { - "item": null, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" - } - }, - "reduction": "PerEntity" - } - } - }, - "result_kind": "state", - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" - ] - }, - "name": "state", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "timing": null - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact accumulator over exact input" - } - ] - }, - "materialization": "NotMaterialized", - "operator": { - "ASAP": { - "FinalizeExactAccumulator": { - "child": 8 - } - } - }, - "result_kind": "instant_vector", - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "timing": null - }, - { - "guarantee": null, - "materialization": "NotMaterialized", - "operator": { - "ASAP": { - "SummaryAgg": { - "child": 9, "family": { "Sketch": [ { @@ -3833,7 +3684,8 @@ "Column": "SampleValue" }, "weight_domain": { - "kind": "unknown_or_signed" + "kind": "samples_of_metric", + "metric": "http_requests_total" } }, "reduction": { @@ -3898,7 +3750,7 @@ "k": 10 } }, - "summary_input": 10 + "summary_input": 8 } } }, @@ -3938,7 +3790,7 @@ "operator": { "NonASAP": { "Project": { - "child": 11, + "child": 9, "cols": [ { "alias": "job", @@ -3993,11 +3845,11 @@ "Operator": 5 }, { - "Operator": 12 + "Operator": 10 } ] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap (Sum acc)", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap", "id": "L4" }, { @@ -4551,23 +4403,42 @@ "SummaryAgg": { "child": 6, "family": { - "ExactAggregate": [ - "Sum", - "Sum" + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" ] }, "filter": null, "grouping": "PerSubpopulationInstance", "input": { - "item": null, + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, "weight": { "Column": "SampleValue" }, "weight_domain": { - "kind": "unknown_or_signed" + "kind": "samples_of_metric", + "metric": "http_requests_total" } }, - "reduction": "PerEntity" + "reduction": { + "Reduce": [ + 2 + ] + } } } }, @@ -4575,25 +4446,6 @@ "schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" - ] - }, - "name": "state", - "nullable": false, - "table": null - }, { "dtype": { "Plain": "utf8" @@ -4604,17 +4456,29 @@ }, { "dtype": { - "Plain": "utf8" + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] }, - "name": "$promql_series_identity", + "name": "state", "nullable": false, "table": null } ], - "time_index": 0, + "time_index": null, "unique_keys": [ [ - 3, 0 ] ] @@ -4622,26 +4486,17 @@ "timing": null }, { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact accumulator over exact input" - } - ] - }, + "guarantee": null, "materialization": "NotMaterialized", "operator": { "ASAP": { - "FinalizeExactAccumulator": { - "child": 7 + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 7 } } }, @@ -4651,166 +4506,15 @@ "fields": [ { "dtype": { - "Plain": "timestamp" + "Plain": "utf8" }, - "name": "ts", - "nullable": false, + "name": "job", + "nullable": true, "table": null }, { "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "timing": null - }, - { - "guarantee": null, - "materialization": "NotMaterialized", - "operator": { - "ASAP": { - "SummaryAgg": { - "child": 8, - "family": { - "Sketch": [ - { - "algorithm": "CmsWithHeap", - "category": "TopK", - "params": { - "CmsWithHeap": { - "depth": 7, - "heap_size": 100, - "width": 272 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "filter": null, - "grouping": "PerSubpopulationInstance", - "input": { - "item": { - "Column": { - "Named": "$promql_series_identity" - } - }, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" - } - }, - "reduction": { - "Reduce": [ - 2 - ] - } - } - } - }, - "result_kind": "state", - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Sketch": [ - { - "algorithm": "CmsWithHeap", - "category": "TopK", - "params": { - "CmsWithHeap": { - "depth": 7, - "heap_size": 100, - "width": 272 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "name": "state", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "timing": null - }, - { - "guarantee": null, - "materialization": "NotMaterialized", - "operator": { - "ASAP": { - "SummaryEstimate": { - "query": { - "TopK": { - "k": 10 - } - }, - "summary_input": 9 - } - } - }, - "result_kind": "instant_vector", - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" + "Plain": "utf8" }, "name": "topk", "nullable": false, @@ -4832,7 +4536,7 @@ "operator": { "NonASAP": { "Project": { - "child": 10, + "child": 8, "cols": [ { "alias": "job", @@ -4887,11 +4591,11 @@ "Operator": 5 }, { - "Operator": 11 + "Operator": 9 } ] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap (Sum acc) · shared Scan", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap · shared Scan", "id": "L5" }, { @@ -5369,23 +5073,42 @@ "SummaryAgg": { "child": 1, "family": { - "ExactAggregate": [ - "Sum", - "Sum" + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" ] }, "filter": null, "grouping": "PerSubpopulationInstance", "input": { - "item": null, + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, "weight": { "Column": "SampleValue" }, "weight_domain": { - "kind": "unknown_or_signed" + "kind": "samples_of_metric", + "metric": "http_requests_total" } }, - "reduction": "PerEntity" + "reduction": { + "Reduce": [ + 2 + ] + } } } }, @@ -5393,25 +5116,6 @@ "schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" - ] - }, - "name": "state", - "nullable": false, - "table": null - }, { "dtype": { "Plain": "utf8" @@ -5422,17 +5126,29 @@ }, { "dtype": { - "Plain": "utf8" + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] }, - "name": "$promql_series_identity", + "name": "state", "nullable": false, "table": null } ], - "time_index": 0, + "time_index": null, "unique_keys": [ [ - 3, 0 ] ] @@ -5440,26 +5156,17 @@ "timing": null }, { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact accumulator over exact input" - } - ] - }, + "guarantee": null, "materialization": "NotMaterialized", "operator": { "ASAP": { - "FinalizeExactAccumulator": { - "child": 6 + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 6 } } }, @@ -5469,168 +5176,17 @@ "fields": [ { "dtype": { - "Plain": "timestamp" + "Plain": "utf8" }, - "name": "ts", - "nullable": false, + "name": "job", + "nullable": true, "table": null }, { "dtype": { - "Plain": "float64" + "Plain": "utf8" }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "timing": null - }, - { - "guarantee": null, - "materialization": "NotMaterialized", - "operator": { - "ASAP": { - "SummaryAgg": { - "child": 7, - "family": { - "Sketch": [ - { - "algorithm": "CmsWithHeap", - "category": "TopK", - "params": { - "CmsWithHeap": { - "depth": 7, - "heap_size": 100, - "width": 272 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "filter": null, - "grouping": "PerSubpopulationInstance", - "input": { - "item": { - "Column": { - "Named": "$promql_series_identity" - } - }, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" - } - }, - "reduction": { - "Reduce": [ - 2 - ] - } - } - } - }, - "result_kind": "state", - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Sketch": [ - { - "algorithm": "CmsWithHeap", - "category": "TopK", - "params": { - "CmsWithHeap": { - "depth": 7, - "heap_size": 100, - "width": 272 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "name": "state", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "timing": null - }, - { - "guarantee": null, - "materialization": "NotMaterialized", - "operator": { - "ASAP": { - "SummaryEstimate": { - "query": { - "TopK": { - "k": 10 - } - }, - "summary_input": 8 - } - } - }, - "result_kind": "instant_vector", - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "topk", + "name": "topk", "nullable": false, "table": null } @@ -5650,7 +5206,7 @@ "operator": { "NonASAP": { "Project": { - "child": 9, + "child": 7, "cols": [ { "alias": "job", @@ -5705,11 +5261,11 @@ "Operator": 5 }, { - "Operator": 10 + "Operator": 8 } ] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap (Sum acc) · shared Scan, TimeRange", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap · shared Scan, TimeRange", "id": "L6" }, { @@ -6383,23 +5939,42 @@ "SummaryAgg": { "child": 7, "family": { - "ExactAggregate": [ - "Sum", - "Sum" + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } + }, + "PerSubpopulationInstance" ] }, "filter": null, "grouping": "PerSubpopulationInstance", "input": { - "item": null, + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, "weight": { "Column": "SampleValue" }, "weight_domain": { - "kind": "unknown_or_signed" + "kind": "samples_of_metric", + "metric": "http_requests_total" } }, - "reduction": "PerEntity" + "reduction": { + "Reduce": [ + 2 + ] + } } } }, @@ -6407,25 +5982,6 @@ "schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" - ] - }, - "name": "state", - "nullable": false, - "table": null - }, { "dtype": { "Plain": "utf8" @@ -6436,17 +5992,29 @@ }, { "dtype": { - "Plain": "utf8" + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } + }, + "PerSubpopulationInstance" + ] }, - "name": "$promql_series_identity", + "name": "state", "nullable": false, "table": null } ], - "time_index": 0, + "time_index": null, "unique_keys": [ [ - 3, 0 ] ] @@ -6454,26 +6022,17 @@ "timing": null }, { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact accumulator over exact input" - } - ] - }, + "guarantee": null, "materialization": "NotMaterialized", "operator": { "ASAP": { - "FinalizeExactAccumulator": { - "child": 8 + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 8 } } }, @@ -6483,161 +6042,10 @@ "fields": [ { "dtype": { - "Plain": "timestamp" + "Plain": "utf8" }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "timing": null - }, - { - "guarantee": null, - "materialization": "NotMaterialized", - "operator": { - "ASAP": { - "SummaryAgg": { - "child": 9, - "family": { - "Sketch": [ - { - "algorithm": "CountSketchWithHeap", - "category": "TopK", - "params": { - "CountSketchWithHeap": { - "depth": 125, - "heap_size": 100, - "width": 30000 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "filter": null, - "grouping": "PerSubpopulationInstance", - "input": { - "item": { - "Column": { - "Named": "$promql_series_identity" - } - }, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" - } - }, - "reduction": { - "Reduce": [ - 2 - ] - } - } - } - }, - "result_kind": "state", - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Sketch": [ - { - "algorithm": "CountSketchWithHeap", - "category": "TopK", - "params": { - "CountSketchWithHeap": { - "depth": 125, - "heap_size": 100, - "width": 30000 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "name": "state", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "timing": null - }, - { - "guarantee": null, - "materialization": "NotMaterialized", - "operator": { - "ASAP": { - "SummaryEstimate": { - "query": { - "TopK": { - "k": 10 - } - }, - "summary_input": 10 - } - } - }, - "result_kind": "instant_vector", - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, + "name": "job", + "nullable": true, "table": null }, { @@ -6664,7 +6072,7 @@ "operator": { "NonASAP": { "Project": { - "child": 11, + "child": 9, "cols": [ { "alias": "job", @@ -6719,11 +6127,11 @@ "Operator": 5 }, { - "Operator": 12 + "Operator": 10 } ] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap (Sum acc)", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap", "id": "L7" }, { @@ -7277,23 +6685,42 @@ "SummaryAgg": { "child": 6, "family": { - "ExactAggregate": [ - "Sum", - "Sum" + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } + }, + "PerSubpopulationInstance" ] }, "filter": null, "grouping": "PerSubpopulationInstance", "input": { - "item": null, + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, "weight": { "Column": "SampleValue" }, "weight_domain": { - "kind": "unknown_or_signed" + "kind": "samples_of_metric", + "metric": "http_requests_total" } }, - "reduction": "PerEntity" + "reduction": { + "Reduce": [ + 2 + ] + } } } }, @@ -7301,25 +6728,6 @@ "schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" - ] - }, - "name": "state", - "nullable": false, - "table": null - }, { "dtype": { "Plain": "utf8" @@ -7330,17 +6738,29 @@ }, { "dtype": { - "Plain": "utf8" + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } + }, + "PerSubpopulationInstance" + ] }, - "name": "$promql_series_identity", + "name": "state", "nullable": false, "table": null } ], - "time_index": 0, + "time_index": null, "unique_keys": [ [ - 3, 0 ] ] @@ -7348,26 +6768,17 @@ "timing": null }, { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact accumulator over exact input" - } - ] - }, + "guarantee": null, "materialization": "NotMaterialized", "operator": { "ASAP": { - "FinalizeExactAccumulator": { - "child": 7 + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 7 } } }, @@ -7377,168 +6788,17 @@ "fields": [ { "dtype": { - "Plain": "timestamp" + "Plain": "utf8" }, - "name": "ts", - "nullable": false, + "name": "job", + "nullable": true, "table": null }, { "dtype": { - "Plain": "float64" + "Plain": "utf8" }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "timing": null - }, - { - "guarantee": null, - "materialization": "NotMaterialized", - "operator": { - "ASAP": { - "SummaryAgg": { - "child": 8, - "family": { - "Sketch": [ - { - "algorithm": "CountSketchWithHeap", - "category": "TopK", - "params": { - "CountSketchWithHeap": { - "depth": 125, - "heap_size": 100, - "width": 30000 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "filter": null, - "grouping": "PerSubpopulationInstance", - "input": { - "item": { - "Column": { - "Named": "$promql_series_identity" - } - }, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" - } - }, - "reduction": { - "Reduce": [ - 2 - ] - } - } - } - }, - "result_kind": "state", - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Sketch": [ - { - "algorithm": "CountSketchWithHeap", - "category": "TopK", - "params": { - "CountSketchWithHeap": { - "depth": 125, - "heap_size": 100, - "width": 30000 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "name": "state", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "timing": null - }, - { - "guarantee": null, - "materialization": "NotMaterialized", - "operator": { - "ASAP": { - "SummaryEstimate": { - "query": { - "TopK": { - "k": 10 - } - }, - "summary_input": 9 - } - } - }, - "result_kind": "instant_vector", - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "topk", + "name": "topk", "nullable": false, "table": null } @@ -7558,7 +6818,7 @@ "operator": { "NonASAP": { "Project": { - "child": 10, + "child": 8, "cols": [ { "alias": "job", @@ -7613,11 +6873,11 @@ "Operator": 5 }, { - "Operator": 11 + "Operator": 9 } ] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap (Sum acc) · shared Scan", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap · shared Scan", "id": "L8" }, { @@ -8095,23 +7355,42 @@ "SummaryAgg": { "child": 1, "family": { - "ExactAggregate": [ - "Sum", - "Sum" + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } + }, + "PerSubpopulationInstance" ] }, "filter": null, "grouping": "PerSubpopulationInstance", "input": { - "item": null, + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, "weight": { "Column": "SampleValue" }, "weight_domain": { - "kind": "unknown_or_signed" + "kind": "samples_of_metric", + "metric": "http_requests_total" } }, - "reduction": "PerEntity" + "reduction": { + "Reduce": [ + 2 + ] + } } } }, @@ -8119,25 +7398,6 @@ "schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" - ] - }, - "name": "state", - "nullable": false, - "table": null - }, { "dtype": { "Plain": "utf8" @@ -8148,17 +7408,29 @@ }, { "dtype": { - "Plain": "utf8" + "Sketch": [ + { + "algorithm": "CountSketchWithHeap", + "category": "TopK", + "params": { + "CountSketchWithHeap": { + "depth": 125, + "heap_size": 100, + "width": 30000 + } + } + }, + "PerSubpopulationInstance" + ] }, - "name": "$promql_series_identity", + "name": "state", "nullable": false, "table": null } ], - "time_index": 0, + "time_index": null, "unique_keys": [ [ - 3, 0 ] ] @@ -8166,26 +7438,17 @@ "timing": null }, { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact accumulator over exact input" - } - ] - }, + "guarantee": null, "materialization": "NotMaterialized", "operator": { "ASAP": { - "FinalizeExactAccumulator": { - "child": 6 + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 6 } } }, @@ -8195,168 +7458,17 @@ "fields": [ { "dtype": { - "Plain": "timestamp" + "Plain": "utf8" }, - "name": "ts", - "nullable": false, + "name": "job", + "nullable": true, "table": null }, { "dtype": { - "Plain": "float64" + "Plain": "utf8" }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "timing": null - }, - { - "guarantee": null, - "materialization": "NotMaterialized", - "operator": { - "ASAP": { - "SummaryAgg": { - "child": 7, - "family": { - "Sketch": [ - { - "algorithm": "CountSketchWithHeap", - "category": "TopK", - "params": { - "CountSketchWithHeap": { - "depth": 125, - "heap_size": 100, - "width": 30000 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "filter": null, - "grouping": "PerSubpopulationInstance", - "input": { - "item": { - "Column": { - "Named": "$promql_series_identity" - } - }, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" - } - }, - "reduction": { - "Reduce": [ - 2 - ] - } - } - } - }, - "result_kind": "state", - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Sketch": [ - { - "algorithm": "CountSketchWithHeap", - "category": "TopK", - "params": { - "CountSketchWithHeap": { - "depth": 125, - "heap_size": 100, - "width": 30000 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "name": "state", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "timing": null - }, - { - "guarantee": null, - "materialization": "NotMaterialized", - "operator": { - "ASAP": { - "SummaryEstimate": { - "query": { - "TopK": { - "k": 10 - } - }, - "summary_input": 8 - } - } - }, - "result_kind": "instant_vector", - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "topk", + "name": "topk", "nullable": false, "table": null } @@ -8376,7 +7488,7 @@ "operator": { "NonASAP": { "Project": { - "child": 9, + "child": 7, "cols": [ { "alias": "job", @@ -8431,11 +7543,11 @@ "Operator": 5 }, { - "Operator": 10 + "Operator": 8 } ] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap (Sum acc) · shared Scan, TimeRange", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap · shared Scan, TimeRange", "id": "L9" } ], @@ -13508,23 +12620,55 @@ "fields": [ { "dtype": { - "Plain": "timestamp" + "Plain": "utf8" }, - "name": "ts", - "nullable": false, + "name": "job", + "nullable": true, "table": null }, { "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" ] }, "name": "state", "nullable": false, "table": null - }, + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 8, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 10, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ { "dtype": { "Plain": "utf8" @@ -13537,31 +12681,43 @@ "dtype": { "Plain": "utf8" }, - "name": "$promql_series_identity", + "name": "topk", "nullable": false, "table": null } ], - "time_index": 0, + "time_index": null, "unique_keys": [ [ - 3, 0 ] ] }, - "producer": 8, + "producer": 9, "role": "Input", "window": "NotApplicable" - }, + } + ], + "nodes": [ { - "consumer": 10, - "data_state": { - "primitive": "Raw", - "timing": "query_time" + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] }, - "grouping": "NotApplicable", - "intermediate_schema": { + "id": 0, + "materialization": "NotMaterialized", + "output_schema": { "closed": true, "fields": [ { @@ -13605,165 +12761,8 @@ ] ] }, - "producer": 9, - "role": "Input", - "window": "NotApplicable" - }, - { - "consumer": 11, - "data_state": { - "primitive": "SummaryState", - "timing": "query_time" - }, - "grouping": "NotApplicable", - "intermediate_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Sketch": [ - { - "algorithm": "CmsWithHeap", - "category": "TopK", - "params": { - "CmsWithHeap": { - "depth": 7, - "heap_size": 100, - "width": 272 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "name": "state", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "producer": 10, - "role": "Input", - "window": "NotApplicable" - }, - { - "consumer": 12, - "data_state": { - "primitive": "Raw", - "timing": "query_time" - }, - "grouping": "NotApplicable", - "intermediate_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "topk", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "producer": 11, - "role": "Input", - "window": "NotApplicable" - } - ], - "nodes": [ - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact over exact inputs" - } - ] - }, - "id": 0, - "materialization": "NotMaterialized", - "output_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "output_state": { - "primitive": "Raw", + "output_state": { + "primitive": "Raw", "timing": "query_time" }, "payload": { @@ -14391,25 +13390,6 @@ "output_schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" - ] - }, - "name": "state", - "nullable": false, - "table": null - }, { "dtype": { "Plain": "utf8" @@ -14420,17 +13400,29 @@ }, { "dtype": { - "Plain": "utf8" + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] }, - "name": "$promql_series_identity", + "name": "state", "nullable": false, "table": null } ], - "time_index": 0, + "time_index": null, "unique_keys": [ [ - 3, 0 ] ] @@ -14444,64 +13436,53 @@ "SummaryAgg": { "child": 7, "family": { - "ExactAggregate": [ - "Sum", - "Sum" + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" ] }, "filter": null, "grouping": "PerSubpopulationInstance", "input": { - "item": null, + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, "weight": { "Column": "SampleValue" }, "weight_domain": { - "kind": "unknown_or_signed" + "kind": "samples_of_metric", + "metric": "http_requests_total" } }, - "reduction": "PerEntity" + "reduction": { + "Reduce": [ + 2 + ] + } } } } }, { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact accumulator over exact input" - } - ] - }, + "guarantee": null, "id": 9, "materialization": "NotMaterialized", "output_schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, { "dtype": { "Plain": "utf8" @@ -14514,15 +13495,14 @@ "dtype": { "Plain": "utf8" }, - "name": "$promql_series_identity", + "name": "topk", "nullable": false, "table": null } ], - "time_index": 0, + "time_index": null, "unique_keys": [ [ - 3, 0 ] ] @@ -14533,8 +13513,13 @@ }, "payload": { "ASAP": { - "FinalizeExactAccumulator": { - "child": 8 + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 8 } } } @@ -14543,146 +13528,6 @@ "guarantee": null, "id": 10, "materialization": "NotMaterialized", - "output_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Sketch": [ - { - "algorithm": "CmsWithHeap", - "category": "TopK", - "params": { - "CmsWithHeap": { - "depth": 7, - "heap_size": 100, - "width": 272 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "name": "state", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "output_state": { - "primitive": "SummaryState", - "timing": "query_time" - }, - "payload": { - "ASAP": { - "SummaryAgg": { - "child": 9, - "family": { - "Sketch": [ - { - "algorithm": "CmsWithHeap", - "category": "TopK", - "params": { - "CmsWithHeap": { - "depth": 7, - "heap_size": 100, - "width": 272 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "filter": null, - "grouping": "PerSubpopulationInstance", - "input": { - "item": { - "Column": { - "Named": "$promql_series_identity" - } - }, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" - } - }, - "reduction": { - "Reduce": [ - 2 - ] - } - } - } - } - }, - { - "guarantee": null, - "id": 11, - "materialization": "NotMaterialized", - "output_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "topk", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "output_state": { - "primitive": "Raw", - "timing": "query_time" - }, - "payload": { - "ASAP": { - "SummaryEstimate": { - "query": { - "TopK": { - "k": 10 - } - }, - "summary_input": 10 - } - } - } - }, - { - "guarantee": null, - "id": 12, - "materialization": "NotMaterialized", "output_schema": { "closed": true, "fields": [ @@ -14717,7 +13562,7 @@ "payload": { "NonASAP": { "Project": { - "child": 11, + "child": 9, "cols": [ { "alias": "job", @@ -14740,11 +13585,11 @@ ], "roots": [ 5, - 12 + 10 ], "scalar_roots": [] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap (Sum acc)", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap", "from_logical": "L4", "id": "P4" }, @@ -15137,23 +13982,55 @@ "fields": [ { "dtype": { - "Plain": "timestamp" + "Plain": "utf8" }, - "name": "ts", - "nullable": false, + "name": "job", + "nullable": true, "table": null }, { "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" ] }, "name": "state", "nullable": false, "table": null - }, + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "producer": 7, + "role": "Input", + "window": "NotApplicable" + }, + { + "consumer": 9, + "data_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "grouping": "NotApplicable", + "intermediate_schema": { + "closed": true, + "fields": [ { "dtype": { "Plain": "utf8" @@ -15166,31 +14043,43 @@ "dtype": { "Plain": "utf8" }, - "name": "$promql_series_identity", + "name": "topk", "nullable": false, "table": null } ], - "time_index": 0, + "time_index": null, "unique_keys": [ [ - 3, 0 ] ] }, - "producer": 7, + "producer": 8, "role": "Input", "window": "NotApplicable" - }, + } + ], + "nodes": [ { - "consumer": 9, - "data_state": { - "primitive": "Raw", - "timing": "query_time" + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] }, - "grouping": "NotApplicable", - "intermediate_schema": { + "id": 0, + "materialization": "NotMaterialized", + "output_schema": { "closed": true, "fields": [ { @@ -15234,174 +14123,17 @@ ] ] }, - "producer": 8, - "role": "Input", - "window": "NotApplicable" - }, - { - "consumer": 10, - "data_state": { - "primitive": "SummaryState", + "output_state": { + "primitive": "Raw", "timing": "query_time" }, - "grouping": "NotApplicable", - "intermediate_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Sketch": [ - { - "algorithm": "CmsWithHeap", - "category": "TopK", - "params": { - "CmsWithHeap": { - "depth": 7, - "heap_size": 100, - "width": 272 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "name": "state", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "producer": 9, - "role": "Input", - "window": "NotApplicable" - }, - { - "consumer": 11, - "data_state": { - "primitive": "Raw", - "timing": "query_time" - }, - "grouping": "NotApplicable", - "intermediate_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "topk", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "producer": 10, - "role": "Input", - "window": "NotApplicable" - } - ], - "nodes": [ - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact over exact inputs" - } - ] - }, - "id": 0, - "materialization": "NotMaterialized", - "output_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "output_state": { - "primitive": "Raw", - "timing": "query_time" - }, - "payload": { - "NonASAP": { - "Scan": { - "predicates": [], - "schema": { - "closed": true, - "fields": [ + "payload": { + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ { "dtype": { "Plain": "timestamp" @@ -15897,25 +14629,6 @@ "output_schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" - ] - }, - "name": "state", - "nullable": false, - "table": null - }, { "dtype": { "Plain": "utf8" @@ -15926,17 +14639,29 @@ }, { "dtype": { - "Plain": "utf8" + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] }, - "name": "$promql_series_identity", + "name": "state", "nullable": false, "table": null } ], - "time_index": 0, + "time_index": null, "unique_keys": [ [ - 3, 0 ] ] @@ -15950,64 +14675,53 @@ "SummaryAgg": { "child": 6, "family": { - "ExactAggregate": [ - "Sum", - "Sum" + "Sketch": [ + { + "algorithm": "CmsWithHeap", + "category": "TopK", + "params": { + "CmsWithHeap": { + "depth": 7, + "heap_size": 100, + "width": 272 + } + } + }, + "PerSubpopulationInstance" ] }, "filter": null, "grouping": "PerSubpopulationInstance", "input": { - "item": null, + "item": { + "Column": { + "Named": "$promql_series_identity" + } + }, "weight": { "Column": "SampleValue" }, "weight_domain": { - "kind": "unknown_or_signed" + "kind": "samples_of_metric", + "metric": "http_requests_total" } }, - "reduction": "PerEntity" + "reduction": { + "Reduce": [ + 2 + ] + } } } } }, { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact accumulator over exact input" - } - ] - }, + "guarantee": null, "id": 8, "materialization": "NotMaterialized", "output_schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, { "dtype": { "Plain": "utf8" @@ -16020,15 +14734,14 @@ "dtype": { "Plain": "utf8" }, - "name": "$promql_series_identity", + "name": "topk", "nullable": false, "table": null } ], - "time_index": 0, + "time_index": null, "unique_keys": [ [ - 3, 0 ] ] @@ -16039,8 +14752,13 @@ }, "payload": { "ASAP": { - "FinalizeExactAccumulator": { - "child": 7 + "SummaryEstimate": { + "query": { + "TopK": { + "k": 10 + } + }, + "summary_input": 7 } } } @@ -16049,146 +14767,6 @@ "guarantee": null, "id": 9, "materialization": "NotMaterialized", - "output_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Sketch": [ - { - "algorithm": "CmsWithHeap", - "category": "TopK", - "params": { - "CmsWithHeap": { - "depth": 7, - "heap_size": 100, - "width": 272 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "name": "state", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "output_state": { - "primitive": "SummaryState", - "timing": "query_time" - }, - "payload": { - "ASAP": { - "SummaryAgg": { - "child": 8, - "family": { - "Sketch": [ - { - "algorithm": "CmsWithHeap", - "category": "TopK", - "params": { - "CmsWithHeap": { - "depth": 7, - "heap_size": 100, - "width": 272 - } - } - }, - "PerSubpopulationInstance" - ] - }, - "filter": null, - "grouping": "PerSubpopulationInstance", - "input": { - "item": { - "Column": { - "Named": "$promql_series_identity" - } - }, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" - } - }, - "reduction": { - "Reduce": [ - 2 - ] - } - } - } - } - }, - { - "guarantee": null, - "id": 10, - "materialization": "NotMaterialized", - "output_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "topk", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "output_state": { - "primitive": "Raw", - "timing": "query_time" - }, - "payload": { - "ASAP": { - "SummaryEstimate": { - "query": { - "TopK": { - "k": 10 - } - }, - "summary_input": 9 - } - } - } - }, - { - "guarantee": null, - "id": 11, - "materialization": "NotMaterialized", "output_schema": { "closed": true, "fields": [ @@ -16223,7 +14801,7 @@ "payload": { "NonASAP": { "Project": { - "child": 10, + "child": 8, "cols": [ { "alias": "job", @@ -16246,11 +14824,11 @@ ], "roots": [ 5, - 11 + 9 ], "scalar_roots": [] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap (Sum acc) · shared Scan", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap · shared Scan", "from_logical": "L5", "id": "P5" }, @@ -16583,119 +15161,6 @@ "timing": "query_time" }, "grouping": "NotApplicable", - "intermediate_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" - ] - }, - "name": "state", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "producer": 6, - "role": "Input", - "window": "NotApplicable" - }, - { - "consumer": 8, - "data_state": { - "primitive": "Raw", - "timing": "query_time" - }, - "grouping": "NotApplicable", - "intermediate_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "producer": 7, - "role": "Input", - "window": "NotApplicable" - }, - { - "consumer": 9, - "data_state": { - "primitive": "SummaryState", - "timing": "query_time" - }, - "grouping": "NotApplicable", "intermediate_schema": { "closed": true, "fields": [ @@ -16736,12 +15201,12 @@ ] ] }, - "producer": 8, + "producer": 6, "role": "Input", "window": "NotApplicable" }, { - "consumer": 10, + "consumer": 8, "data_state": { "primitive": "Raw", "timing": "query_time" @@ -16774,7 +15239,7 @@ ] ] }, - "producer": 9, + "producer": 7, "role": "Input", "window": "NotApplicable" } @@ -16919,167 +15384,7 @@ } ] }, - "id": 1, - "materialization": "NotMaterialized", - "output_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "output_state": { - "primitive": "Raw", - "timing": "query_time" - }, - "payload": { - "NonASAP": { - "TimeRange": { - "child": 0, - "kind": "range", - "range": { - "nanos": 0, - "secs": 60 - } - } - } - } - }, - { - "guarantee": null, - "id": 2, - "materialization": "NotMaterialized", - "output_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Rate", - "Rate" - ] - }, - "name": "state", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "output_state": { - "primitive": "SummaryState", - "timing": "query_time" - }, - "payload": { - "ASAP": { - "SummaryAgg": { - "child": 1, - "family": { - "ExactAggregate": [ - "Rate", - "Rate" - ] - }, - "filter": null, - "grouping": "PerSubpopulationInstance", - "input": { - "item": null, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" - } - }, - "reduction": "PerEntity" - } - } - } - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact accumulator over exact input" - } - ] - }, - "id": 3, + "id": 1, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -17130,43 +15435,65 @@ "timing": "query_time" }, "payload": { - "ASAP": { - "FinalizeExactAccumulator": { - "child": 2 + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } } } } }, { "guarantee": null, - "id": 4, + "id": 2, "materialization": "NotMaterialized", "output_schema": { "closed": true, "fields": [ { "dtype": { - "Plain": "utf8" + "Plain": "timestamp" }, - "name": "job", - "nullable": true, + "name": "ts", + "nullable": false, "table": null }, { "dtype": { "ExactAggregate": [ - "Sum", - "Sum" + "Rate", + "Rate" ] }, "name": "state", "nullable": false, "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null } ], - "time_index": null, + "time_index": 0, "unique_keys": [ [ + 3, 0 ] ] @@ -17178,11 +15505,11 @@ "payload": { "ASAP": { "SummaryAgg": { - "child": 3, + "child": 1, "family": { "ExactAggregate": [ - "Sum", - "Sum" + "Rate", + "Rate" ] }, "filter": null, @@ -17196,11 +15523,7 @@ "kind": "unknown_or_signed" } }, - "reduction": { - "Reduce": [ - 2 - ] - } + "reduction": "PerEntity" } } } @@ -17221,11 +15544,27 @@ } ] }, - "id": 5, + "id": 3, "materialization": "NotMaterialized", "output_schema": { "closed": true, "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, { "dtype": { "Plain": "utf8" @@ -17236,16 +15575,17 @@ }, { "dtype": { - "Plain": "float64" + "Plain": "utf8" }, - "name": "sum", + "name": "$promql_series_identity", "nullable": false, "table": null } ], - "time_index": null, + "time_index": 0, "unique_keys": [ [ + 3, 0 ] ] @@ -17257,24 +15597,24 @@ "payload": { "ASAP": { "FinalizeExactAccumulator": { - "child": 4 + "child": 2 } } } }, { "guarantee": null, - "id": 6, + "id": 4, "materialization": "NotMaterialized", "output_schema": { "closed": true, "fields": [ { "dtype": { - "Plain": "timestamp" + "Plain": "utf8" }, - "name": "ts", - "nullable": false, + "name": "job", + "nullable": true, "table": null }, { @@ -17287,28 +15627,11 @@ "name": "state", "nullable": false, "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null } ], - "time_index": 0, + "time_index": null, "unique_keys": [ [ - 3, 0 ] ] @@ -17320,7 +15643,7 @@ "payload": { "ASAP": { "SummaryAgg": { - "child": 1, + "child": 3, "family": { "ExactAggregate": [ "Sum", @@ -17338,7 +15661,11 @@ "kind": "unknown_or_signed" } }, - "reduction": "PerEntity" + "reduction": { + "Reduce": [ + 2 + ] + } } } } @@ -17359,27 +15686,11 @@ } ] }, - "id": 7, + "id": 5, "materialization": "NotMaterialized", "output_schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, { "dtype": { "Plain": "utf8" @@ -17390,17 +15701,16 @@ }, { "dtype": { - "Plain": "utf8" + "Plain": "float64" }, - "name": "$promql_series_identity", + "name": "sum", "nullable": false, "table": null } ], - "time_index": 0, + "time_index": null, "unique_keys": [ [ - 3, 0 ] ] @@ -17412,14 +15722,14 @@ "payload": { "ASAP": { "FinalizeExactAccumulator": { - "child": 6 + "child": 4 } } } }, { "guarantee": null, - "id": 8, + "id": 6, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -17468,7 +15778,7 @@ "payload": { "ASAP": { "SummaryAgg": { - "child": 7, + "child": 1, "family": { "Sketch": [ { @@ -17497,7 +15807,8 @@ "Column": "SampleValue" }, "weight_domain": { - "kind": "unknown_or_signed" + "kind": "samples_of_metric", + "metric": "http_requests_total" } }, "reduction": { @@ -17511,7 +15822,7 @@ }, { "guarantee": null, - "id": 9, + "id": 7, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -17552,14 +15863,14 @@ "k": 10 } }, - "summary_input": 8 + "summary_input": 6 } } } }, { "guarantee": null, - "id": 10, + "id": 8, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -17595,7 +15906,7 @@ "payload": { "NonASAP": { "Project": { - "child": 9, + "child": 7, "cols": [ { "alias": "job", @@ -17618,11 +15929,11 @@ ], "roots": [ 5, - 10 + 8 ], "scalar_roots": [] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap (Sum acc) · shared Scan, TimeRange", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap · shared Scan, TimeRange", "from_logical": "L6", "id": "P6" }, @@ -17798,103 +16109,7 @@ "window": "NotApplicable" }, { - "consumer": 4, - "data_state": { - "primitive": "Raw", - "timing": "query_time" - }, - "grouping": "NotApplicable", - "intermediate_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "producer": 3, - "role": "Input", - "window": "NotApplicable" - }, - { - "consumer": 5, - "data_state": { - "primitive": "SummaryState", - "timing": "query_time" - }, - "grouping": "NotApplicable", - "intermediate_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" - ] - }, - "name": "state", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "producer": 4, - "role": "Input", - "window": "NotApplicable" - }, - { - "consumer": 7, + "consumer": 4, "data_state": { "primitive": "Raw", "timing": "query_time" @@ -17944,36 +16159,20 @@ ] ] }, - "producer": 6, + "producer": 3, "role": "Input", "window": "NotApplicable" }, { - "consumer": 8, + "consumer": 5, "data_state": { - "primitive": "Raw", + "primitive": "SummaryState", "timing": "query_time" }, "grouping": "NotApplicable", "intermediate_schema": { "closed": true, "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, { "dtype": { "Plain": "utf8" @@ -17984,29 +16183,31 @@ }, { "dtype": { - "Plain": "utf8" + "ExactAggregate": [ + "Sum", + "Sum" + ] }, - "name": "$promql_series_identity", + "name": "state", "nullable": false, "table": null } ], - "time_index": 0, + "time_index": null, "unique_keys": [ [ - 3, 0 ] ] }, - "producer": 7, + "producer": 4, "role": "Input", "window": "NotApplicable" }, { - "consumer": 9, + "consumer": 7, "data_state": { - "primitive": "SummaryState", + "primitive": "Raw", "timing": "query_time" }, "grouping": "NotApplicable", @@ -18023,12 +16224,9 @@ }, { "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" - ] + "Plain": "float64" }, - "name": "state", + "name": "value", "nullable": false, "table": null }, @@ -18057,12 +16255,12 @@ ] ] }, - "producer": 8, + "producer": 6, "role": "Input", "window": "NotApplicable" }, { - "consumer": 10, + "consumer": 8, "data_state": { "primitive": "Raw", "timing": "query_time" @@ -18112,12 +16310,12 @@ ] ] }, - "producer": 9, + "producer": 7, "role": "Input", "window": "NotApplicable" }, { - "consumer": 11, + "consumer": 9, "data_state": { "primitive": "SummaryState", "timing": "query_time" @@ -18163,12 +16361,12 @@ ] ] }, - "producer": 10, + "producer": 8, "role": "Input", "window": "NotApplicable" }, { - "consumer": 12, + "consumer": 10, "data_state": { "primitive": "Raw", "timing": "query_time" @@ -18201,7 +16399,7 @@ ] ] }, - "producer": 11, + "producer": 9, "role": "Input", "window": "NotApplicable" } @@ -18663,213 +16861,16 @@ }, { "dtype": { - "Plain": "float64" - }, - "name": "sum", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "output_state": { - "primitive": "Raw", - "timing": "query_time" - }, - "payload": { - "ASAP": { - "FinalizeExactAccumulator": { - "child": 4 - } - } - } - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact over exact inputs" - } - ] - }, - "id": 6, - "materialization": "NotMaterialized", - "output_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "output_state": { - "primitive": "Raw", - "timing": "query_time" - }, - "payload": { - "NonASAP": { - "Scan": { - "predicates": [], - "schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "source": { - "TimeSeries": { - "metric": "http_requests_total" - } - } - } - } - } - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact over exact inputs" - } - ] - }, - "id": 7, - "materialization": "NotMaterialized", - "output_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" + "Plain": "float64" }, - "name": "$promql_series_identity", + "name": "sum", "nullable": false, "table": null } ], - "time_index": 0, + "time_index": null, "unique_keys": [ [ - 3, 0 ] ] @@ -18879,21 +16880,30 @@ "timing": "query_time" }, "payload": { - "NonASAP": { - "TimeRange": { - "child": 6, - "kind": "range", - "range": { - "nanos": 0, - "secs": 60 - } + "ASAP": { + "FinalizeExactAccumulator": { + "child": 4 } } } }, { - "guarantee": null, - "id": 8, + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 6, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -18908,12 +16918,9 @@ }, { "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" - ] + "Plain": "float64" }, - "name": "state", + "name": "value", "nullable": false, "table": null }, @@ -18943,31 +16950,62 @@ ] }, "output_state": { - "primitive": "SummaryState", + "primitive": "Raw", "timing": "query_time" }, "payload": { - "ASAP": { - "SummaryAgg": { - "child": 7, - "family": { - "ExactAggregate": [ - "Sum", - "Sum" + "NonASAP": { + "Scan": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "$promql_series_identity", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [ + [ + 3, + 0 + ] ] }, - "filter": null, - "grouping": "PerSubpopulationInstance", - "input": { - "item": null, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" + "source": { + "TimeSeries": { + "metric": "http_requests_total" } - }, - "reduction": "PerEntity" + } } } } @@ -18984,11 +17022,11 @@ "provenance": [ { "kind": "exact", - "reason": "exact accumulator over exact input" + "reason": "exact over exact inputs" } ] }, - "id": 9, + "id": 7, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -19039,16 +17077,21 @@ "timing": "query_time" }, "payload": { - "ASAP": { - "FinalizeExactAccumulator": { - "child": 8 + "NonASAP": { + "TimeRange": { + "child": 6, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } } } } }, { "guarantee": null, - "id": 10, + "id": 8, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -19097,7 +17140,7 @@ "payload": { "ASAP": { "SummaryAgg": { - "child": 9, + "child": 7, "family": { "Sketch": [ { @@ -19126,7 +17169,8 @@ "Column": "SampleValue" }, "weight_domain": { - "kind": "unknown_or_signed" + "kind": "samples_of_metric", + "metric": "http_requests_total" } }, "reduction": { @@ -19140,7 +17184,7 @@ }, { "guarantee": null, - "id": 11, + "id": 9, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -19181,14 +17225,14 @@ "k": 10 } }, - "summary_input": 10 + "summary_input": 8 } } } }, { "guarantee": null, - "id": 12, + "id": 10, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -19224,7 +17268,7 @@ "payload": { "NonASAP": { "Project": { - "child": 11, + "child": 9, "cols": [ { "alias": "job", @@ -19247,11 +17291,11 @@ ], "roots": [ 5, - 12 + 10 ], "scalar_roots": [] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap (Sum acc)", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap", "from_logical": "L7", "id": "P7" }, @@ -19489,120 +17533,8 @@ }, "grouping": "NotApplicable", "intermediate_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" - ] - }, - "name": "state", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "producer": 4, - "role": "Input", - "window": "NotApplicable" - }, - { - "consumer": 6, - "data_state": { - "primitive": "Raw", - "timing": "query_time" - }, - "grouping": "NotApplicable", - "intermediate_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "producer": 0, - "role": "Input", - "window": "NotApplicable" - }, - { - "consumer": 7, - "data_state": { - "primitive": "Raw", - "timing": "query_time" - }, - "grouping": "NotApplicable", - "intermediate_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, + "closed": true, + "fields": [ { "dtype": { "Plain": "utf8" @@ -19613,29 +17545,31 @@ }, { "dtype": { - "Plain": "utf8" + "ExactAggregate": [ + "Sum", + "Sum" + ] }, - "name": "$promql_series_identity", + "name": "state", "nullable": false, "table": null } ], - "time_index": 0, + "time_index": null, "unique_keys": [ [ - 3, 0 ] ] }, - "producer": 6, + "producer": 4, "role": "Input", "window": "NotApplicable" }, { - "consumer": 8, + "consumer": 6, "data_state": { - "primitive": "SummaryState", + "primitive": "Raw", "timing": "query_time" }, "grouping": "NotApplicable", @@ -19652,12 +17586,9 @@ }, { "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" - ] + "Plain": "float64" }, - "name": "state", + "name": "value", "nullable": false, "table": null }, @@ -19686,12 +17617,12 @@ ] ] }, - "producer": 7, + "producer": 0, "role": "Input", "window": "NotApplicable" }, { - "consumer": 9, + "consumer": 7, "data_state": { "primitive": "Raw", "timing": "query_time" @@ -19741,12 +17672,12 @@ ] ] }, - "producer": 8, + "producer": 6, "role": "Input", "window": "NotApplicable" }, { - "consumer": 10, + "consumer": 8, "data_state": { "primitive": "SummaryState", "timing": "query_time" @@ -19792,12 +17723,12 @@ ] ] }, - "producer": 9, + "producer": 7, "role": "Input", "window": "NotApplicable" }, { - "consumer": 11, + "consumer": 9, "data_state": { "primitive": "Raw", "timing": "query_time" @@ -19830,7 +17761,7 @@ ] ] }, - "producer": 10, + "producer": 8, "role": "Input", "window": "NotApplicable" } @@ -20097,223 +18028,24 @@ "ASAP": { "SummaryAgg": { "child": 1, - "family": { - "ExactAggregate": [ - "Rate", - "Rate" - ] - }, - "filter": null, - "grouping": "PerSubpopulationInstance", - "input": { - "item": null, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" - } - }, - "reduction": "PerEntity" - } - } - } - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact accumulator over exact input" - } - ] - }, - "id": 3, - "materialization": "NotMaterialized", - "output_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "output_state": { - "primitive": "Raw", - "timing": "query_time" - }, - "payload": { - "ASAP": { - "FinalizeExactAccumulator": { - "child": 2 - } - } - } - }, - { - "guarantee": null, - "id": 4, - "materialization": "NotMaterialized", - "output_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" - ] - }, - "name": "state", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "output_state": { - "primitive": "SummaryState", - "timing": "query_time" - }, - "payload": { - "ASAP": { - "SummaryAgg": { - "child": 3, - "family": { - "ExactAggregate": [ - "Sum", - "Sum" - ] - }, - "filter": null, - "grouping": "PerSubpopulationInstance", - "input": { - "item": null, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" - } - }, - "reduction": { - "Reduce": [ - 2 - ] - } - } - } - } - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact accumulator over exact input" - } - ] - }, - "id": 5, - "materialization": "NotMaterialized", - "output_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "sum", - "nullable": false, - "table": null - } - ], - "time_index": null, - "unique_keys": [ - [ - 0 - ] - ] - }, - "output_state": { - "primitive": "Raw", - "timing": "query_time" - }, - "payload": { - "ASAP": { - "FinalizeExactAccumulator": { - "child": 4 + "family": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "filter": null, + "grouping": "PerSubpopulationInstance", + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" } } } @@ -20330,11 +18062,11 @@ "provenance": [ { "kind": "exact", - "reason": "exact over exact inputs" + "reason": "exact accumulator over exact input" } ] }, - "id": 6, + "id": 3, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -20385,31 +18117,26 @@ "timing": "query_time" }, "payload": { - "NonASAP": { - "TimeRange": { - "child": 0, - "kind": "range", - "range": { - "nanos": 0, - "secs": 60 - } + "ASAP": { + "FinalizeExactAccumulator": { + "child": 2 } } } }, { "guarantee": null, - "id": 7, + "id": 4, "materialization": "NotMaterialized", "output_schema": { "closed": true, "fields": [ { "dtype": { - "Plain": "timestamp" + "Plain": "utf8" }, - "name": "ts", - "nullable": false, + "name": "job", + "nullable": true, "table": null }, { @@ -20422,28 +18149,11 @@ "name": "state", "nullable": false, "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null } ], - "time_index": 0, + "time_index": null, "unique_keys": [ [ - 3, 0 ] ] @@ -20455,7 +18165,7 @@ "payload": { "ASAP": { "SummaryAgg": { - "child": 6, + "child": 3, "family": { "ExactAggregate": [ "Sum", @@ -20473,7 +18183,11 @@ "kind": "unknown_or_signed" } }, - "reduction": "PerEntity" + "reduction": { + "Reduce": [ + 2 + ] + } } } } @@ -20494,7 +18208,64 @@ } ] }, - "id": 8, + "id": 5, + "materialization": "NotMaterialized", + "output_schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "job", + "nullable": true, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "output_state": { + "primitive": "Raw", + "timing": "query_time" + }, + "payload": { + "ASAP": { + "FinalizeExactAccumulator": { + "child": 4 + } + } + } + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "exact over exact inputs" + } + ] + }, + "id": 6, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -20545,16 +18316,21 @@ "timing": "query_time" }, "payload": { - "ASAP": { - "FinalizeExactAccumulator": { - "child": 7 + "NonASAP": { + "TimeRange": { + "child": 0, + "kind": "range", + "range": { + "nanos": 0, + "secs": 60 + } } } } }, { "guarantee": null, - "id": 9, + "id": 7, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -20603,7 +18379,7 @@ "payload": { "ASAP": { "SummaryAgg": { - "child": 8, + "child": 6, "family": { "Sketch": [ { @@ -20632,7 +18408,8 @@ "Column": "SampleValue" }, "weight_domain": { - "kind": "unknown_or_signed" + "kind": "samples_of_metric", + "metric": "http_requests_total" } }, "reduction": { @@ -20646,7 +18423,7 @@ }, { "guarantee": null, - "id": 10, + "id": 8, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -20687,14 +18464,14 @@ "k": 10 } }, - "summary_input": 9 + "summary_input": 7 } } } }, { "guarantee": null, - "id": 11, + "id": 9, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -20730,7 +18507,7 @@ "payload": { "NonASAP": { "Project": { - "child": 10, + "child": 8, "cols": [ { "alias": "job", @@ -20753,11 +18530,11 @@ ], "roots": [ 5, - 11 + 9 ], "scalar_roots": [] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap (Sum acc) · shared Scan", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap · shared Scan", "from_logical": "L8", "id": "P8" }, @@ -20894,111 +18671,17 @@ }, { "dtype": { - "ExactAggregate": [ - "Rate", - "Rate" - ] - }, - "name": "state", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "producer": 2, - "role": "Input", - "window": "NotApplicable" - }, - { - "consumer": 4, - "data_state": { - "primitive": "Raw", - "timing": "query_time" - }, - "grouping": "NotApplicable", - "intermediate_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "producer": 3, - "role": "Input", - "window": "NotApplicable" - }, - { - "consumer": 5, - "data_state": { - "primitive": "SummaryState", - "timing": "query_time" - }, - "grouping": "NotApplicable", - "intermediate_schema": { - "closed": true, - "fields": [ - { - "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "state", + "nullable": false, + "table": null + }, + { + "dtype": { "Plain": "utf8" }, "name": "job", @@ -21007,29 +18690,27 @@ }, { "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" - ] + "Plain": "utf8" }, - "name": "state", + "name": "$promql_series_identity", "nullable": false, "table": null } ], - "time_index": null, + "time_index": 0, "unique_keys": [ [ + 3, 0 ] ] }, - "producer": 4, + "producer": 2, "role": "Input", "window": "NotApplicable" }, { - "consumer": 6, + "consumer": 4, "data_state": { "primitive": "Raw", "timing": "query_time" @@ -21079,12 +18760,12 @@ ] ] }, - "producer": 1, + "producer": 3, "role": "Input", "window": "NotApplicable" }, { - "consumer": 7, + "consumer": 5, "data_state": { "primitive": "SummaryState", "timing": "query_time" @@ -21095,10 +18776,10 @@ "fields": [ { "dtype": { - "Plain": "timestamp" + "Plain": "utf8" }, - "name": "ts", - "nullable": false, + "name": "job", + "nullable": true, "table": null }, { @@ -21111,38 +18792,21 @@ "name": "state", "nullable": false, "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null } ], - "time_index": 0, + "time_index": null, "unique_keys": [ [ - 3, 0 ] ] }, - "producer": 6, + "producer": 4, "role": "Input", "window": "NotApplicable" }, { - "consumer": 8, + "consumer": 6, "data_state": { "primitive": "Raw", "timing": "query_time" @@ -21192,12 +18856,12 @@ ] ] }, - "producer": 7, + "producer": 1, "role": "Input", "window": "NotApplicable" }, { - "consumer": 9, + "consumer": 7, "data_state": { "primitive": "SummaryState", "timing": "query_time" @@ -21243,12 +18907,12 @@ ] ] }, - "producer": 8, + "producer": 6, "role": "Input", "window": "NotApplicable" }, { - "consumer": 10, + "consumer": 8, "data_state": { "primitive": "Raw", "timing": "query_time" @@ -21281,7 +18945,7 @@ ] ] }, - "producer": 9, + "producer": 7, "role": "Input", "window": "NotApplicable" } @@ -21773,161 +19437,6 @@ "guarantee": null, "id": 6, "materialization": "NotMaterialized", - "output_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "ExactAggregate": [ - "Sum", - "Sum" - ] - }, - "name": "state", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "output_state": { - "primitive": "SummaryState", - "timing": "query_time" - }, - "payload": { - "ASAP": { - "SummaryAgg": { - "child": 1, - "family": { - "ExactAggregate": [ - "Sum", - "Sum" - ] - }, - "filter": null, - "grouping": "PerSubpopulationInstance", - "input": { - "item": null, - "weight": { - "Column": "SampleValue" - }, - "weight_domain": { - "kind": "unknown_or_signed" - } - }, - "reduction": "PerEntity" - } - } - } - }, - { - "guarantee": { - "bound": { - "op": "zero" - }, - "failure_probability": { - "op": "zero" - }, - "metric": "absolute_value", - "provenance": [ - { - "kind": "exact", - "reason": "exact accumulator over exact input" - } - ] - }, - "id": 7, - "materialization": "NotMaterialized", - "output_schema": { - "closed": true, - "fields": [ - { - "dtype": { - "Plain": "timestamp" - }, - "name": "ts", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "float64" - }, - "name": "value", - "nullable": false, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "job", - "nullable": true, - "table": null - }, - { - "dtype": { - "Plain": "utf8" - }, - "name": "$promql_series_identity", - "nullable": false, - "table": null - } - ], - "time_index": 0, - "unique_keys": [ - [ - 3, - 0 - ] - ] - }, - "output_state": { - "primitive": "Raw", - "timing": "query_time" - }, - "payload": { - "ASAP": { - "FinalizeExactAccumulator": { - "child": 6 - } - } - } - }, - { - "guarantee": null, - "id": 8, - "materialization": "NotMaterialized", "output_schema": { "closed": true, "fields": [ @@ -21975,7 +19484,7 @@ "payload": { "ASAP": { "SummaryAgg": { - "child": 7, + "child": 1, "family": { "Sketch": [ { @@ -22004,7 +19513,8 @@ "Column": "SampleValue" }, "weight_domain": { - "kind": "unknown_or_signed" + "kind": "samples_of_metric", + "metric": "http_requests_total" } }, "reduction": { @@ -22018,7 +19528,7 @@ }, { "guarantee": null, - "id": 9, + "id": 7, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -22059,14 +19569,14 @@ "k": 10 } }, - "summary_input": 8 + "summary_input": 6 } } } }, { "guarantee": null, - "id": 10, + "id": 8, "materialization": "NotMaterialized", "output_schema": { "closed": true, @@ -22102,7 +19612,7 @@ "payload": { "NonASAP": { "Project": { - "child": 9, + "child": 7, "cols": [ { "alias": "job", @@ -22125,11 +19635,11 @@ ], "roots": [ 5, - 10 + 8 ], "scalar_roots": [] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap (Sum acc) · shared Scan, TimeRange", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap · shared Scan, TimeRange", "from_logical": "L9", "id": "P9" } @@ -22213,27 +19723,27 @@ "P2": { "per_node": { "0": { - "cost": 46.4, + "cost": 46.4000096, "detail": "scan 4000000 samples", "multiplier": 2 }, "1": { - "cost": 4.0, + "cost": 4.0000048, "detail": "time range 60s: pass 4000000 rows", "multiplier": 1 }, "10": { - "cost": 0.001, + "cost": 0.0010048, "detail": "limit to 1000 rows", "multiplier": 1 }, "11": { - "cost": 0.001, + "cost": 0.0010048, "detail": "operator: pass 1000 rows", "multiplier": 1 }, "2": { - "cost": 4.0, + "cost": 4.8, "detail": "build exact Rate accumulator into 1000000 states: 4000000 rows x depth 1", "multiplier": 1 }, @@ -22243,7 +19753,7 @@ "multiplier": 1 }, "4": { - "cost": 1.0, + "cost": 1.00008, "detail": "build exact Sum accumulator into 100 states: 1000000 rows x depth 1", "multiplier": 1 }, @@ -22253,12 +19763,12 @@ "multiplier": 1 }, "6": { - "cost": 4.0, + "cost": 4.0000048, "detail": "time range 60s: pass 4000000 rows", "multiplier": 1 }, "7": { - "cost": 4.0, + "cost": 4.8, "detail": "build exact Sum accumulator into 1000000 states: 4000000 rows x depth 1", "multiplier": 1 }, @@ -22268,34 +19778,34 @@ "multiplier": 1 }, "9": { - "cost": 14.0, + "cost": 14.048, "detail": "sort 1000000 rows in 100 partitions", "multiplier": 1 } }, "source": "analytical-cost-v1 (illustrative statistics)", - "total": 79.40210000000002, + "total": 81.05020880000001, "unit": "cpu_ms_per_workload_evaluation" }, "P3": { "per_node": { "0": { - "cost": 46.4, + "cost": 46.4000096, "detail": "scan 4000000 samples", "multiplier": 2 }, "1": { - "cost": 8.0, + "cost": 8.0000096, "detail": "time range 60s: pass 4000000 rows", "multiplier": 2 }, "10": { - "cost": 0.001, + "cost": 0.0010048, "detail": "operator: pass 1000 rows", "multiplier": 1 }, "2": { - "cost": 4.0, + "cost": 4.8, "detail": "build exact Rate accumulator into 1000000 states: 4000000 rows x depth 1", "multiplier": 1 }, @@ -22305,7 +19815,7 @@ "multiplier": 1 }, "4": { - "cost": 1.0, + "cost": 1.00008, "detail": "build exact Sum accumulator into 100 states: 1000000 rows x depth 1", "multiplier": 1 }, @@ -22315,7 +19825,7 @@ "multiplier": 1 }, "6": { - "cost": 4.0, + "cost": 4.8, "detail": "build exact Sum accumulator into 1000000 states: 4000000 rows x depth 1", "multiplier": 1 }, @@ -22325,35 +19835,35 @@ "multiplier": 1 }, "8": { - "cost": 14.0, + "cost": 14.048, "detail": "sort 1000000 rows in 100 partitions", "multiplier": 1 }, "9": { - "cost": 0.001, + "cost": 0.0010048, "detail": "limit to 1000 rows", "multiplier": 1 } }, "source": "analytical-cost-v1 (illustrative statistics)", - "total": 79.40210000000002, + "total": 81.05020880000001, "unit": "cpu_ms_per_workload_evaluation" } }, "rejected": [ { "id": "P4", - "reason": "q2: CmsWithHeap needs non-negative update weights, and these are not proven non-negative", + "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target EpsilonDelta { epsilon: 0.01, delta: 0.001 }", "valid": false }, { "id": "P5", - "reason": "q2: CmsWithHeap needs non-negative update weights, and these are not proven non-negative", + "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target EpsilonDelta { epsilon: 0.01, delta: 0.001 }", "valid": false }, { "id": "P6", - "reason": "q2: CmsWithHeap needs non-negative update weights, and these are not proven non-negative", + "reason": "q2: the output has no guarantee, and the deployment has no evidence that it meets target EpsilonDelta { epsilon: 0.01, delta: 0.001 }", "valid": false }, { @@ -22373,12 +19883,12 @@ }, { "id": "P2", - "reason": "costlier: 79.402 vs 79.402 cpu_ms_per_workload_evaluation", + "reason": "tied with the selected candidate P1 at 81.050 cpu_ms_per_workload_evaluation; lost on enumeration order", "valid": true }, { "id": "P3", - "reason": "costlier: 79.402 vs 79.402 cpu_ms_per_workload_evaluation", + "reason": "tied with the selected candidate P1 at 81.050 cpu_ms_per_workload_evaluation; lost on enumeration order", "valid": true } ], From 9199e2f62da3020b0ea80b62b47696705212612e Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:48:50 +0000 Subject: [PATCH 05/15] feat(plan-selection): price candidates per second of wall time Plan selection now prices a candidate per second, so queries with different recurrences add up, and takes each query's recurrence: - A NotMaterialized node is charged once per consumer at the sum of their rates (a shared NotMaterialized input still costs what reading it separately does). - A MatAtQuery node is computed once at the combined rate of the queries reaching it; queries at the same interval run together. - A MatAtIngestion node is priced over one second of arriving rows, once per second. - One-off, scheduled and unknown queries are spread over horizon_s. ResourceCalibration gains horizon_s (default 3600 s) and cost_per_retained_byte_second, the price of state kept between computations. Every materialized node's kept state is charged per byte per second: twice its output at ingestion time (the window being built and the completed one), once at query time. Memory held only during a query gets no per-second charge. select_plan, run_stage1_to_stage3 and run_stage1_to_stage3_from_pass1 take the recurrences; the planner pipeline and stage_pipeline read them from the workload. The cost model is described in the stage3_plan_selection module docs. The viewer fixtures are regenerated. --- .../asap-aware-mapping/src/analytical_cost.rs | 26 + .../src/pass/stage1_to_stage3.rs | 14 +- .../src/physical_plan_cost_model.rs | 8 + .../src/stage3_plan_selection.rs | 605 +++++++++++++++--- crates/asap-aware-mapping/src/test_support.rs | 1 + .../src/physical_planner/candidates.rs | 1 + .../tests/common/mod.rs | 1 + crates/devtools/src/bin/stage_pipeline.rs | 12 +- crates/devtools/src/planning_stages.rs | 22 +- crates/frontend-promql/tests/support.rs | 2 + crates/frontend-sql/tests/pearson_corr.rs | 1 + crates/integration-tests/src/lib.rs | 1 + .../tests/planner_layering_example1.rs | 7 + crates/planner/tests/e2e_plan.rs | 7 +- crates/planner/tests/plan_stages.rs | 5 + crates/planner/tests/summary_sharing.rs | 5 + .../examples/planner-layering-example1.json | 166 ++--- tools/dag-viewer/examples/scalar-queries.json | 54 +- 18 files changed, 727 insertions(+), 211 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 0756106b8..1dd2f9eea 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -151,25 +151,51 @@ pub fn cache_hit_ratios( /// Conversion from physical dimensions to one deployment-specific objective. /// Memory's coefficient means cost units per retained byte over this model's /// explicit comparison scope; it is not mixed with a rate implicitly. +/// +/// Plan selection prices per second of wall time and also reads +/// `cost_per_retained_byte_second`, the price of state kept across +/// evaluations per byte per second, and `horizon_s`, the seconds over which +/// it spreads one-off, scheduled and unknown queries. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ResourceCalibration { pub cost_per_cpu_op: f64, pub cost_per_scan_byte: f64, pub cost_per_retained_byte: f64, + #[serde(default)] + pub cost_per_retained_byte_second: f64, + #[serde(default = "default_horizon_s")] + pub horizon_s: f64, pub version: String, } +/// One hour. +pub const DEFAULT_HORIZON_S: f64 = 3_600.0; + +fn default_horizon_s() -> f64 { + DEFAULT_HORIZON_S +} + impl ResourceCalibration { pub fn validate(&self) -> Result<(), AnalyticalCostError> { for (name, value) in [ ("cost_per_cpu_op", self.cost_per_cpu_op), ("cost_per_scan_byte", self.cost_per_scan_byte), ("cost_per_retained_byte", self.cost_per_retained_byte), + ( + "cost_per_retained_byte_second", + self.cost_per_retained_byte_second, + ), ] { if !value.is_finite() || value < 0.0 { return Err(AnalyticalCostError::InvalidCalibration(name, value)); } } + if !self.horizon_s.is_finite() || self.horizon_s <= 0.0 { + return Err(AnalyticalCostError::InvalidCalibration( + "horizon_s", + self.horizon_s, + )); + } if self.cost_per_cpu_op == 0.0 && self.cost_per_scan_byte == 0.0 && self.cost_per_retained_byte == 0.0 diff --git a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs index 054b46380..dd6050c47 100644 --- a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs +++ b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs @@ -31,7 +31,7 @@ use asap_types::ir::schema_support::with_promql_series_identity; use asap_types::ir::{OperatorNode, QueryRoot}; use asap_types::post_asap::SketchAlgorithm; use asap_types::types::AccuracyTarget; -use asap_types::workload::{DataWorkload, QueryLanguage}; +use asap_types::workload::{DataWorkload, QueryLanguage, QueryRecurrence}; use super::{OptimizationInput, OptimizationPass, OptimizeError, PlanOutput, QueryPlan}; use crate::asap_aware_cse::identical_expression_variants; @@ -137,6 +137,7 @@ impl Stage1ToStage3Run { pub fn run_stage1_to_stage3( roots: Vec<(Id, QueryRoot)>, targets: &[Option], + recurrence: &[QueryRecurrence], data: &DataWorkload, models: PlanningModels<'_>, ) -> Result, PerComputationCandidateError> { @@ -144,6 +145,7 @@ pub fn run_stage1_to_stage3( Ok(run_stage1_to_stage3_from_pass1( per_computation, targets, + recurrence, data, models, )) @@ -157,6 +159,7 @@ pub fn run_stage1_to_stage3( pub fn run_stage1_to_stage3_from_pass1( per_computation: PerComputationCandidates, targets: &[Option], + recurrence: &[QueryRecurrence], data: &DataWorkload, models: PlanningModels<'_>, ) -> Stage1ToStage3Run { @@ -236,7 +239,7 @@ pub fn run_stage1_to_stage3_from_pass1( break; } } - let selection = select_plan(&physical, targets, data, models); + let selection = select_plan(&physical, targets, recurrence, data, models); Stage1ToStage3Run { stage1_pass1_per_computation: per_computation, combinations, @@ -400,8 +403,13 @@ impl OptimizationPass for Stage1ToStage3Pass { .entries() .map(|entry| Some(entry.requirements.accuracy.target())) .collect(); + let recurrence: Vec<_> = workload + .query_workload() + .entries() + .map(|entry| entry.recurrence) + .collect(); let data = workload.data_workload().cloned().unwrap_or_default(); - let run = run_stage1_to_stage3(roots, &targets, &data, input.models)?; + let run = run_stage1_to_stage3(roots, &targets, &recurrence, &data, input.models)?; let Stage1ToStage3Run { stage2_physical_asap_optimization: physical, stage3_plan_selection: selection, diff --git a/crates/asap-aware-mapping/src/physical_plan_cost_model.rs b/crates/asap-aware-mapping/src/physical_plan_cost_model.rs index 4670c5511..defabfb01 100644 --- a/crates/asap-aware-mapping/src/physical_plan_cost_model.rs +++ b/crates/asap-aware-mapping/src/physical_plan_cost_model.rs @@ -602,6 +602,8 @@ mod tests { cost_per_cpu_op: 1.0, cost_per_scan_byte: 1.0, cost_per_retained_byte: 1.0, + cost_per_retained_byte_second: 0.0, + horizon_s: crate::analytical_cost::DEFAULT_HORIZON_S, version: "test-v1".into(), } } @@ -688,6 +690,8 @@ mod tests { cost_per_cpu_op: 0.0, cost_per_scan_byte: 0.0, cost_per_retained_byte: 0.0, + cost_per_retained_byte_second: 0.0, + horizon_s: crate::analytical_cost::DEFAULT_HORIZON_S, version: "unused-base-v1".into(), }; let candidates = @@ -815,6 +819,8 @@ mod tests { cost_per_cpu_op: 0.0, cost_per_scan_byte: 0.0, cost_per_retained_byte: 0.0, + cost_per_retained_byte_second: 0.0, + horizon_s: crate::analytical_cost::DEFAULT_HORIZON_S, version: "handoff-only-v1".into(), }; let space = crate::replacement::search_workload_with( @@ -937,6 +943,8 @@ mod tests { cost_per_cpu_op: 0.0, cost_per_scan_byte: 1.0, cost_per_retained_byte: 0.0, + cost_per_retained_byte_second: 0.0, + horizon_s: crate::analytical_cost::DEFAULT_HORIZON_S, version: "disk-only-v1".into(), }, ) diff --git a/crates/asap-aware-mapping/src/stage3_plan_selection.rs b/crates/asap-aware-mapping/src/stage3_plan_selection.rs index e99f2d23c..eac4ea7c2 100644 --- a/crates/asap-aware-mapping/src/stage3_plan_selection.rs +++ b/crates/asap-aware-mapping/src/stage3_plan_selection.rs @@ -17,16 +17,37 @@ //! building every candidate (e.g. a dynamic program over target nesting) is //! deferred to #671. //! -//! A node is charged once per computation (#509): a materialized node -//! (`MatAtIngestion` or `MatAtQuery`) once, since its output is stored and -//! reused; a `NotMaterialized` node once per consumer that computes it, since -//! it is recomputed for each use and not stored. Its consumers are the -//! queries whose sub-DAG reaches it and the materialized nodes above it, -//! counted without looking through a materialized node: a node below one is -//! computed when that node is, so the materialized node counts as one -//! consumer. Stage 2 does not duplicate a shared node; the multiplier is -//! recorded on its cost ([`NodeCost::multiplier`]). Every node is -//! `NotMaterialized` until #604, so a shared node is charged once per query. +//! ## Cost model: cost per second of wall time +//! +//! A candidate's cost is what running it costs per second ([`COST_UNIT`]), +//! so queries with different recurrences add up. Each node's price for one +//! computation is multiplied by how often it is computed per second +//! ([`NodeCost::per_second`]): +//! +//! * A `NotMaterialized` node is recomputed for each consumer that reads it +//! and is not stored: it is charged once per consumer +//! ([`NodeCost::multiplier`]) at the sum of their rates. Its consumers are +//! the queries whose sub-DAG reaches it and the materialized nodes above +//! it, counted without looking through a materialized node: a node below +//! one is computed when that node is. Stage 2 does not duplicate a shared +//! node, so a shared `NotMaterialized` input costs what reading it +//! separately does (#509 Example 4). +//! * A `MatAtQuery` node is computed once for all the queries that reach +//! it, at their combined rate: queries repeating at the same interval are +//! evaluated together, so each distinct interval counts once. +//! * A `MatAtIngestion` node runs as data arrives: it is priced over one +//! second of arriving rows, once per second, whatever its readers' rates. +//! * A query repeating every `I` ms runs `1000 / I` times per second, and an +//! estimated rate runs at that rate. One-off, scheduled and unknown +//! queries are spread over the calibration's `horizon_s` (default +//! 3600 s): `invocations / horizon_s` per second. +//! +//! State a materialized node keeps between computations is charged +//! `cost_per_retained_byte_second` per byte per second: its output's bytes +//! for `MatAtQuery`, and twice them for `MatAtIngestion` (the window being +//! built and the completed one queries read). Memory a query-time +//! computation holds only while it runs is not charged per second; the +//! per-computation memory term below applies to it as to every node. //! //! Prices come from [`crate::analytical_cost::estimate_operator`] over edge //! statistics derived from the [`DataWorkload`] and a fixed default group @@ -42,7 +63,7 @@ //! A cost error on any candidate aborts selection; it is reachable only from //! bad declared rates. Latency bounds and deployment capabilities are not //! checked yet. -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::rc::Rc; use asap_types::ir::operator_properties::BinaryOpKind; @@ -54,12 +75,12 @@ use asap_types::ir::{ }; use asap_types::post_asap::sketch::KLL_MAX_K; use asap_types::post_asap::{ - AccuracyGuarantee, CompositionOperator, ErrorMetric, FieldDataType, Materialization, - SketchAlgorithm, SketchParams, SketchStatistic, WeightDomain, + AccuracyGuarantee, CompositionOperator, ErrorMetric, ExecutionTiming, FieldDataType, + Materialization, SketchAlgorithm, SketchParams, SketchStatistic, WeightDomain, }; use asap_types::pre_asap::{AggIntent, DataType, Reduction, ScalarValue, Schema}; use asap_types::types::AccuracyTarget; -use asap_types::workload::{DataWorkload, MetricType}; +use asap_types::workload::{DataWorkload, MetricType, QueryRecurrence, RepeatedDemand}; use thiserror::Error; use crate::accuracy::composition::{ @@ -81,8 +102,25 @@ use crate::physical_operator_statistics::{ use crate::stage2_physical_asap_optimization::PhysicalCandidate; use crate::PlanningModels; -pub const COST_UNIT: &str = "cpu_ms_per_workload_evaluation"; -pub const COST_SOURCE: &str = "analytical-cost-v1 (illustrative statistics)"; +/// Cost per second of wall time; one unit is one CPU-millisecond under +/// [`illustrative_calibration`]. +pub const COST_UNIT: &str = "cost_per_second"; +pub const COST_SOURCE: &str = "analytical-cost-v2 (illustrative statistics)"; + +/// The coefficients plan selection prices with until a deployment is +/// calibrated: 1 ns of CPU per operation, 0.1 ns per scanned or held byte, +/// 1 GB kept for a second costs 1/8 vCPU (125 CPU-ms), and one-off work is +/// spread over one hour. +pub fn illustrative_calibration() -> ResourceCalibration { + ResourceCalibration { + cost_per_cpu_op: 1e-6, + cost_per_scan_byte: 1e-7, + cost_per_retained_byte: 1e-7, + cost_per_retained_byte_second: 1.25e-7, + horizon_s: crate::analytical_cost::DEFAULT_HORIZON_S, + version: "illustrative-v2".into(), + } +} /// Groups assumed for every `by (...)` reduction, absent group-count evidence. const DEFAULT_GROUP_COUNT: u64 = 100; @@ -91,12 +129,15 @@ const DEFAULT_SERIES: u64 = 1_000; const DEFAULT_ROWS_PER_SECOND: f64 = 1_000.0; const DEFAULT_LOOKBACK_MS: u64 = 60_000; -/// `cost` is one computation's price times `multiplier`, the times the -/// node is computed per workload evaluation ([`computations`]). +/// `cost` is the node's price per second: one computation's price times +/// `per_second`, the times it is computed per second, plus any kept state's +/// price ([`computations`], [`price`]). `multiplier` is the number of +/// consumers it is computed for (1 when materialized). #[derive(Debug, Clone, PartialEq)] pub struct NodeCost { pub cost: f64, pub multiplier: u64, + pub per_second: f64, pub detail: String, } @@ -129,11 +170,15 @@ pub struct Selection { #[derive(Debug, Error)] pub enum SelectionError { - #[error("candidate {candidate} has {roots} roots but {targets} accuracy targets were given")] + #[error( + "candidate {candidate} has {roots} roots but {targets} accuracy targets and \ + {recurrences} recurrences were given" + )] TargetCount { candidate: String, roots: usize, targets: usize, + recurrences: usize, }, #[error("candidate {candidate}, node {node:?}: {error}")] Cost { @@ -146,12 +191,13 @@ pub enum SelectionError { } /// Stage 3 (plan selection): reject candidates that miss a query's accuracy -/// target, price the rest and -/// select the cheapest (the first on ties). `targets[i]` is the requirement -/// of `candidate.roots[i]`; `None` imposes none. +/// target, price the rest per second and select the cheapest (the first on +/// ties). `targets[i]` is the requirement of `candidate.roots[i]` (`None` +/// imposes none) and `recurrence[i]` how often it is evaluated. pub fn select_plan( cands: &[PhysicalCandidate], targets: &[Option], + recurrence: &[QueryRecurrence], data: &DataWorkload, models: PlanningModels<'_>, ) -> Result { @@ -159,11 +205,12 @@ pub fn select_plan( let mut rejected = Vec::new(); let mut best: Option<(&str, f64)> = None; for candidate in cands { - if candidate.roots.len() != targets.len() { + if candidate.roots.len() != targets.len() || candidate.roots.len() != recurrence.len() { return Err(SelectionError::TargetCount { candidate: candidate.id.clone(), roots: candidate.roots.len(), targets: targets.len(), + recurrences: recurrence.len(), }); } if let Some(reason) = accuracy_violation(candidate, targets, data, &models) { @@ -174,11 +221,15 @@ pub fn select_plan( }); continue; } - let cost = price(&candidate.dag, data).map_err(|(node, error)| SelectionError::Cost { - candidate: candidate.id.clone(), - node, - error, - })?; + let calibration = illustrative_calibration(); + let cost = + price(&candidate.dag, recurrence, data, &calibration).map_err(|(node, error)| { + SelectionError::Cost { + candidate: candidate.id.clone(), + node, + error, + } + })?; if best.is_none_or(|(_, total)| cheaper(cost.total, total)) { best = Some((&candidate.id, cost.total)); } @@ -633,18 +684,87 @@ struct Shape { rows_per_ms: f64, } -/// How many times each node of `dag` is computed per workload evaluation: -/// 1 for a materialized node; for a `NotMaterialized` one, the number of -/// consumers that reach it, where a consumer is a query (operator or scalar -/// root) or a materialized node, and a walk stops at the first materialized -/// node it meets. A node read several times within one query's walk counts -/// once for it. -fn computations(dag: &PhysicalASAPDAG) -> HashMap { - let materialized: HashSet<_> = dag +/// Evaluations per second of a node computed once for `queries` together. +/// Repeating queries with equal intervals are evaluated together, so each +/// interval counts once; estimated rates add. One-off, scheduled and unknown +/// queries run as one batch: the most invocations among them, spread over +/// `horizon_s`. +fn evaluation_rate<'r>( + queries: impl IntoIterator, + horizon_s: f64, +) -> Result { + let mut intervals = BTreeSet::new(); + let mut estimated = 0.0; + let mut invocations = 0u64; + for recurrence in queries { + match recurrence { + QueryRecurrence::Repeated( + RepeatedDemand::FixedInterval(interval) + | RepeatedDemand::FixedIntervalAt { interval, .. }, + ) => { + if interval.0 == 0 { + return Err(AnalyticalCostError::InvalidRecurrence); + } + intervals.insert(interval.0); + } + QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(estimate)) => { + let rate = estimate.expected_rate.0; + if !rate.is_finite() || rate < 0.0 { + return Err(AnalyticalCostError::InvalidRecurrence); + } + estimated += rate; + } + QueryRecurrence::Repeated(RepeatedDemand::Scheduled(times)) => { + invocations = invocations.max(times.len() as u64); + } + QueryRecurrence::OneTime { + invocations: count, .. + } => invocations = invocations.max(*count), + QueryRecurrence::Unknown => invocations = invocations.max(1), + } + } + let repeated: f64 = intervals.into_iter().map(|ms| 1_000.0 / ms as f64).sum(); + Ok(repeated + estimated + invocations as f64 / horizon_s) +} + +/// The nodes each query reads first, in query order: query `i` is the scalar +/// root keyed `i` when there is one, else the next operator root. +fn query_starts(dag: &PhysicalASAPDAG) -> Vec> { + let mut roots = dag.roots.iter(); + (0..dag.roots.len() + dag.scalar_roots.len()) + .map(|query| { + match dag + .scalar_roots + .iter() + .find(|(position, _)| *position == query) + { + Some((_, expr)) => expr.operator_refs().into_iter().copied().collect(), + None => roots.next().into_iter().copied().collect(), + } + }) + .collect() +} + +/// How often each node of `dag` is computed: `(multiplier, per_second)` +/// (module doc, "Cost model"). A materialized node is computed once for all +/// its readers: once per second at ingestion time (`MatAtIngestion`), or at +/// the combined rate of the queries reaching it (`MatAtQuery`, +/// [`evaluation_rate`]). A `NotMaterialized` node is computed once per +/// consumer that reaches it, at that consumer's rate. A consumer is a query +/// (operator or scalar root), at its own rate, or a materialized node, at +/// that node's rate; a walk stops at the first materialized node it meets, +/// and a node read several times within one consumer's walk counts once for +/// it. Its `multiplier` is that number of consumers. +fn computations( + dag: &PhysicalASAPDAG, + recurrence: &[QueryRecurrence], + horizon_s: f64, +) -> Result, AnalyticalCostError> { + let materialized: HashMap<_, _> = dag .nodes .iter() .filter(|n| n.materialization != Materialization::NotMaterialized) - .map(|n| n.id) + .map(|n| (n.id, n.materialization)) .collect(); let inputs = |id: PhysicalASAPNodeId| { dag.edges @@ -652,55 +772,86 @@ fn computations(dag: &PhysicalASAPDAG) -> HashMap { .filter(move |e| e.consumer == id) .map(|e| e.producer) }; - // Each consumer's starting nodes: what a query reads, or a materialized - // node's inputs. - let consumers = dag - .roots - .iter() - .map(|root| vec![*root]) - .chain( - dag.scalar_roots - .iter() - .map(|(_, expr)| expr.operator_refs().into_iter().copied().collect()), - ) - .chain( - dag.nodes - .iter() - .filter(|n| materialized.contains(&n.id)) - .map(|n| inputs(n.id).collect()), - ); - let mut count: HashMap<_, u64> = HashMap::new(); - for start in consumers { + let starts = query_starts(dag); + // The queries reaching each node, through any node. + let mut reached: HashMap> = HashMap::new(); + for (query, start) in starts.iter().enumerate() { + let mut stack = start.clone(); + while let Some(id) = stack.pop() { + let queries = reached.entry(id).or_default(); + if queries.last() == Some(&query) { + continue; + } + queries.push(query); + stack.extend(inputs(id)); + } + } + let mut rate_of_materialized = HashMap::new(); + for (&id, materialization) in &materialized { + let rate = match materialization { + Materialization::MatAtIngestion(_) => 1.0, + _ => evaluation_rate( + reached + .get(&id) + .into_iter() + .flatten() + .filter_map(|&q| recurrence.get(q)), + horizon_s, + )?, + }; + rate_of_materialized.insert(id, rate); + } + // Each consumer's starting nodes and its rate. + let mut consumers = Vec::new(); + for (query, start) in starts.into_iter().enumerate() { + let rate = evaluation_rate(recurrence.get(query), horizon_s)?; + consumers.push((start, rate)); + } + for (&id, &rate) in &rate_of_materialized { + consumers.push((inputs(id).collect(), rate)); + } + let mut count: HashMap<_, (u64, f64)> = HashMap::new(); + for (start, rate) in consumers { let mut seen = HashSet::new(); let mut stack: Vec = start; while let Some(id) = stack.pop() { - if materialized.contains(&id) || !seen.insert(id) { + if materialized.contains_key(&id) || !seen.insert(id) { continue; } - *count.entry(id).or_default() += 1; + let entry = count.entry(id).or_default(); + entry.0 += 1; + entry.1 += rate; stack.extend(inputs(id)); } } - dag.nodes + Ok(dag + .nodes .iter() .map(|n| { - let times = if materialized.contains(&n.id) { - 1 - } else { - count.get(&n.id).copied().unwrap_or(0) + let times = match rate_of_materialized.get(&n.id) { + Some(&rate) => (1, rate), + None => count.get(&n.id).copied().unwrap_or((0, 0.0)), }; (n.id, times) }) - .collect() + .collect()) } -/// Price every node of `dag` once per computation ([`computations`]). Nodes -/// are exported children first, so each node's input statistics are known -/// when it is reached. +/// Price every node of `dag` per second of wall time (module doc, "Cost +/// model"). Nodes are exported children first, so each node's input +/// statistics are known when it is reached. A query-time node is priced per +/// computation over the rows its time ranges keep, times how often it is +/// computed per second ([`computations`]); an ingestion-time node over one +/// second of arriving rows. A materialized node's kept state is charged per +/// byte per second. fn price( dag: &PhysicalASAPDAG, + recurrence: &[QueryRecurrence], data: &DataWorkload, + calibration: &ResourceCalibration, ) -> Result { + let first = dag.nodes.first().map(|n| n.id).unwrap_or_default(); + calibration.validate().map_err(|error| (first, error))?; let series = data .input_cardinality .value @@ -714,17 +865,13 @@ fn price( .map_or(DEFAULT_ROWS_PER_SECOND, |rate| rate.0) / 1_000.0, }; - let calibration = ResourceCalibration { - cost_per_cpu_op: 1e-6, - cost_per_scan_byte: 1e-7, - cost_per_retained_byte: 1e-7, - version: "illustrative-v1".into(), - }; let nodes: HashMap<_, _> = dag.nodes.iter().map(|n| (n.id, n)).collect(); - let multipliers = computations(dag); + let multipliers = + computations(dag, recurrence, calibration.horizon_s).map_err(|error| (first, error))?; let mut output: HashMap = HashMap::new(); let mut per_node = BTreeMap::new(); for node in &dag.nodes { + let ingestion = node.output_state.timing == ExecutionTiming::IngestionTime; let inputs: Vec<_> = dag .edges .iter() @@ -746,17 +893,23 @@ fn price( promql: None, }; let groups = |reduction: &Reduction| { - match reduction { + let groups = match reduction { Reduction::Reduce(keys) if keys.keys().is_empty() && !keys.is_without() => 1, Reduction::Reduce(keys) if !keys.is_without() => DEFAULT_GROUP_COUNT, _ => shape.series, + }; + // One second of arriving rows does not bound the groups a + // maintained state holds. + match ingestion { + true => groups, + false => groups.min(input.rows.max(1)), } - .min(input.rows.max(1)) }; let (out, estimate, detail) = match &node.payload { Payload::NonASAP(operator) => match operator { NonASAPOp::Scan { .. } => { - // A scan reads what its time range keeps. + // At ingestion time, one second of arriving rows; at + // query time, what its time range keeps. let lookback = dag .edges .iter() @@ -769,6 +922,7 @@ fn price( }) .max() .unwrap_or(DEFAULT_LOOKBACK_MS); + let lookback = if ingestion { 1_000 } else { lookback }; let out = edge(((shape.rows_per_ms * lookback as f64).round() as u64).max(1)); let estimate = estimate_operator( PhysicalOperator::Scan, @@ -922,15 +1076,32 @@ fn price( ), }; let cost = estimate - .and_then(|estimate| estimate.calibrated_cost(&calibration)) + .and_then(|estimate| estimate.calibrated_cost(calibration)) .map_err(|error| (node.id, error))?; - let multiplier = multipliers[&node.id]; + let (multiplier, per_second) = multipliers[&node.id]; + let mut cost = cost * per_second; + let mut detail = format!("{detail}; x {per_second:.4}/s"); + // State kept between computations: the window being built and the + // completed one at ingestion time, the kept output at query time. + let kept = match node.materialization { + Materialization::NotMaterialized => 0, + Materialization::MatAtIngestion(_) => 2 * out.bytes, + _ => out.bytes, + }; + if kept > 0 { + cost += calibration.cost_per_retained_byte_second * kept as f64; + detail += &format!("; keeps {kept} bytes"); + } + if !cost.is_finite() { + return Err((node.id, AnalyticalCostError::Overflow)); + } output.insert(node.id, out); per_node.insert( node.id, NodeCost { - cost: cost * multiplier as f64, + cost, multiplier, + per_second, detail, }, ); @@ -1100,6 +1271,18 @@ mod tests { use asap_types::pre_asap::expr_ir::ArithmeticOpKind; use asap_types::workload::{Evidence, Rate}; + /// `n` queries repeating every 10 s, as Example 1's panels do. + fn every_10s(n: usize) -> Vec { + vec![repeating(10_000); n] + } + + /// A query repeating every `interval_ms`. + fn repeating(interval_ms: u32) -> QueryRecurrence { + QueryRecurrence::Repeated(RepeatedDemand::FixedInterval( + asap_types::workload::RepetitionInterval(interval_ms), + )) + } + fn data() -> DataWorkload { DataWorkload { ingestion_rate: Evidence { @@ -1175,6 +1358,7 @@ mod tests { let selection = select_plan( &candidates, &[Some(strict)], + &every_10s(1), &data(), PlanningModels::builtin(), ) @@ -1217,6 +1401,7 @@ mod tests { let selection = select_plan( &candidates(), &[Some(target)], + &every_10s(1), &data(), PlanningModels::builtin(), ) @@ -1251,6 +1436,7 @@ mod tests { let selection = select_plan( &candidates(), &[Some(target.clone())], + &every_10s(1), &data, PlanningModels::builtin(), ) @@ -1327,8 +1513,14 @@ mod tests { /// candidate is admitted and priced. #[test] fn count_min_is_admitted_without_a_target() { - let selection = - select_plan(&candidates(), &[None], &data(), PlanningModels::builtin()).unwrap(); + let selection = select_plan( + &candidates(), + &[None], + &every_10s(1), + &data(), + PlanningModels::builtin(), + ) + .unwrap(); assert!(selection.costs.contains_key("P2")); assert!(selection.rejected.iter().all(|r| r.valid)); } @@ -1347,6 +1539,7 @@ mod tests { let selection = select_plan( &candidates, &[Some(target)], + &every_10s(1), &data(), PlanningModels::builtin().with_evidence(&EveryOutputMeasuredExact), ) @@ -1384,7 +1577,12 @@ mod tests { let [outer, inner] = candidate.dag.roots[..] else { panic!("two roots") }; - let cost = price(&candidate.dag, &data()).unwrap(); + // Kept state is priced apart; this compares computations only. + let calibration = ResourceCalibration { + cost_per_retained_byte_second: 0.0, + ..illustrative_calibration() + }; + let cost = price(&candidate.dag, &every_10s(2), &data(), &calibration).unwrap(); let multipliers: BTreeMap<_, _> = cost .per_node .iter() @@ -1406,10 +1604,11 @@ mod tests { .find(|n| n.id == inner) .unwrap(); node.materialization = Materialization::MatAtQuery(asap_types::ir::Medium::Memory); - let stored = price(&candidate.dag, &data()).unwrap(); + let stored = price(&candidate.dag, &every_10s(2), &data(), &calibration).unwrap(); for (id, c) in &stored.per_node { assert_eq!(c.multiplier, 1, "node {id} is computed once"); - assert_eq!(c.cost, cost.per_node[id].cost / multipliers[id] as f64); + let once = cost.per_node[id].cost / multipliers[id] as f64; + assert!((c.cost - once).abs() <= 1e-12 * once, "node {id}"); } assert!(stored.total < cost.total); } @@ -1494,6 +1693,7 @@ mod tests { let builtin = select_plan( &candidates, &[Some(target.clone())], + &every_10s(1), &data(), PlanningModels::builtin(), ) @@ -1512,6 +1712,7 @@ mod tests { let measured = select_plan( &candidates, &[Some(target)], + &every_10s(1), &data(), PlanningModels::builtin().with_evidence(&EveryOutputMeasuredExact), ) @@ -1539,6 +1740,7 @@ mod tests { let selection = select_plan( &candidates, &[Some(target.clone())], + &every_10s(1), &data(), PlanningModels::builtin(), ) @@ -1561,8 +1763,14 @@ mod tests { fn exact_l2_state_is_priced_like_exact_quantile_state() { let aggregate_cost = |query: &str| { let candidates = all_candidates(query, &AccuracyTarget::Exact); - let selection = - select_plan(&candidates, &[None], &data(), PlanningModels::builtin()).unwrap(); + let selection = select_plan( + &candidates, + &[None], + &every_10s(1), + &data(), + PlanningModels::builtin(), + ) + .unwrap(); let exact = candidates .iter() .find(|c| !has_summary_estimate(c)) @@ -1724,6 +1932,7 @@ mod tests { let selection = select_plan( &candidates, &[Some(target.clone())], + &every_10s(1), &data, PlanningModels::builtin(), ) @@ -1745,8 +1954,14 @@ mod tests { let mut twin = candidates[0].clone(); twin.id = "P1-twin".into(); candidates.push(twin); - let selection = - select_plan(&candidates, &[None], &data(), PlanningModels::builtin()).unwrap(); + let selection = select_plan( + &candidates, + &[None], + &every_10s(1), + &data(), + PlanningModels::builtin(), + ) + .unwrap(); assert_eq!(selection.selected, "P1"); let reason = &selection.rejected[0].reason; assert!( @@ -1767,6 +1982,7 @@ mod tests { let selection = select_plan( &candidates, &[Some(target)], + &every_10s(1), &data(), PlanningModels::builtin(), ) @@ -1858,6 +2074,7 @@ mod tests { let run = crate::pass::run_stage1_to_stage3( vec![(0, QueryRoot::Operator(root))], &[Some(selected.clone())], + &[QueryRecurrence::Unknown], data, models, ) @@ -2357,4 +2574,214 @@ mod tests { .iter() .all(|c| !selection.costs.contains_key(&c.id))); } + + /// `quantile_over_time(0.99, m[1m])` with its KLL candidate chosen, its + /// state materialized as `materialization`. + fn kll_dag(materialization: Materialization) -> PhysicalASAPDAG { + use asap_types::ir::{ + apply_materialization_timings, MaterializationAssignment, TimingMemo, + }; + let root = lower_promql( + "quantile_over_time(0.99, m[1m])", + AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.01, + }, + ); + let inventory = crate::per_computation_candidates::generate_per_computation_candidates( + vec![(0, QueryRoot::Operator(root))], + ) + .unwrap(); + let choice: Vec<_> = inventory + .targets + .iter() + .map(|t| { + t.choices + .candidates + .iter() + .position(|c| { + *c == crate::per_computation_candidates::ComputationCandidate::Summary( + SketchAlgorithm::Kll, + ) + }) + .unwrap_or(0) + }) + .collect(); + let roots = + crate::per_computation_candidates::compose_logical_candidate(&inventory, &choice) + .unwrap(); + let QueryRoot::Operator(root) = &roots[0].1 else { + panic!("operator root") + }; + let state = OperatorNode::reachable(root) + .into_iter() + .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) + .expect("a KLL state"); + let mut assignment = + MaterializationAssignment::with_default(Materialization::NotMaterialized); + assignment.set(&state, materialization); + let timed = + apply_materialization_timings(root, &assignment, &mut TimingMemo::new()).unwrap(); + asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids(&[ + QueryRoot::Operator(timed), + ]) + .unwrap() + .dag + } + + const AT_INGESTION: Materialization = + Materialization::MatAtIngestion(asap_types::ir::Medium::Memory); + const AT_QUERY: Materialization = Materialization::MatAtQuery(asap_types::ir::Medium::Memory); + + fn node_of(dag: &PhysicalASAPDAG, pick: impl Fn(&Payload) -> bool) -> PhysicalASAPNodeId { + dag.nodes.iter().find(|n| pick(&n.payload)).unwrap().id + } + + fn is_scan(payload: &Payload) -> bool { + matches!(payload, Payload::NonASAP(NonASAPOp::Scan { .. })) + } + + fn is_build(payload: &Payload) -> bool { + matches!(payload, Payload::ASAP(ASAPOp::SummaryAgg { .. })) + } + + fn priced(dag: &PhysicalASAPDAG, recurrence: &[QueryRecurrence]) -> CandidateCost { + price(dag, recurrence, &data(), &illustrative_calibration()).unwrap() + } + + fn close(a: f64, b: f64) -> bool { + (a - b).abs() <= 1e-12 * a.abs().max(b.abs()).max(1e-300) + } + + /// An ingestion-time node is priced over one second of arriving rows, + /// once per second, whatever its readers' evaluation rate; a query-time + /// node scales with that rate. + #[test] + fn ingestion_time_node_is_priced_per_second_of_arriving_rows() { + let dag = kll_dag(AT_INGESTION); + let (fast, slow) = ( + priced(&dag, &[repeating(1_000)]), + priced(&dag, &[repeating(10_000)]), + ); + for id in [node_of(&dag, is_scan), node_of(&dag, is_build)] { + assert_eq!(fast.per_node[&id].cost, slow.per_node[&id].cost); + assert_eq!(fast.per_node[&id].per_second, 1.0); + } + let estimate = dag.roots[0]; + assert!(close( + fast.per_node[&estimate].cost, + 10.0 * slow.per_node[&estimate].cost + )); + } + + /// Kept state is charged per byte per second: twice the state's bytes at + /// ingestion time (the window being built and the completed one), once + /// at query time; state computed and dropped within a query has none. + #[test] + fn kept_state_is_charged_per_byte_per_second() { + let cost = |dag: &PhysicalASAPDAG, w| { + let calibration = ResourceCalibration { + cost_per_retained_byte_second: w, + ..illustrative_calibration() + }; + price(dag, &every_10s(1), &data(), &calibration).unwrap() + }; + for (materialization, windows) in [(AT_INGESTION, 2.0), (AT_QUERY, 1.0)] { + let dag = kll_dag(materialization); + let build = node_of(&dag, is_build); + let (with, without) = (cost(&dag, 1.25e-7), cost(&dag, 0.0)); + let Payload::ASAP(ASAPOp::SummaryAgg { family, .. }) = + &dag.nodes.iter().find(|n| n.id == build).unwrap().payload + else { + unreachable!() + }; + // One state per series: data() declares 10 000. + let kept = windows * 10_000.0 * summary_shape(family).1 as f64; + let term = with.per_node[&build].cost - without.per_node[&build].cost; + assert!(close(term, 1.25e-7 * kept), "{materialization:?}: {term}"); + assert!(close(with.total - without.total, term)); + } + let transient = kll_dag(Materialization::NotMaterialized); + assert_eq!(cost(&transient, 1.25e-7).total, cost(&transient, 0.0).total); + } + + /// A `NotMaterialized` node read by two queries is computed for each, at + /// each one's rate; `MatAtQuery`, it is computed once at their combined + /// rate, where queries at the same interval run together. + #[test] + fn rates_follow_per_consumer_charging() { + let root = lower_promql("sum by (job) (rate(m[1m]))", AccuracyTarget::Exact); + let one = optimize_physical_asap("L", &[QueryRoot::Operator(root.clone())]) + .unwrap() + .dag; + let mut two = optimize_physical_asap( + "L", + &[QueryRoot::Operator(root.clone()), QueryRoot::Operator(root)], + ) + .unwrap() + .dag; + let without_memory = ResourceCalibration { + cost_per_retained_byte_second: 0.0, + ..illustrative_calibration() + }; + let total = |dag: &PhysicalASAPDAG, recurrence: &[QueryRecurrence]| { + price(dag, recurrence, &data(), &without_memory) + .unwrap() + .total + }; + let single = total(&one, &every_10s(1)); + let mixed = [repeating(10_000), repeating(20_000)]; + assert!(close(total(&two, &every_10s(2)), 2.0 * single)); + assert!(close(total(&two, &mixed), 1.5 * single)); + assert_eq!(two.roots[0], two.roots[1], "one root node for both queries"); + for node in &mut two.nodes { + node.materialization = AT_QUERY; + } + assert!(close(total(&two, &every_10s(2)), single)); + assert!(close(total(&two, &mixed), 1.5 * single)); + } + + /// One-off and unknown recurrence are spread over the horizon: + /// invocations × per-evaluation cost / horizon_s. + #[test] + fn one_off_recurrence_is_spread_over_the_horizon() { + let root = lower_promql("sum by (job) (rate(m[1m]))", AccuracyTarget::Exact); + let dag = optimize_physical_asap("L", &[QueryRoot::Operator(root)]) + .unwrap() + .dag; + let horizon = illustrative_calibration().horizon_s; + assert_eq!(horizon, 3_600.0); + let per_evaluation = priced(&dag, &[repeating(1_000)]).total; + let once = priced( + &dag, + &[QueryRecurrence::OneTime { + invocations: 3, + execute_at: None, + }], + ) + .total; + assert!(close(once, 3.0 * per_evaluation / horizon), "{once}"); + assert!(close( + priced(&dag, &[QueryRecurrence::Unknown]).total, + per_evaluation / horizon + )); + } + + /// A calibration outside its domain is an error, not a price. + #[test] + fn invalid_calibration_is_rejected() { + let dag = kll_dag(Materialization::NotMaterialized); + for calibration in [ + ResourceCalibration { + cost_per_retained_byte_second: -1.0, + ..illustrative_calibration() + }, + ResourceCalibration { + horizon_s: 0.0, + ..illustrative_calibration() + }, + ] { + assert!(price(&dag, &every_10s(1), &data(), &calibration).is_err()); + } + } } diff --git a/crates/asap-aware-mapping/src/test_support.rs b/crates/asap-aware-mapping/src/test_support.rs index 9b37c5615..597f5530d 100644 --- a/crates/asap-aware-mapping/src/test_support.rs +++ b/crates/asap-aware-mapping/src/test_support.rs @@ -224,6 +224,7 @@ pub(crate) fn cms_summary(query: &str) -> OperatorNode { let run = crate::pass::run_stage1_to_stage3( vec![(0, QueryRoot::Operator(root))], &[Some(target)], + &[asap_types::workload::QueryRecurrence::Unknown], &DataWorkload::default(), crate::PlanningModels::builtin(), ) diff --git a/crates/asap-physical-operators/src/physical_planner/candidates.rs b/crates/asap-physical-operators/src/physical_planner/candidates.rs index 36075d825..7aad45bee 100644 --- a/crates/asap-physical-operators/src/physical_planner/candidates.rs +++ b/crates/asap-physical-operators/src/physical_planner/candidates.rs @@ -379,6 +379,7 @@ mod tests { let run = asap_aware_mapping::run_stage1_to_stage3( vec![(0, planner_types::ir::QueryRoot::Operator(root))], &[Some(planner_types::types::AccuracyTarget::Exact)], + &[planner_types::workload::QueryRecurrence::Unknown], &workload.data_workload.unwrap(), asap_aware_mapping::PlanningModels::builtin(), ) diff --git a/crates/asap-physical-operators/tests/common/mod.rs b/crates/asap-physical-operators/tests/common/mod.rs index 4f79f7d83..d04894d4e 100644 --- a/crates/asap-physical-operators/tests/common/mod.rs +++ b/crates/asap-physical-operators/tests/common/mod.rs @@ -71,6 +71,7 @@ fn run_stage1_to_stage3( asap_aware_mapping::run_stage1_to_stage3( vec![(0, QueryRoot::Operator(root))], &[Some(accuracy)], + &[planner_types::workload::QueryRecurrence::Unknown], &data, asap_aware_mapping::PlanningModels::builtin(), ) diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index fbd325e37..42e807ce7 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -229,7 +229,7 @@ fn run(args: Vec) -> Result<(), Failure> { }; let catalog = sql_catalog(&tables)?; let mut names = HashSet::new(); - let (mut described, mut roots) = (Vec::new(), Vec::new()); + let (mut described, mut roots, mut recurrence) = (Vec::new(), Vec::new(), Vec::new()); for (index, (language, text, name)) in queries.into_iter().enumerate() { let (name, defaulted) = match name { Some(name) => (name, false), @@ -252,6 +252,7 @@ fn run(args: Vec) -> Result<(), Failure> { QueryLanguage::PromQL => lower_promql_roots(&one)?, _ => lower_sql_roots(&one, &catalog)?, }); + recurrence.extend(one.query_workload.entries().map(|entry| entry.recurrence)); let mut query = workload_queries(&one).remove(0); query["id"] = json!(name); described.push(query); @@ -261,8 +262,13 @@ fn run(args: Vec) -> Result<(), Failure> { data_ingestion_interval: declared(DurationMs(interval_ms)), ..Default::default() }; - let result = - run_planning_stages_over(roots.clone(), &targets, &data, Some(max_combinations)); + let result = run_planning_stages_over( + roots.clone(), + &targets, + &recurrence, + &data, + Some(max_combinations), + ); document_of(described, &roots, result, max_displayed)? } _ => return Err("give exactly one of --example or --promql/--sql queries".into()), diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index 01616d3ef..1b5171a9e 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -11,7 +11,7 @@ use asap_aware_mapping::PlanningModels; use asap_types::ir::schema_support::with_promql_series_identity; use asap_types::ir::QueryRoot; use asap_types::types::AccuracyTarget; -use asap_types::workload::{DataWorkload, PlanningWorkload}; +use asap_types::workload::{DataWorkload, PlanningWorkload, QueryRecurrence}; #[derive(Debug)] pub struct PlanningStagesRun { @@ -118,15 +118,22 @@ pub fn run_planning_stages( .entries() .map(|entry| entry.requirements.accuracy.target()) .collect(); + let recurrence: Vec<_> = workload + .query_workload + .entries() + .map(|entry| entry.recurrence) + .collect(); let data = workload.data_workload.clone().unwrap_or_default(); - run_planning_stages_over(roots, &targets, &data, max_combinations) + run_planning_stages_over(roots, &targets, &recurrence, &data, max_combinations) } /// [`run_planning_stages`] over roots lowered from any mix of query -/// languages: query `i` is `roots[i]` with accuracy target `targets[i]`. +/// languages: query `i` is `roots[i]` with accuracy target `targets[i]`, +/// evaluated as `recurrence[i]` says. pub fn run_planning_stages_over( roots: Vec, targets: &[AccuracyTarget], + recurrence: &[QueryRecurrence], data: &DataWorkload, max_combinations: Option, ) -> Result { @@ -142,8 +149,13 @@ pub fn run_planning_stages_over( limit, }); } - let run = - run_stage1_to_stage3_from_pass1(per_computation, &targets, data, PlanningModels::builtin()); + let run = run_stage1_to_stage3_from_pass1( + per_computation, + &targets, + recurrence, + data, + PlanningModels::builtin(), + ); for candidate in &run.stage1_logical_asap_optimization { if let Ok(roots) = &candidate.roots { validate(roots)?; diff --git a/crates/frontend-promql/tests/support.rs b/crates/frontend-promql/tests/support.rs index f133b5887..d1424a541 100644 --- a/crates/frontend-promql/tests/support.rs +++ b/crates/frontend-promql/tests/support.rs @@ -120,6 +120,7 @@ pub fn run_stage1_to_stage3( asap_aware_mapping::run_stage1_to_stage3( vec![(0, asap_types::ir::QueryRoot::Operator(root))], &[Some(accuracy)], + &[asap_types::workload::QueryRecurrence::Unknown], &DataWorkload::default(), asap_aware_mapping::PlanningModels::builtin(), ) @@ -171,6 +172,7 @@ pub fn logical_plan(root: &Rc) -> Result, String> let run = asap_aware_mapping::run_stage1_to_stage3( vec![(0, asap_types::ir::QueryRoot::Operator(Rc::clone(root)))], &[None], + &[asap_types::workload::QueryRecurrence::Unknown], &DataWorkload::default(), asap_aware_mapping::PlanningModels::builtin(), ) diff --git a/crates/frontend-sql/tests/pearson_corr.rs b/crates/frontend-sql/tests/pearson_corr.rs index 2f19daf9c..6f1b8dd47 100644 --- a/crates/frontend-sql/tests/pearson_corr.rs +++ b/crates/frontend-sql/tests/pearson_corr.rs @@ -151,6 +151,7 @@ async fn corr_survives_exact_plan_compilation() { let run = asap_aware_mapping::run_stage1_to_stage3( vec![(0, asap_types::ir::QueryRoot::Operator(query.clone()))], &[Some(asap_types::types::AccuracyTarget::Exact)], + &[asap_types::workload::QueryRecurrence::Unknown], &Default::default(), asap_aware_mapping::PlanningModels::builtin(), ) diff --git a/crates/integration-tests/src/lib.rs b/crates/integration-tests/src/lib.rs index 12593fd49..435730358 100644 --- a/crates/integration-tests/src/lib.rs +++ b/crates/integration-tests/src/lib.rs @@ -158,6 +158,7 @@ pub mod planning { asap_aware_mapping::run_stage1_to_stage3( vec![(0, QueryRoot::Operator(root))], &[Some(accuracy)], + &[asap_types::workload::QueryRecurrence::Unknown], &DataWorkload::default(), asap_aware_mapping::PlanningModels::builtin(), ) diff --git a/crates/integration-tests/tests/planner_layering_example1.rs b/crates/integration-tests/tests/planner_layering_example1.rs index b69e46df8..bd2265cb3 100644 --- a/crates/integration-tests/tests/planner_layering_example1.rs +++ b/crates/integration-tests/tests/planner_layering_example1.rs @@ -107,6 +107,12 @@ fn physical_with(run: &Run, option: Q2Option) -> &Physical { p } +/// Example 1's panels repeat every 10 s. +fn every_10s() -> asap_types::workload::QueryRecurrence { + use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval}; + QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(10_000))) +} + // ── Workload ───────────────────────────────────────────────────────────── /// The encoded workload is valid and normalizes to Q1 then Q2. @@ -560,6 +566,7 @@ fn materialized_case() -> Run { let selection = select_plan( std::slice::from_ref(&p.candidate), &[Some(AccuracyTarget::Exact), Some(AccuracyTarget::Exact)], + &[every_10s(), every_10s()], workload.data_workload.as_ref().unwrap(), PlanningModels::builtin(), ) diff --git a/crates/planner/tests/e2e_plan.rs b/crates/planner/tests/e2e_plan.rs index 0ffa94fcc..f580415d8 100644 --- a/crates/planner/tests/e2e_plan.rs +++ b/crates/planner/tests/e2e_plan.rs @@ -128,7 +128,12 @@ async fn facade_plans_match_run_stage1_to_stage3() { targets.push(Some(accuracy)); } let data = workload.data_workload.clone().unwrap(); - let run = run_stage1_to_stage3(roots, &targets, &data, models).unwrap(); + let recurrence: Vec<_> = workload + .query_workload + .entries() + .map(|entry| entry.recurrence) + .collect(); + let run = run_stage1_to_stage3(roots, &targets, &recurrence, &data, models).unwrap(); let selection = run .stage3_plan_selection .as_ref() diff --git a/crates/planner/tests/plan_stages.rs b/crates/planner/tests/plan_stages.rs index 5e1d8e14a..6cf65abc7 100644 --- a/crates/planner/tests/plan_stages.rs +++ b/crates/planner/tests/plan_stages.rs @@ -41,6 +41,11 @@ fn promql_stages(query: &str, epsilon: f64) -> Stage1ToStage3Run { run_stage1_to_stage3( roots.into_iter().enumerate().collect(), &[Some(target)], + &workload + .query_workload + .entries() + .map(|entry| entry.recurrence) + .collect::>(), &workload.data_workload.unwrap(), PlanningModels::builtin(), ) diff --git a/crates/planner/tests/summary_sharing.rs b/crates/planner/tests/summary_sharing.rs index e8d47b126..75c0b670e 100644 --- a/crates/planner/tests/summary_sharing.rs +++ b/crates/planner/tests/summary_sharing.rs @@ -95,6 +95,11 @@ fn promql_stages(queries: &[(&str, f64)]) -> Stage1ToStage3Run { run_stage1_to_stage3( roots.into_iter().enumerate().collect(), &targets, + &workload + .query_workload + .entries() + .map(|entry| entry.recurrence) + .collect::>(), &workload.data_workload.clone().unwrap(), PlanningModels::builtin(), ) diff --git a/tools/dag-viewer/examples/planner-layering-example1.json b/tools/dag-viewer/examples/planner-layering-example1.json index f6d7045f4..c8b820cdd 100644 --- a/tools/dag-viewer/examples/planner-layering-example1.json +++ b/tools/dag-viewer/examples/planner-layering-example1.json @@ -19651,203 +19651,203 @@ "P1": { "per_node": { "0": { - "cost": 23.2000048, - "detail": "scan 4000000 samples", + "cost": 2.32000048, + "detail": "scan 4000000 samples; x 0.1000/s", "multiplier": 1 }, "1": { - "cost": 4.0000048, - "detail": "time range 60s: pass 4000000 rows", + "cost": 0.40000048000000005, + "detail": "time range 60s: pass 4000000 rows; x 0.1000/s", "multiplier": 1 }, "10": { - "cost": 14.048, - "detail": "sort 1000000 rows in 100 partitions", + "cost": 1.4048, + "detail": "sort 1000000 rows in 100 partitions; x 0.1000/s", "multiplier": 1 }, "11": { - "cost": 0.0010048, - "detail": "limit to 1000 rows", + "cost": 0.00010048, + "detail": "limit to 1000 rows; x 0.1000/s", "multiplier": 1 }, "12": { - "cost": 0.0010048, - "detail": "operator: pass 1000 rows", + "cost": 0.00010048, + "detail": "operator: pass 1000 rows; x 0.1000/s", "multiplier": 1 }, "2": { - "cost": 4.8, - "detail": "build exact Rate accumulator into 1000000 states: 4000000 rows x depth 1", + "cost": 0.48, + "detail": "build exact Rate accumulator into 1000000 states: 4000000 rows x depth 1; x 0.1000/s", "multiplier": 1 }, "3": { - "cost": 1.0, - "detail": "finalize 1000000 accumulators", + "cost": 0.1, + "detail": "finalize 1000000 accumulators; x 0.1000/s", "multiplier": 1 }, "4": { - "cost": 1.00008, - "detail": "build exact Sum accumulator into 100 states: 1000000 rows x depth 1", + "cost": 0.10000800000000001, + "detail": "build exact Sum accumulator into 100 states: 1000000 rows x depth 1; x 0.1000/s", "multiplier": 1 }, "5": { - "cost": 0.00009999999999999999, - "detail": "finalize 100 accumulators", + "cost": 9.999999999999999e-6, + "detail": "finalize 100 accumulators; x 0.1000/s", "multiplier": 1 }, "6": { - "cost": 23.2000048, - "detail": "scan 4000000 samples", + "cost": 2.32000048, + "detail": "scan 4000000 samples; x 0.1000/s", "multiplier": 1 }, "7": { - "cost": 4.0000048, - "detail": "time range 60s: pass 4000000 rows", + "cost": 0.40000048000000005, + "detail": "time range 60s: pass 4000000 rows; x 0.1000/s", "multiplier": 1 }, "8": { - "cost": 4.8, - "detail": "build exact Sum accumulator into 1000000 states: 4000000 rows x depth 1", + "cost": 0.48, + "detail": "build exact Sum accumulator into 1000000 states: 4000000 rows x depth 1; x 0.1000/s", "multiplier": 1 }, "9": { - "cost": 1.0, - "detail": "finalize 1000000 accumulators", + "cost": 0.1, + "detail": "finalize 1000000 accumulators; x 0.1000/s", "multiplier": 1 } }, - "source": "analytical-cost-v1 (illustrative statistics)", - "total": 81.05020880000001, - "unit": "cpu_ms_per_workload_evaluation" + "source": "analytical-cost-v2 (illustrative statistics)", + "total": 8.10502088, + "unit": "cost_per_second" }, "P2": { "per_node": { "0": { - "cost": 46.4000096, - "detail": "scan 4000000 samples", + "cost": 4.64000096, + "detail": "scan 4000000 samples; x 0.2000/s", "multiplier": 2 }, "1": { - "cost": 4.0000048, - "detail": "time range 60s: pass 4000000 rows", + "cost": 0.40000048000000005, + "detail": "time range 60s: pass 4000000 rows; x 0.1000/s", "multiplier": 1 }, "10": { - "cost": 0.0010048, - "detail": "limit to 1000 rows", + "cost": 0.00010048, + "detail": "limit to 1000 rows; x 0.1000/s", "multiplier": 1 }, "11": { - "cost": 0.0010048, - "detail": "operator: pass 1000 rows", + "cost": 0.00010048, + "detail": "operator: pass 1000 rows; x 0.1000/s", "multiplier": 1 }, "2": { - "cost": 4.8, - "detail": "build exact Rate accumulator into 1000000 states: 4000000 rows x depth 1", + "cost": 0.48, + "detail": "build exact Rate accumulator into 1000000 states: 4000000 rows x depth 1; x 0.1000/s", "multiplier": 1 }, "3": { - "cost": 1.0, - "detail": "finalize 1000000 accumulators", + "cost": 0.1, + "detail": "finalize 1000000 accumulators; x 0.1000/s", "multiplier": 1 }, "4": { - "cost": 1.00008, - "detail": "build exact Sum accumulator into 100 states: 1000000 rows x depth 1", + "cost": 0.10000800000000001, + "detail": "build exact Sum accumulator into 100 states: 1000000 rows x depth 1; x 0.1000/s", "multiplier": 1 }, "5": { - "cost": 0.00009999999999999999, - "detail": "finalize 100 accumulators", + "cost": 9.999999999999999e-6, + "detail": "finalize 100 accumulators; x 0.1000/s", "multiplier": 1 }, "6": { - "cost": 4.0000048, - "detail": "time range 60s: pass 4000000 rows", + "cost": 0.40000048000000005, + "detail": "time range 60s: pass 4000000 rows; x 0.1000/s", "multiplier": 1 }, "7": { - "cost": 4.8, - "detail": "build exact Sum accumulator into 1000000 states: 4000000 rows x depth 1", + "cost": 0.48, + "detail": "build exact Sum accumulator into 1000000 states: 4000000 rows x depth 1; x 0.1000/s", "multiplier": 1 }, "8": { - "cost": 1.0, - "detail": "finalize 1000000 accumulators", + "cost": 0.1, + "detail": "finalize 1000000 accumulators; x 0.1000/s", "multiplier": 1 }, "9": { - "cost": 14.048, - "detail": "sort 1000000 rows in 100 partitions", + "cost": 1.4048, + "detail": "sort 1000000 rows in 100 partitions; x 0.1000/s", "multiplier": 1 } }, - "source": "analytical-cost-v1 (illustrative statistics)", - "total": 81.05020880000001, - "unit": "cpu_ms_per_workload_evaluation" + "source": "analytical-cost-v2 (illustrative statistics)", + "total": 8.10502088, + "unit": "cost_per_second" }, "P3": { "per_node": { "0": { - "cost": 46.4000096, - "detail": "scan 4000000 samples", + "cost": 4.64000096, + "detail": "scan 4000000 samples; x 0.2000/s", "multiplier": 2 }, "1": { - "cost": 8.0000096, - "detail": "time range 60s: pass 4000000 rows", + "cost": 0.8000009600000001, + "detail": "time range 60s: pass 4000000 rows; x 0.2000/s", "multiplier": 2 }, "10": { - "cost": 0.0010048, - "detail": "operator: pass 1000 rows", + "cost": 0.00010048, + "detail": "operator: pass 1000 rows; x 0.1000/s", "multiplier": 1 }, "2": { - "cost": 4.8, - "detail": "build exact Rate accumulator into 1000000 states: 4000000 rows x depth 1", + "cost": 0.48, + "detail": "build exact Rate accumulator into 1000000 states: 4000000 rows x depth 1; x 0.1000/s", "multiplier": 1 }, "3": { - "cost": 1.0, - "detail": "finalize 1000000 accumulators", + "cost": 0.1, + "detail": "finalize 1000000 accumulators; x 0.1000/s", "multiplier": 1 }, "4": { - "cost": 1.00008, - "detail": "build exact Sum accumulator into 100 states: 1000000 rows x depth 1", + "cost": 0.10000800000000001, + "detail": "build exact Sum accumulator into 100 states: 1000000 rows x depth 1; x 0.1000/s", "multiplier": 1 }, "5": { - "cost": 0.00009999999999999999, - "detail": "finalize 100 accumulators", + "cost": 9.999999999999999e-6, + "detail": "finalize 100 accumulators; x 0.1000/s", "multiplier": 1 }, "6": { - "cost": 4.8, - "detail": "build exact Sum accumulator into 1000000 states: 4000000 rows x depth 1", + "cost": 0.48, + "detail": "build exact Sum accumulator into 1000000 states: 4000000 rows x depth 1; x 0.1000/s", "multiplier": 1 }, "7": { - "cost": 1.0, - "detail": "finalize 1000000 accumulators", + "cost": 0.1, + "detail": "finalize 1000000 accumulators; x 0.1000/s", "multiplier": 1 }, "8": { - "cost": 14.048, - "detail": "sort 1000000 rows in 100 partitions", + "cost": 1.4048, + "detail": "sort 1000000 rows in 100 partitions; x 0.1000/s", "multiplier": 1 }, "9": { - "cost": 0.0010048, - "detail": "limit to 1000 rows", + "cost": 0.00010048, + "detail": "limit to 1000 rows; x 0.1000/s", "multiplier": 1 } }, - "source": "analytical-cost-v1 (illustrative statistics)", - "total": 81.05020880000001, - "unit": "cpu_ms_per_workload_evaluation" + "source": "analytical-cost-v2 (illustrative statistics)", + "total": 8.10502088, + "unit": "cost_per_second" } }, "rejected": [ @@ -19883,12 +19883,12 @@ }, { "id": "P2", - "reason": "tied with the selected candidate P1 at 81.050 cpu_ms_per_workload_evaluation; lost on enumeration order", + "reason": "tied with the selected candidate P1 at 8.105 cost_per_second; lost on enumeration order", "valid": true }, { "id": "P3", - "reason": "tied with the selected candidate P1 at 81.050 cpu_ms_per_workload_evaluation; lost on enumeration order", + "reason": "tied with the selected candidate P1 at 8.105 cost_per_second; lost on enumeration order", "valid": true } ], diff --git a/tools/dag-viewer/examples/scalar-queries.json b/tools/dag-viewer/examples/scalar-queries.json index 0ba6d84ff..1f8b7839b 100644 --- a/tools/dag-viewer/examples/scalar-queries.json +++ b/tools/dag-viewer/examples/scalar-queries.json @@ -2687,69 +2687,69 @@ "P1": { "per_node": { "0": { - "cost": 0.0390016, - "detail": "scan 15000 samples", + "cost": 0.000010833777777777777, + "detail": "scan 15000 samples; x 0.0003/s", "multiplier": 1 }, "1": { - "cost": 0.015001599999999999, - "detail": "time range 15s: pass 15000 rows", + "cost": 4.167111111111111e-6, + "detail": "time range 15s: pass 15000 rows; x 0.0003/s", "multiplier": 1 }, "10": { - "cost": 0.01508, - "detail": "build exact Sum accumulator into 100 states: 15000 rows x depth 1", + "cost": 4.188888888888888e-6, + "detail": "build exact Sum accumulator into 100 states: 15000 rows x depth 1; x 0.0003/s", "multiplier": 1 }, "11": { - "cost": 0.00009999999999999999, - "detail": "finalize 100 accumulators", + "cost": 2.7777777777777774e-8, + "detail": "finalize 100 accumulators; x 0.0003/s", "multiplier": 1 }, "2": { - "cost": 0.0150008, - "detail": "build exact Sum accumulator into 1 states: 15000 rows x depth 1", + "cost": 4.1668888888888884e-6, + "detail": "build exact Sum accumulator into 1 states: 15000 rows x depth 1; x 0.0003/s", "multiplier": 1 }, "3": { - "cost": 1e-6, - "detail": "finalize 1 accumulators", + "cost": 2.7777777777777777e-10, + "detail": "finalize 1 accumulators; x 0.0003/s", "multiplier": 1 }, "4": { - "cost": 0.0630032, - "detail": "scan 15000 samples", + "cost": 0.000017500888888888888, + "detail": "scan 15000 samples; x 0.0003/s", "multiplier": 1 }, "5": { - "cost": 0.0150032, - "detail": "time range 15s: pass 15000 rows", + "cost": 4.167555555555556e-6, + "detail": "time range 15s: pass 15000 rows; x 0.0003/s", "multiplier": 1 }, "6": { - "cost": 0.0150008, - "detail": "build exact Sum accumulator into 1 states: 15000 rows x depth 1", + "cost": 4.1668888888888884e-6, + "detail": "build exact Sum accumulator into 1 states: 15000 rows x depth 1; x 0.0003/s", "multiplier": 1 }, "7": { - "cost": 1e-6, - "detail": "finalize 1 accumulators", + "cost": 2.7777777777777777e-10, + "detail": "finalize 1 accumulators; x 0.0003/s", "multiplier": 1 }, "8": { - "cost": 0.0870048, - "detail": "scan 15000 samples", + "cost": 0.000024168, + "detail": "scan 15000 samples; x 0.0003/s", "multiplier": 1 }, "9": { - "cost": 0.015004799999999999, - "detail": "time range 15s: pass 15000 rows", + "cost": 4.168e-6, + "detail": "time range 15s: pass 15000 rows; x 0.0003/s", "multiplier": 1 } }, - "source": "analytical-cost-v1 (illustrative statistics)", - "total": 0.2792028, - "unit": "cpu_ms_per_workload_evaluation" + "source": "analytical-cost-v2 (illustrative statistics)", + "total": 0.00007755633333333333, + "unit": "cost_per_second" } }, "rejected": [], From ad5754bd657968f55bf619f609c8d37340184c57 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:38:50 +0000 Subject: [PATCH 06/15] fix(plan-selection): charge memory only as kept state per second Plan selection no longer charges a computation for the memory it holds while it runs (#682's per-computation peak-memory term, including the growing exact state of a query-time aggregate). Memory is charged only as state a materialized node keeps between computations: kept bytes times cost_per_retained_byte_second. The kept state is the node's output, or the values an exact aggregate keeps per group when that is more, so a kept exact quantile or distinct count is still sized by what it keeps. Peak capacity belongs to cost model 2 (#683). Every term is now a per-second cost; the per-computation memory coefficient is ignored. The tests that compared exact state against sketches now keep the state at query time, where its size is charged. --- .../src/stage3_plan_selection.rs | 149 +++++++++++++++--- .../examples/planner-layering-example1.json | 64 ++++---- tools/dag-viewer/examples/scalar-queries.json | 20 +-- 3 files changed, 171 insertions(+), 62 deletions(-) diff --git a/crates/asap-aware-mapping/src/stage3_plan_selection.rs b/crates/asap-aware-mapping/src/stage3_plan_selection.rs index eac4ea7c2..c0de14ba7 100644 --- a/crates/asap-aware-mapping/src/stage3_plan_selection.rs +++ b/crates/asap-aware-mapping/src/stage3_plan_selection.rs @@ -42,17 +42,18 @@ //! queries are spread over the calibration's `horizon_s` (default //! 3600 s): `invocations / horizon_s` per second. //! -//! State a materialized node keeps between computations is charged -//! `cost_per_retained_byte_second` per byte per second: its output's bytes -//! for `MatAtQuery`, and twice them for `MatAtIngestion` (the window being -//! built and the completed one queries read). Memory a query-time -//! computation holds only while it runs is not charged per second; the -//! per-computation memory term below applies to it as to every node. +//! Memory is charged only as state kept between computations: a +//! materialized node's state, `cost_per_retained_byte_second` per byte per +//! second, once for `MatAtQuery` and twice for `MatAtIngestion` (the window +//! being built and the completed one queries read). Memory a computation +//! holds only while it runs is not charged, and `cost_per_retained_byte` is +//! ignored: peak capacity belongs to cost model 2 (#683). A computation is +//! charged for its CPU operations and scanned bytes only. //! //! Prices come from [`crate::analytical_cost::estimate_operator`] over edge //! statistics derived from the [`DataWorkload`] and a fixed default group //! count; summary build and estimation are priced as rows × sketch depth and -//! groups × k. State is priced by its bytes: a sketch by its parameters, a +//! groups × k. Kept state is sized by its bytes: a sketch by its parameters, a //! fixed-size exact accumulator (sum, count, min, max, rate, ...) as one //! value per group, and exact state that grows with the data (quantile, //! top-k, distinct count, L2, entropy keep every value or item) as one value @@ -108,14 +109,15 @@ pub const COST_UNIT: &str = "cost_per_second"; pub const COST_SOURCE: &str = "analytical-cost-v2 (illustrative statistics)"; /// The coefficients plan selection prices with until a deployment is -/// calibrated: 1 ns of CPU per operation, 0.1 ns per scanned or held byte, +/// calibrated: 1 ns of CPU per operation, 0.1 ns per scanned byte, /// 1 GB kept for a second costs 1/8 vCPU (125 CPU-ms), and one-off work is /// spread over one hour. pub fn illustrative_calibration() -> ResourceCalibration { ResourceCalibration { cost_per_cpu_op: 1e-6, cost_per_scan_byte: 1e-7, - cost_per_retained_byte: 1e-7, + // Plan selection charges no memory per computation. + cost_per_retained_byte: 0.0, cost_per_retained_byte_second: 1.25e-7, horizon_s: crate::analytical_cost::DEFAULT_HORIZON_S, version: "illustrative-v2".into(), @@ -852,6 +854,10 @@ fn price( ) -> Result { let first = dag.nodes.first().map(|n| n.id).unwrap_or_default(); calibration.validate().map_err(|error| (first, error))?; + let per_computation = ResourceCalibration { + cost_per_retained_byte: 0.0, + ..calibration.clone() + }; let series = data .input_cardinality .value @@ -1075,18 +1081,24 @@ fn price( format!("{} rows", input.rows), ), }; + let estimate = estimate.map_err(|error| (node.id, error))?; + // A computation is charged for its CPU work and scanned bytes; memory + // only as kept state, below. let cost = estimate - .and_then(|estimate| estimate.calibrated_cost(calibration)) + .calibrated_cost(&per_computation) .map_err(|error| (node.id, error))?; let (multiplier, per_second) = multipliers[&node.id]; let mut cost = cost * per_second; let mut detail = format!("{detail}; x {per_second:.4}/s"); // State kept between computations: the window being built and the // completed one at ingestion time, the kept output at query time. + // The state is what the node holds: its output, or the values an + // exact aggregate keeps per group when that is more. + let state = out.bytes.max(estimate.peak_memory_bytes()); let kept = match node.materialization { Materialization::NotMaterialized => 0, - Materialization::MatAtIngestion(_) => 2 * out.bytes, - _ => out.bytes, + Materialization::MatAtIngestion(_) => 2 * state, + _ => state, }; if kept > 0 { cost += calibration.cost_per_retained_byte_second * kept as f64; @@ -1722,10 +1734,29 @@ mod tests { } } - /// Exact quantile state keeps every value of its group: over one group of - /// 10 000 values it costs more than a quantile sketch sized for + /// `candidates` with every state node (a summary build or an exact + /// aggregate) kept at query time, so selection charges its kept bytes. + fn keeping_state(mut candidates: Vec) -> Vec { + for candidate in &mut candidates { + for node in &mut candidate.dag.nodes { + if matches!( + node.payload, + Payload::ASAP(ASAPOp::SummaryAgg { .. }) + | Payload::NonASAP(NonASAPOp::Aggregate { .. }) + ) { + node.materialization = + Materialization::MatAtQuery(asap_types::ir::Medium::Memory); + } + } + } + candidates + } + + /// Kept exact quantile state keeps every value of its group: over one + /// group of 10 000 values it costs more than a quantile sketch sized for /// ε = 0.01, δ = 0.01, which is selected; over 100 groups of 100 values it - /// is smaller than the sketch and is selected. + /// is smaller than the sketch and is selected. (State held only while a + /// query runs is not charged.) #[test] fn exact_quantile_state_costs_more_than_a_sketch_over_large_groups() { let target = AccuracyTarget::EpsilonDelta { @@ -1736,7 +1767,7 @@ mod tests { ("quantile(0.99, rate(x[5m]))", true), ("quantile by (job) (0.99, rate(x[5m]))", false), ] { - let candidates = all_candidates(query, &target); + let candidates = keeping_state(all_candidates(query, &target)); let selection = select_plan( &candidates, &[Some(target.clone())], @@ -1911,9 +1942,10 @@ mod tests { assert_eq!(ops(16), (2.0 - 2f64.powi(-15)) * 6.0); } - /// Exact distinct-count state keeps every item of its group: over 10 000 - /// groups of 300 rows it is smaller than a distinct-count sketch sized for - /// ε = 0.01 and is selected; over one group of 3 000 000 rows a sketch is. + /// Kept exact distinct-count state keeps every item of its group: over + /// 10 000 groups of 300 rows it is smaller than a distinct-count sketch + /// sized for ε = 0.01 and is selected; over one group of 3 000 000 rows a + /// sketch is. #[test] fn exact_distinct_count_beats_a_sketch_over_small_groups() { let target = AccuracyTarget::EpsilonDelta { @@ -1928,7 +1960,7 @@ mod tests { ..data() }; for (data, sketch_wins) in [(data(), false), (one_series, true)] { - let candidates = all_candidates("distinct_over_time(x[5m])", &target); + let candidates = keeping_state(all_candidates("distinct_over_time(x[5m])", &target)); let selection = select_plan( &candidates, &[Some(target.clone())], @@ -2705,6 +2737,83 @@ mod tests { assert_eq!(cost(&transient, 1.25e-7).total, cost(&transient, 0.0).total); } + /// Memory is charged only as kept bytes per second: the memory a + /// computation holds while it runs (a sketch being built, an exact + /// quantile's values) adds nothing, whatever the per-computation memory + /// coefficient. + #[test] + fn memory_held_only_while_a_query_runs_is_not_charged() { + for query in [ + "quantile_over_time(0.99, m[1m])", + "quantile by (job) (0.99, m)", + ] { + let target = AccuracyTarget::Exact; + let root = lower_promql(query, target); + let dag = optimize_physical_asap("L", &[QueryRoot::Operator(root)]) + .unwrap() + .dag; + let total = |per_computation_memory| { + let calibration = ResourceCalibration { + cost_per_retained_byte: per_computation_memory, + ..illustrative_calibration() + }; + price(&dag, &every_10s(1), &data(), &calibration) + .unwrap() + .total + }; + assert_eq!(total(0.0), total(1.0), "{query}"); + } + for materialization in [AT_INGESTION, AT_QUERY, Materialization::NotMaterialized] { + let dag = kll_dag(materialization); + let total = |per_computation_memory| { + let calibration = ResourceCalibration { + cost_per_retained_byte: per_computation_memory, + ..illustrative_calibration() + }; + price(&dag, &every_10s(1), &data(), &calibration) + .unwrap() + .total + }; + assert_eq!(total(0.0), total(1.0), "{materialization:?}"); + } + } + + /// Kept exact state that grows with the data is charged by the values it + /// keeps: a materialized exact quantile keeps every value of its group, + /// a sum one value per group. + #[test] + fn kept_exact_state_is_charged_by_the_values_it_keeps() { + let kept = |query: &str| { + let root = lower_promql(query, AccuracyTarget::Exact); + let mut dag = optimize_physical_asap("L", &[QueryRoot::Operator(root)]) + .unwrap() + .dag; + let aggregate = dag + .nodes + .iter_mut() + .find(|n| matches!(n.payload, Payload::NonASAP(NonASAPOp::Aggregate { .. }))) + .unwrap(); + aggregate.materialization = AT_QUERY; + let id = aggregate.id; + let cost = |w| { + let calibration = ResourceCalibration { + cost_per_retained_byte_second: w, + ..illustrative_calibration() + }; + price(&dag, &every_10s(1), &data(), &calibration) + .unwrap() + .per_node[&id] + .cost + }; + (cost(1.0) - cost(0.0)).round() as u64 + }; + let (quantile, sum) = ( + kept("quantile by (job) (0.99, m)"), + kept("sum by (job) (m)"), + ); + assert!(quantile > 10 * sum, "{quantile} vs {sum}"); + } + /// A `NotMaterialized` node read by two queries is computed for each, at /// each one's rate; `MatAtQuery`, it is computed once at their combined /// rate, where queries at the same interval run together. diff --git a/tools/dag-viewer/examples/planner-layering-example1.json b/tools/dag-viewer/examples/planner-layering-example1.json index c8b820cdd..bc7878e9c 100644 --- a/tools/dag-viewer/examples/planner-layering-example1.json +++ b/tools/dag-viewer/examples/planner-layering-example1.json @@ -19651,32 +19651,32 @@ "P1": { "per_node": { "0": { - "cost": 2.32000048, + "cost": 2.32, "detail": "scan 4000000 samples; x 0.1000/s", "multiplier": 1 }, "1": { - "cost": 0.40000048000000005, + "cost": 0.4, "detail": "time range 60s: pass 4000000 rows; x 0.1000/s", "multiplier": 1 }, "10": { - "cost": 1.4048, + "cost": 1.4000000000000001, "detail": "sort 1000000 rows in 100 partitions; x 0.1000/s", "multiplier": 1 }, "11": { - "cost": 0.00010048, + "cost": 0.0001, "detail": "limit to 1000 rows; x 0.1000/s", "multiplier": 1 }, "12": { - "cost": 0.00010048, + "cost": 0.0001, "detail": "operator: pass 1000 rows; x 0.1000/s", "multiplier": 1 }, "2": { - "cost": 0.48, + "cost": 0.4, "detail": "build exact Rate accumulator into 1000000 states: 4000000 rows x depth 1; x 0.1000/s", "multiplier": 1 }, @@ -19686,7 +19686,7 @@ "multiplier": 1 }, "4": { - "cost": 0.10000800000000001, + "cost": 0.1, "detail": "build exact Sum accumulator into 100 states: 1000000 rows x depth 1; x 0.1000/s", "multiplier": 1 }, @@ -19696,17 +19696,17 @@ "multiplier": 1 }, "6": { - "cost": 2.32000048, + "cost": 2.32, "detail": "scan 4000000 samples; x 0.1000/s", "multiplier": 1 }, "7": { - "cost": 0.40000048000000005, + "cost": 0.4, "detail": "time range 60s: pass 4000000 rows; x 0.1000/s", "multiplier": 1 }, "8": { - "cost": 0.48, + "cost": 0.4, "detail": "build exact Sum accumulator into 1000000 states: 4000000 rows x depth 1; x 0.1000/s", "multiplier": 1 }, @@ -19717,33 +19717,33 @@ } }, "source": "analytical-cost-v2 (illustrative statistics)", - "total": 8.10502088, + "total": 7.94021, "unit": "cost_per_second" }, "P2": { "per_node": { "0": { - "cost": 4.64000096, + "cost": 4.64, "detail": "scan 4000000 samples; x 0.2000/s", "multiplier": 2 }, "1": { - "cost": 0.40000048000000005, + "cost": 0.4, "detail": "time range 60s: pass 4000000 rows; x 0.1000/s", "multiplier": 1 }, "10": { - "cost": 0.00010048, + "cost": 0.0001, "detail": "limit to 1000 rows; x 0.1000/s", "multiplier": 1 }, "11": { - "cost": 0.00010048, + "cost": 0.0001, "detail": "operator: pass 1000 rows; x 0.1000/s", "multiplier": 1 }, "2": { - "cost": 0.48, + "cost": 0.4, "detail": "build exact Rate accumulator into 1000000 states: 4000000 rows x depth 1; x 0.1000/s", "multiplier": 1 }, @@ -19753,7 +19753,7 @@ "multiplier": 1 }, "4": { - "cost": 0.10000800000000001, + "cost": 0.1, "detail": "build exact Sum accumulator into 100 states: 1000000 rows x depth 1; x 0.1000/s", "multiplier": 1 }, @@ -19763,12 +19763,12 @@ "multiplier": 1 }, "6": { - "cost": 0.40000048000000005, + "cost": 0.4, "detail": "time range 60s: pass 4000000 rows; x 0.1000/s", "multiplier": 1 }, "7": { - "cost": 0.48, + "cost": 0.4, "detail": "build exact Sum accumulator into 1000000 states: 4000000 rows x depth 1; x 0.1000/s", "multiplier": 1 }, @@ -19778,34 +19778,34 @@ "multiplier": 1 }, "9": { - "cost": 1.4048, + "cost": 1.4000000000000001, "detail": "sort 1000000 rows in 100 partitions; x 0.1000/s", "multiplier": 1 } }, "source": "analytical-cost-v2 (illustrative statistics)", - "total": 8.10502088, + "total": 7.9402099999999995, "unit": "cost_per_second" }, "P3": { "per_node": { "0": { - "cost": 4.64000096, + "cost": 4.64, "detail": "scan 4000000 samples; x 0.2000/s", "multiplier": 2 }, "1": { - "cost": 0.8000009600000001, + "cost": 0.8, "detail": "time range 60s: pass 4000000 rows; x 0.2000/s", "multiplier": 2 }, "10": { - "cost": 0.00010048, + "cost": 0.0001, "detail": "operator: pass 1000 rows; x 0.1000/s", "multiplier": 1 }, "2": { - "cost": 0.48, + "cost": 0.4, "detail": "build exact Rate accumulator into 1000000 states: 4000000 rows x depth 1; x 0.1000/s", "multiplier": 1 }, @@ -19815,7 +19815,7 @@ "multiplier": 1 }, "4": { - "cost": 0.10000800000000001, + "cost": 0.1, "detail": "build exact Sum accumulator into 100 states: 1000000 rows x depth 1; x 0.1000/s", "multiplier": 1 }, @@ -19825,7 +19825,7 @@ "multiplier": 1 }, "6": { - "cost": 0.48, + "cost": 0.4, "detail": "build exact Sum accumulator into 1000000 states: 4000000 rows x depth 1; x 0.1000/s", "multiplier": 1 }, @@ -19835,18 +19835,18 @@ "multiplier": 1 }, "8": { - "cost": 1.4048, + "cost": 1.4000000000000001, "detail": "sort 1000000 rows in 100 partitions; x 0.1000/s", "multiplier": 1 }, "9": { - "cost": 0.00010048, + "cost": 0.0001, "detail": "limit to 1000 rows; x 0.1000/s", "multiplier": 1 } }, "source": "analytical-cost-v2 (illustrative statistics)", - "total": 8.10502088, + "total": 7.940209999999999, "unit": "cost_per_second" } }, @@ -19883,12 +19883,12 @@ }, { "id": "P2", - "reason": "tied with the selected candidate P1 at 8.105 cost_per_second; lost on enumeration order", + "reason": "tied with the selected candidate P1 at 7.940 cost_per_second; lost on enumeration order", "valid": true }, { "id": "P3", - "reason": "tied with the selected candidate P1 at 8.105 cost_per_second; lost on enumeration order", + "reason": "tied with the selected candidate P1 at 7.940 cost_per_second; lost on enumeration order", "valid": true } ], diff --git a/tools/dag-viewer/examples/scalar-queries.json b/tools/dag-viewer/examples/scalar-queries.json index 1f8b7839b..6d861ee4f 100644 --- a/tools/dag-viewer/examples/scalar-queries.json +++ b/tools/dag-viewer/examples/scalar-queries.json @@ -2687,17 +2687,17 @@ "P1": { "per_node": { "0": { - "cost": 0.000010833777777777777, + "cost": 0.000010833333333333334, "detail": "scan 15000 samples; x 0.0003/s", "multiplier": 1 }, "1": { - "cost": 4.167111111111111e-6, + "cost": 4.166666666666667e-6, "detail": "time range 15s: pass 15000 rows; x 0.0003/s", "multiplier": 1 }, "10": { - "cost": 4.188888888888888e-6, + "cost": 4.166666666666667e-6, "detail": "build exact Sum accumulator into 100 states: 15000 rows x depth 1; x 0.0003/s", "multiplier": 1 }, @@ -2707,7 +2707,7 @@ "multiplier": 1 }, "2": { - "cost": 4.1668888888888884e-6, + "cost": 4.166666666666667e-6, "detail": "build exact Sum accumulator into 1 states: 15000 rows x depth 1; x 0.0003/s", "multiplier": 1 }, @@ -2717,17 +2717,17 @@ "multiplier": 1 }, "4": { - "cost": 0.000017500888888888888, + "cost": 0.0000175, "detail": "scan 15000 samples; x 0.0003/s", "multiplier": 1 }, "5": { - "cost": 4.167555555555556e-6, + "cost": 4.166666666666667e-6, "detail": "time range 15s: pass 15000 rows; x 0.0003/s", "multiplier": 1 }, "6": { - "cost": 4.1668888888888884e-6, + "cost": 4.166666666666667e-6, "detail": "build exact Sum accumulator into 1 states: 15000 rows x depth 1; x 0.0003/s", "multiplier": 1 }, @@ -2737,18 +2737,18 @@ "multiplier": 1 }, "8": { - "cost": 0.000024168, + "cost": 0.000024166666666666663, "detail": "scan 15000 samples; x 0.0003/s", "multiplier": 1 }, "9": { - "cost": 4.168e-6, + "cost": 4.166666666666667e-6, "detail": "time range 15s: pass 15000 rows; x 0.0003/s", "multiplier": 1 } }, "source": "analytical-cost-v2 (illustrative statistics)", - "total": 0.00007755633333333333, + "total": 0.00007752833333333333, "unit": "cost_per_second" } }, From 2381c151b3bd6910394f98ae464897c9e88e2db1 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:55:37 +0000 Subject: [PATCH 07/15] feat(plan-selection): bind the sign of confirmed counter samples An exact sum or mean of relative-error values keeps the relative bound only over non-negative values. When the folded values are raw samples of one metric, or plain sums of them (an exact sum accumulator, a renaming, or a heap estimate whose weights carry the SamplesOfMetric requirement), Stage 1 now names the unknown sign by that metric, values_non_negative[M], and gates the bound on it. Plan selection binds it to 1 when the data workload declares M a counter; a gauge, an undeclared metric, rate, other arithmetic or a name such as _total leave it unbound, so the target stays unmet. Other folds keep the bare values_non_negative, which nothing binds. --- .../src/accuracy/composition.rs | 80 ++++++++++++- .../src/accuracy/evidence.rs | 7 ++ .../src/per_computation_candidates.rs | 108 +++++++++++++++++- .../src/stage3_plan_selection.rs | 56 ++++++++- crates/types/src/workload.rs | 1 - 5 files changed, 241 insertions(+), 11 deletions(-) diff --git a/crates/asap-aware-mapping/src/accuracy/composition.rs b/crates/asap-aware-mapping/src/accuracy/composition.rs index 1d9442799..b0b738c4f 100644 --- a/crates/asap-aware-mapping/src/accuracy/composition.rs +++ b/crates/asap-aware-mapping/src/accuracy/composition.rs @@ -48,6 +48,23 @@ pub(crate) fn source_norm_statistic(norm: &str, source: &str) -> String { format!("{norm}[{source}]") } +/// Whether every value folded is non-negative, for values that are raw +/// samples of `metric` or plain sums of them, e.g. +/// `values_non_negative[m]`: an indicator, 1 when proven. Plan selection +/// binds it to 1 when the data workload declares `metric` a counter, and +/// leaves it unbound otherwise. +pub(crate) fn samples_sign_statistic(metric: &str) -> String { + format!("{VALUES_NON_NEGATIVE}[{metric}]") +} + +/// The metric a [`samples_sign_statistic`] names. +pub(crate) fn parse_samples_sign_statistic(statistic: &str) -> Option<&str> { + statistic + .strip_prefix(VALUES_NON_NEGATIVE)? + .strip_prefix('[')? + .strip_suffix(']') +} + /// The norm and source a [`source_norm_statistic`] names. pub(crate) fn parse_source_norm_statistic(statistic: &str) -> Option<(&'static str, &str)> { [STREAM_L1_NORM, STREAM_L2_NORM] @@ -352,11 +369,31 @@ impl DefaultAccuracyModel { let mut provenance = Vec::new(); let count = row_count(stats, &mut provenance); let bound = if needs_sign && stats.values_non_negative.is_none() { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: VALUES_NON_NEGATIVE.into(), - }); - BoundExpr::Unknown { - statistic: VALUES_NON_NEGATIVE.into(), + match &stats.sign_statistic { + // The sign of one metric's samples: the bound, gated on it. + Some(statistic) => { + provenance.push(GuaranteeSource::UnavailableStatistic { + statistic: statistic.clone(), + }); + BoundExpr::Product { + factors: vec![ + BoundExpr::Max { + terms: inputs.iter().map(|g| g.bound.clone()).collect(), + }, + BoundExpr::Unknown { + statistic: statistic.clone(), + }, + ], + } + } + None => { + provenance.push(GuaranteeSource::UnavailableStatistic { + statistic: VALUES_NON_NEGATIVE.into(), + }); + BoundExpr::Unknown { + statistic: VALUES_NON_NEGATIVE.into(), + } + } } } else { BoundExpr::Max { @@ -1292,6 +1329,39 @@ mod tests { assert!(with(Some(false)).is_err(), "{op:?}"); } } + /// Over the samples of one metric, an unknown sign is named by that + /// metric ([`samples_sign_statistic`]) and gates the bound: binding it + /// to 1 (proven) gives the relative bound, and left unbound it stays + /// unknown. + #[test] + fn a_sign_over_one_metrics_samples_gates_the_relative_bound() { + let statistic = samples_sign_statistic("m"); + assert_eq!(parse_samples_sign_statistic(&statistic), Some("m")); + assert_eq!(parse_samples_sign_statistic(VALUES_NON_NEGATIVE), None); + for op in [ + CompositionOperator::ExactSum, + CompositionOperator::ExactAverage, + ] { + let out = DefaultAccuracyModel + .propagate( + &op, + &[rel(0.01)], + None, + &PropagationStats { + sign_statistic: Some(statistic.clone()), + input_row_count: Some(3), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(out.bound.evaluate(), None, "{op:?}"); + let proven = out.bind(&|s| (s == statistic).then_some(1.0)); + assert_eq!(proven.bound.evaluate(), Some(0.01), "{op:?}"); + let other = out.bind(&|s| (s == samples_sign_statistic("n")).then_some(1.0)); + assert_eq!(other.bound.evaluate(), None, "{op:?}"); + } + } + #[test] fn exact_average_has_its_own_absolute_error_rule() { let out = DefaultAccuracyModel diff --git a/crates/asap-aware-mapping/src/accuracy/evidence.rs b/crates/asap-aware-mapping/src/accuracy/evidence.rs index e7b2aff01..fcb316575 100644 --- a/crates/asap-aware-mapping/src/accuracy/evidence.rs +++ b/crates/asap-aware-mapping/src/accuracy/evidence.rs @@ -57,6 +57,13 @@ pub struct PropagationStats { /// the multiplicative relative-error rule, which is unsound across a /// sign change. pub values_non_negative: Option, + /// The statistic an unknown sign is named by when the folded values are + /// one metric's raw samples or plain sums of them + /// ([`samples_sign_statistic`](crate::accuracy::composition::samples_sign_statistic)): + /// the relative bound is then gated on it, and plan selection binds it + /// to 1 when the metric is declared a counter. `None`: the bare + /// `values_non_negative`, which nothing binds. + pub sign_statistic: Option, /// Number of input rows an exact aggregation consumes (e.g. the number /// of groups a function folds), for exact aggregate union bounds /// bound over per-input failures. diff --git a/crates/asap-aware-mapping/src/per_computation_candidates.rs b/crates/asap-aware-mapping/src/per_computation_candidates.rs index 4fb8a9083..681addda6 100644 --- a/crates/asap-aware-mapping/src/per_computation_candidates.rs +++ b/crates/asap-aware-mapping/src/per_computation_candidates.rs @@ -90,6 +90,7 @@ use asap_types::pre_asap::{ use asap_types::types::AccuracyTarget; use thiserror::Error; +use crate::accuracy::composition::samples_sign_statistic; use crate::accuracy::composition::{ additive_as_absolute, StreamNorm, DIVISION_OPERAND_DOMAINS, GROUP_ROW_COUNT, PRODUCT_OPERAND_DOMAINS, QUOTIENT_OPERAND_DOMAINS, @@ -1017,7 +1018,8 @@ fn classic_hll_guarantee( /// subquery with an explicit resolution, `⌈range / resolution⌉`. `None` /// when that count is not in the query. fn fold_stats(reduction: &Reduction, input: &Rc) -> Option { - match reduction { + let sign_statistic = estimates_samples_of(input).map(|metric| samples_sign_statistic(&metric)); + let stats = match reduction { // The input cardinality bounds a PromQL fold's rows (series), not // the rows of a SQL group. Reduction::Reduce(_) if reads_table(input) => Some(PropagationStats { @@ -1036,6 +1038,54 @@ fn fold_stats(reduction: &Reduction, input: &Rc) -> Option None, }, + }?; + Some(PropagationStats { + sign_statistic, + ..stats + }) +} + +/// The metric whose raw samples, or plain sums of them, a composed node's +/// values are or estimate, or `None`: [`samples_of_metric`]'s strict rule +/// over a composed candidate. An exact sum accumulator, a renaming and a +/// heap estimate whose update weights carry the +/// [`WeightDomain::SamplesOfMetric`] requirement keep it; anything else +/// (rate, arithmetic, other functions) ends it. +fn estimates_samples_of(node: &OperatorNode) -> Option { + match &node.operator { + Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) => match child.asap()? { + ASAPOp::SummaryAgg { + family: FieldDataType::ExactAggregate(ExactKind::Sum, _), + child, + filter: None, + .. + } => estimates_samples_of(child), + _ => None, + }, + Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { + match summary_input.asap()? { + ASAPOp::SummaryAgg { + input: + SummaryUpdate { + weight_domain: WeightDomain::SamplesOfMetric { metric }, + .. + }, + filter: None, + .. + } => Some(metric.clone()), + _ => None, + } + } + Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) + if cols.len() == child.schema.fields.len() + && cols + .iter() + .enumerate() + .all(|(i, item)| item.expr == ScalarExpr::Column(i)) => + { + estimates_samples_of(child) + } + _ => samples_of_metric(node), } } @@ -2970,4 +3020,60 @@ mod tests { assert_eq!(inventory.combinations(), (n - whole) * m + whole); assert_eq!(inventory.combinations(), enumerated); } + + /// The metric whose raw samples, or plain sums of them, a composed node's + /// values estimate: a whole-expression heap's estimates and an exact + /// sum over the samples are; a rate or any other function is not. A + /// fold over them names its unknown sign by that metric. + #[test] + fn values_estimating_samples_name_their_sign_by_the_metric() { + let inventory = inventory("topk by (job) (10, sum_over_time(m[1m]))"); + let topk = topk_target(&inventory); + for (candidate, root) in composed_with(&inventory, topk) { + let estimate = OperatorNode::reachable(&root) + .into_iter() + .find(|n| { + matches!( + n.asap(), + Some( + ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + ) + ) + }) + .unwrap(); + assert_eq!( + estimates_samples_of(&estimate).as_deref(), + Some("m"), + "{candidate:?}" + ); + let stats = fold_stats(&Reduction::by(vec![]), &estimate).unwrap(); + assert_eq!( + stats.sign_statistic, + Some(crate::accuracy::composition::samples_sign_statistic("m")) + ); + } + let rate = inventory_root("sum by (job) (rate(m[1m]))"); + let finalized = OperatorNode::reachable(&rate) + .into_iter() + .filter(|n| matches!(n.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. }))) + .collect::>(); + assert!(!finalized.is_empty()); + for node in finalized { + assert_eq!(estimates_samples_of(&node), None); + } + let stats = fold_stats(&Reduction::by(vec![]), &rate).unwrap(); + assert_eq!(stats.sign_statistic, None); + } + + /// The first composed candidate of `query`. + fn inventory_root(query: &str) -> Rc { + let inventory = inventory(query); + let composed = + compose_logical_candidate(&inventory, &vec![0; inventory.targets.len()]).unwrap(); + let QueryRoot::Operator(root) = &composed[0].1 else { + panic!("operator root") + }; + root.clone() + } } diff --git a/crates/asap-aware-mapping/src/stage3_plan_selection.rs b/crates/asap-aware-mapping/src/stage3_plan_selection.rs index c0de14ba7..4581cf49e 100644 --- a/crates/asap-aware-mapping/src/stage3_plan_selection.rs +++ b/crates/asap-aware-mapping/src/stage3_plan_selection.rs @@ -85,8 +85,8 @@ use asap_types::workload::{DataWorkload, MetricType, QueryRecurrence, RepeatedDe use thiserror::Error; use crate::accuracy::composition::{ - parse_source_norm_statistic, DIVISION_OPERAND_DOMAINS, INPUT_ROW_COUNT, - PRODUCT_OPERAND_DOMAINS, STREAM_L1_NORM, + parse_samples_sign_statistic, parse_source_norm_statistic, DIVISION_OPERAND_DOMAINS, + INPUT_ROW_COUNT, PRODUCT_OPERAND_DOMAINS, STREAM_L1_NORM, }; use crate::accuracy::estimators::hll; use crate::accuracy::{OperandDomain, PropagationStats}; @@ -450,11 +450,17 @@ fn bound_root_guarantee( } /// A data-workload statistic Stage 1 left symbolic: the input row count, -/// one source's stream norm, or classic HLL's failure probability given one -/// source's distinct bound, from evidence that holds at any planning time. +/// one source's stream norm or classic HLL's failure probability given one +/// source's distinct bound, from evidence that holds at any planning time, +/// or the sign of one metric's samples, from its declared type. fn workload_statistic(statistic: &str, data: &DataWorkload) -> Option { // Without a planning time, only evidence that cannot expire is fresh. const ANY_TIME: u64 = u64::MAX; + // The sign of one metric's samples: proven (1) when the metric is + // declared a counter; no type is inferred from its name. + if let Some(metric) = parse_samples_sign_statistic(statistic) { + return (data.metric_types.get(metric) == Some(&MetricType::Counter)).then_some(1.0); + } if statistic == INPUT_ROW_COUNT { return data .input_cardinality @@ -1473,6 +1479,48 @@ mod tests { } } + /// A relative bound gated on the sign of one metric's samples is bound + /// when the data workload declares that metric a counter, and stays + /// unknown for a gauge, an undeclared metric or a counter by another + /// name. + #[test] + fn sign_of_counter_samples_is_bound_from_the_declared_type() { + use crate::accuracy::composition::samples_sign_statistic; + let gated = AccuracyGuarantee { + metric: ErrorMetric::RelativeValue, + bound: BoundExpr::Product { + factors: vec![ + BoundExpr::Constant { value: 0.01 }, + BoundExpr::Unknown { + statistic: samples_sign_statistic("m"), + }, + ], + }, + failure_probability: ProbabilityExpr::Constant { value: 0.001 }, + provenance: vec![], + }; + let node = lower_promql("sum(m)", AccuracyTarget::Exact); + let root = QueryRoot::Operator(Rc::new((*node).clone().with_guarantee(Some(gated)))); + let with = |types: &[(&str, MetricType)]| DataWorkload { + metric_types: types.iter().map(|(m, t)| (m.to_string(), *t)).collect(), + ..data() + }; + let bound = |data: &DataWorkload| { + bound_root_guarantee(&root, data, &PlanningModels::builtin()) + .unwrap() + .bound + .evaluate() + }; + assert_eq!(bound(&with(&[("m", MetricType::Counter)])), Some(0.01)); + for types in [ + vec![("m", MetricType::Gauge)], + vec![("m_total", MetricType::Counter)], + vec![], + ] { + assert_eq!(bound(&with(&types)), None, "{types:?}"); + } + } + /// The selected plan's open requirement becomes a proof when confirmed /// and stays unproven otherwise; the executor reads no data workload. #[test] diff --git a/crates/types/src/workload.rs b/crates/types/src/workload.rs index 140865e0f..c3da37ec3 100644 --- a/crates/types/src/workload.rs +++ b/crates/types/src/workload.rs @@ -557,7 +557,6 @@ pub struct DataWorkload { /// without a bound leaves that probability unknown. #[serde(default)] pub max_distinct_per_evaluation: Evidence>, - /// Declared type of each metric, by metric name (Prometheus `# TYPE` /// metadata). The planner never infers a type from a name such as /// `_total`; an undeclared metric's samples may take any value. From ab17dc1ddc5956137382c654dfe98f1f04e11a40 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:17:18 +0000 Subject: [PATCH 08/15] feat(devtools): limit Stage 1 candidates, Pass 2's included, while enumerating stage_pipeline's --max-combinations (default 10^4) now counts Stage 1 candidates as they are enumerated, Pass 2's identical-expression candidates included, and stops with an error as soon as the count passes the limit; it never truncates. Exit status 3 (the viewer server's 422) is unchanged. The library stays uncapped: Stage 1 is exposed as a lazy iterator (stage1_logical_asap_optimization) and Stage 2 and Stage 3 run over the candidates a caller enumerated (run_stage2_and_stage3); run_stage1_to_stage3_from_pass1 enumerates them all. planning_stages takes the limit as an Option. --- crates/asap-aware-mapping/src/pass/mod.rs | 5 +- .../src/pass/stage1_to_stage3.rs | 168 ++++++++++++------ crates/devtools/src/bin/stage_pipeline.rs | 36 ++-- crates/devtools/src/planning_stages.rs | 53 +++--- crates/devtools/tests/stage_pipeline.rs | 63 ++++--- 5 files changed, 196 insertions(+), 129 deletions(-) diff --git a/crates/asap-aware-mapping/src/pass/mod.rs b/crates/asap-aware-mapping/src/pass/mod.rs index de7e9ac02..3f37907a8 100644 --- a/crates/asap-aware-mapping/src/pass/mod.rs +++ b/crates/asap-aware-mapping/src/pass/mod.rs @@ -35,8 +35,9 @@ use crate::per_computation_candidates::PerComputationCandidateError; use crate::stage3_plan_selection::{Selection, SelectionError}; pub use stage1_to_stage3::{ - run_stage1_to_stage3, run_stage1_to_stage3_from_pass1, IdenticalExpressionCandidate, - LogicalCandidate, PrunedCandidate, Stage1ToStage3Pass, Stage1ToStage3Run, + run_stage1_to_stage3, run_stage1_to_stage3_from_pass1, run_stage2_and_stage3, + stage1_logical_asap_optimization, IdenticalExpressionCandidate, LogicalCandidate, + PrunedCandidate, Stage1Candidate, Stage1ToStage3Pass, Stage1ToStage3Run, }; static DEFAULT_COST_MODEL: DefaultCostModel = DefaultCostModel; diff --git a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs index dd6050c47..60abb16a7 100644 --- a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs +++ b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs @@ -151,9 +151,84 @@ pub fn run_stage1_to_stage3( )) } -/// [`run_stage1_to_stage3`] after Pass 1: enumerates every whole-workload +/// One Stage 1 candidate in enumeration order, before Stage 2 numbers it. +#[derive(Debug)] +pub struct Stage1Candidate { + /// Human-readable name, used only for display. + pub display_name: String, + /// The Pass 1 choice (`targets[t].choices.candidates[choice[t]]`); a + /// Pass 2 candidate has its Pass 1 candidate's. + pub choice: Vec, + pub roots: Result, PerComputationCandidateError>, + /// For a Pass 2 candidate: the position, in the enumeration, of the + /// Pass 1 candidate it shares the identical sub-DAGs of, and the shared + /// groups (operator kinds, children first). + pub pass2_of: Option<(usize, Vec)>, +} + +/// Stage 1 (logical ASAP-aware optimization) over the Pass 1 inventory, one +/// candidate at a time: each composed Pass 1 candidate (every choice of a +/// per-computation candidate for every target, the last target varying +/// fastest), followed by its Pass 2 identical-expression candidates. Lazy +/// and uncapped: a caller that limits the candidates stops consuming it. +pub fn stage1_logical_asap_optimization( + per_computation: &PerComputationCandidates, +) -> impl Iterator + '_ { + let query_of_target = query_of_target(per_computation); + let mut choice = Some(vec![0; per_computation.targets.len()]); + let mut queue = std::collections::VecDeque::new(); + let mut position = 0usize; + std::iter::from_fn(move || { + while queue.is_empty() { + let current = choice.as_mut()?; + let skip = per_computation.absorbed_choice_varies(current); + let this = current.clone(); + if !next_choice(per_computation, current) { + choice = None; + } + if skip { + continue; + } + let display_name = display_name(per_computation, &query_of_target, &this); + let roots = compose_logical_candidate(per_computation, &this) + .map(|roots| roots.into_iter().map(|(_, root)| root).collect::>()); + let (groups, variants) = match &roots { + Ok(roots) => ( + crate::asap_aware_cse::sharing_groups(roots), + identical_expression_variants(roots), + ), + Err(_) => (Vec::new(), Vec::new()), + }; + let from = position + queue.len(); + queue.push_back(Stage1Candidate { + display_name: display_name.clone(), + choice: this.clone(), + roots, + pass2_of: None, + }); + for variant in variants { + let shared: Vec = variant + .shared_groups + .iter() + .map(|&g| groups[g].nodes[0].operator.kind_name().to_string()) + .collect(); + queue.push_back(Stage1Candidate { + display_name: format!("{display_name} · shared {}", shared.join(", ")), + choice: this.clone(), + roots: Ok(variant.roots), + pass2_of: Some((from, shared)), + }); + } + } + position += 1; + queue.pop_front() + }) +} + +/// [`run_stage1_to_stage3`] after Pass 1: enumerates every Stage 1 /// candidate of `per_computation` (the Pass 1 inventory of the frontends' -/// roots), so a caller can read the inventory, e.g. its +/// roots, [`stage1_logical_asap_optimization`]), so a caller can read the +/// inventory, e.g. its /// [`combinations`](PerComputationCandidates::combinations), before /// enumerating. pub fn run_stage1_to_stage3_from_pass1( @@ -162,82 +237,57 @@ pub fn run_stage1_to_stage3_from_pass1( recurrence: &[QueryRecurrence], data: &DataWorkload, models: PlanningModels<'_>, +) -> Stage1ToStage3Run { + let stage1 = stage1_logical_asap_optimization(&per_computation).collect(); + run_stage2_and_stage3(per_computation, stage1, targets, recurrence, data, models) +} + +/// Stage 2 and Stage 3 over `stage1`, the Stage 1 candidates of +/// `per_computation` in enumeration order (all of them, or the ones a caller +/// enumerated): candidate `n` (from 1) is `L`, implemented as `P`. +pub fn run_stage2_and_stage3( + per_computation: PerComputationCandidates, + stage1: Vec, + targets: &[Option], + recurrence: &[QueryRecurrence], + data: &DataWorkload, + models: PlanningModels<'_>, ) -> Stage1ToStage3Run { let combinations = per_computation.combinations(); - let query_of_target = query_of_target(&per_computation); let mut logical: Vec = Vec::new(); let mut physical = Vec::new(); let mut pruned = Vec::new(); let mut pass2 = Vec::new(); - // Stage 2 for one Stage 1 candidate, numbered after the ones before it. - let mut add = |logical: &mut Vec, - display_name: String, - choice: &[usize], - roots: Result, PerComputationCandidateError>| { - let number = logical.len() + 1; + for (index, candidate) in stage1.into_iter().enumerate() { + let number = index + 1; let id = format!("L{number}"); - if let Ok(roots) = &roots { + if let Ok(roots) = &candidate.roots { match optimize_physical_asap(&id, roots) { - Ok(mut candidate) => { - candidate.id = format!("P{number}"); - candidate.display_name = display_name.clone(); - physical.push(candidate); + Ok(mut implemented) => { + implemented.id = format!("P{number}"); + implemented.display_name = candidate.display_name.clone(); + physical.push(implemented); } Err(error) => pruned.push(PrunedCandidate { from_logical: id.clone(), - display_name: display_name.clone(), + display_name: candidate.display_name.clone(), reason: error.to_string(), }), } } - logical.push(LogicalCandidate { - id: id.clone(), - display_name, - choice: choice.to_vec(), - roots, - }); - id - }; - let mut choice = vec![0; per_computation.targets.len()]; - loop { - if per_computation.absorbed_choice_varies(&choice) { - if next_choice(&per_computation, &mut choice) { - continue; - } - break; - } - let display_name = display_name(&per_computation, &query_of_target, &choice); - let roots = compose_logical_candidate(&per_computation, &choice) - .map(|roots| roots.into_iter().map(|(_, root)| root).collect::>()); - let (groups, variants) = match &roots { - Ok(roots) => ( - crate::asap_aware_cse::sharing_groups(roots), - identical_expression_variants(roots), - ), - Err(_) => (Vec::new(), Vec::new()), - }; - let from_logical = add(&mut logical, display_name.clone(), &choice, roots); - for variant in variants { - let shared: Vec = variant - .shared_groups - .iter() - .map(|&g| groups[g].nodes[0].operator.kind_name().to_string()) - .collect(); - let id = add( - &mut logical, - format!("{display_name} · shared {}", shared.join(", ")), - &choice, - Ok(variant.roots), - ); + if let Some((from, shared)) = candidate.pass2_of { pass2.push(IdenticalExpressionCandidate { - id, - from_logical: from_logical.clone(), + id: id.clone(), + from_logical: format!("L{}", from + 1), shared, }); } - if !next_choice(&per_computation, &mut choice) { - break; - } + logical.push(LogicalCandidate { + id, + display_name: candidate.display_name, + choice: candidate.choice, + roots: candidate.roots, + }); } let selection = select_plan(&physical, targets, recurrence, data, models); Stage1ToStage3Run { diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index 42e807ce7..fb72cb70a 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -35,10 +35,10 @@ // When Pass 1 fails, the document still has the Stage 0 DAG; Stage 1 has // `error` and no candidates, and nothing is selected. // -// A workload with more than `--max-combinations` (default 10^4) Pass 1 -// whole-workload combinations is refused before Stage 1 enumerates any (Pass -// 2 adds candidates to each, which is known only once it is composed): the -// run exits with status 3, the count and the limit, and writes no document. +// A workload with more than `--max-combinations` (default 10^4) Stage 1 +// candidates, Pass 2's included, is refused as soon as enumeration passes +// the limit: the run exits with status 3 and the limit, and writes no +// document. // Planning never caps or truncates the candidates it ranks; only the display // does (`--max-displayed`). // @@ -79,16 +79,16 @@ const USAGE: &str = [--name ]... [--table ...] [--epsilon --delta ] [--interval-ms ])) \ [--max-combinations ] [--max-displayed ] --out "; -/// The default `--max-combinations`: the most Pass 1 whole-workload -/// combinations a run enumerates. +/// The default `--max-combinations`: the most Stage 1 candidates (Pass 2's +/// included) a run enumerates. const MAX_COMBINATIONS: usize = 10_000; /// Why a run wrote no document. enum Failure { /// Bad arguments or a query that does not plan (exit status 2). Usage(String), - /// More candidate combinations than `--max-combinations` (exit status 3). - TooManyCombinations { combinations: usize, limit: usize }, + /// More Stage 1 candidates than `--max-combinations` (exit status 3). + TooManyCandidates(PlanningStagesError), } impl From for Failure { @@ -110,14 +110,8 @@ fn main() { eprintln!("stage_pipeline: {message}\n{USAGE}"); std::process::exit(2); } - Err(Failure::TooManyCombinations { - combinations, - limit, - }) => { - eprintln!( - "stage_pipeline: workload has {combinations} candidate combinations \ -(limit {limit}); planning it needs early pruning (#673)" - ); + Err(Failure::TooManyCandidates(error)) => { + eprintln!("stage_pipeline: {error}"); std::process::exit(3); } } @@ -148,13 +142,9 @@ fn document_of( "stage2_physical_asap_optimization": { "candidates": [], "pruned": [] }, "stage3_plan_selection": { "costs": {}, "selected": null, "rejected": [] }, })), - Err(PlanningStagesError::TooManyCombinations { - combinations, - limit, - }) => Err(Failure::TooManyCombinations { - combinations, - limit, - }), + Err(error @ PlanningStagesError::TooManyCandidates { .. }) => { + Err(Failure::TooManyCandidates(error)) + } Err(error @ PlanningStagesError::Validation(_)) => Err(error.to_string().into()), } } diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index 1b5171a9e..0468037c2 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -2,7 +2,7 @@ //! one pipeline behind `stage_pipeline` and the planner-layering acceptance //! tests, so both see the same candidates, ids and selection. -use asap_aware_mapping::pass::run_stage1_to_stage3_from_pass1; +use asap_aware_mapping::pass::{run_stage2_and_stage3, stage1_logical_asap_optimization}; pub use asap_aware_mapping::pass::{LogicalCandidate, PrunedCandidate}; use asap_aware_mapping::per_computation_candidates::generate_per_computation_candidates; use asap_aware_mapping::stage2_physical_asap_optimization::PhysicalCandidate; @@ -64,9 +64,9 @@ pub fn lower_sql_roots( pub enum PlanningStagesError { /// Pass 1 failed: there are no candidates, only the frontends' roots. Pass1(String), - /// Stage 1 would enumerate more whole-workload candidates than the - /// caller's limit; nothing was enumerated. - TooManyCombinations { combinations: usize, limit: usize }, + /// Stage 1 enumerated more candidates (Pass 1's and Pass 2's) than the + /// caller's limit; enumeration stopped there and nothing was planned. + TooManyCandidates { limit: usize }, /// The frontends' roots or a composed candidate failed validation. Validation(String), } @@ -75,12 +75,9 @@ impl std::fmt::Display for PlanningStagesError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Pass1(error) => write!(f, "Pass 1: {error}"), - Self::TooManyCombinations { - combinations, - limit, - } => write!( + Self::TooManyCandidates { limit } => write!( f, - "workload has {combinations} candidate combinations (limit {limit}); \ + "workload has more than {limit} Stage 1 candidates, Pass 2's included; \ planning it needs early pruning (#673)" ), Self::Validation(error) => write!(f, "validation: {error}"), @@ -98,15 +95,16 @@ impl From for String { /// identical-expression rule), /// physical ASAP-aware optimization and plan selection over `roots`, the /// lowered entries of `workload`: the library's planner pipeline -/// ([`run_stage1_to_stage3_from_pass1`]) with the built-in models. Logical -/// candidates are every choice of a per-computation candidate for every -/// target, the last target varying fastest, each followed by its Pass 2 -/// candidates. Every candidate is enumerated -/// and priced, so the selection is the cheapest of all; a caller that shows -/// fewer cuts only its display. With `max_combinations`, a workload with more -/// whole-workload candidates is refused after Pass 1, before any is -/// enumerated. Plan selection reads only the accuracy targets and the data -/// workload. The frontends' roots and every composed candidate are +/// ([`stage1_logical_asap_optimization`], [`run_stage2_and_stage3`]) with +/// the built-in models. Logical candidates are every choice of a +/// per-computation candidate for every target, the last target varying +/// fastest, each followed by its Pass 2 candidates. Every candidate is +/// enumerated and priced, so the selection is the cheapest of all; a caller +/// that shows fewer cuts only its display. With `max_combinations`, Stage 1 +/// candidates (Pass 2's included) are counted as they are enumerated, and a +/// workload with more is refused as soon as the count passes the limit; the +/// library itself never caps. Plan selection reads the accuracy targets, +/// the recurrences and the data workload. The frontends' roots and every composed candidate are /// validated. pub fn run_planning_stages( workload: &PlanningWorkload, @@ -142,15 +140,20 @@ pub fn run_planning_stages_over( let per_computation = generate_per_computation_candidates(roots.iter().cloned().enumerate().collect()) .map_err(|e| PlanningStagesError::Pass1(e.to_string()))?; - let combinations = per_computation.combinations(); - if let Some(limit) = max_combinations.filter(|limit| combinations > *limit) { - return Err(PlanningStagesError::TooManyCombinations { - combinations, - limit, - }); + // Count Stage 1 candidates as they are enumerated and stop as soon as + // the count passes the limit; never truncate. + let mut stage1 = Vec::new(); + for candidate in stage1_logical_asap_optimization(&per_computation) { + if max_combinations.is_some_and(|limit| stage1.len() == limit) { + return Err(PlanningStagesError::TooManyCandidates { + limit: stage1.len(), + }); + } + stage1.push(candidate); } - let run = run_stage1_to_stage3_from_pass1( + let run = run_stage2_and_stage3( per_computation, + stage1, &targets, recurrence, data, diff --git a/crates/devtools/tests/stage_pipeline.rs b/crates/devtools/tests/stage_pipeline.rs index 4e4fa24c6..0cb261ec1 100644 --- a/crates/devtools/tests/stage_pipeline.rs +++ b/crates/devtools/tests/stage_pipeline.rs @@ -3,8 +3,10 @@ use std::collections::HashSet; use std::path::PathBuf; use std::process::Command; -use asap_devtools::planner_layering_examples::{pattern_a, PatternARecurrence}; -use asap_devtools::planning_stages::{lower_promql_roots, run_planning_stages}; +use asap_devtools::planner_layering_examples::{example1, pattern_a, PatternARecurrence}; +use asap_devtools::planning_stages::{ + lower_promql_roots, run_planning_stages, PlanningStagesError, +}; use asap_types::ir::flat::FlatDag; use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPDAGDocument}; use asap_types::ir::QueryRoot; @@ -296,22 +298,30 @@ fn failure(args: &[&str]) -> (Option, String) { ) } -/// Above `--max-combinations`, the run is rejected before enumeration with -/// the count and the limit, rather than truncated. +/// `--max-combinations` limits the Stage 1 candidates, Pass 2's included: +/// Example 1 has 9 (3 Pass 1 candidates, each with 2 Pass 2 candidates). +/// Above the limit the run stops with an error as soon as the count passes +/// it, rather than truncating. #[test] -fn workload_above_the_combination_limit_is_rejected() { - let (status, stderr) = failure(&["--example", "planner-layering-1", "--max-combinations", "2"]); - assert_eq!(status, Some(3)); - assert!( - stderr.contains( - "workload has 3 candidate combinations (limit 2); planning it needs early pruning (#673)" - ), - "{stderr}" - ); - // At the limit, it plans. The limit counts Pass 1's combinations; the - // document lists every Stage 1 candidate, each Pass 1 candidate with its - // 2 Pass 2 candidates. - let document = generate(&["--max-combinations", "3"]); +fn workload_above_the_candidate_limit_is_rejected() { + for limit in ["3", "8"] { + let (status, stderr) = failure(&[ + "--example", + "planner-layering-1", + "--max-combinations", + limit, + ]); + assert_eq!(status, Some(3), "{limit}"); + assert!( + stderr.contains(&format!( + "workload has more than {limit} Stage 1 candidates, Pass 2's included; \ + planning it needs early pruning (#673)" + )), + "{stderr}" + ); + } + // At the limit, it plans every candidate. + let document = generate(&["--max-combinations", "9"]); assert_eq!( document["stage1_logical_asap_optimization"]["combinations"], 9 @@ -319,9 +329,9 @@ fn workload_above_the_combination_limit_is_rejected() { } /// The default limit is 10^4: nine approximate Top-K queries (3^9 = 19683 -/// combinations) are rejected without being enumerated. +/// Pass 1 combinations) are rejected once the count passes it. #[test] -fn default_combination_limit_is_ten_thousand() { +fn default_candidate_limit_is_ten_thousand() { let queries: Vec = (1..=9) .map(|i| format!("topk(5, rate(m{i}[5m]))")) .collect(); @@ -333,11 +343,24 @@ fn default_combination_limit_is_ten_thousand() { let (status, stderr) = failure(&args); assert_eq!(status, Some(3)); assert!( - stderr.contains("workload has 19683 candidate combinations (limit 10000)"), + stderr.contains("workload has more than 10000 Stage 1 candidates"), "{stderr}" ); } +/// The library never caps: without a limit every candidate is planned. +#[test] +fn planning_stages_without_a_limit_plan_every_candidate() { + let workload = example1(); + let run = run_planning_stages(&workload, lower_promql_roots(&workload).unwrap(), None).unwrap(); + assert_eq!(run.stage1_logical_asap_optimization.len(), 9); + let limited = run_planning_stages(&workload, lower_promql_roots(&workload).unwrap(), Some(8)); + assert!(matches!( + limited, + Err(PlanningStagesError::TooManyCandidates { limit: 8 }) + )); +} + #[test] fn table_declared_twice_is_rejected() { let table = From e664e59e00d2056228ab0266cc9b222b61b7aa46 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:24:02 +0000 Subject: [PATCH 09/15] perf(logical-asap-optimization): enumerate distinct Pass 2 candidates directly The identical-expression rule tried all 2^g subsets of the sharing groups and dropped those that repeat a candidate. With nested groups most subsets repeat (14 groups: 16 384 subsets for 130 candidates). Sharing a set of groups also shares the groups it leaves as one node; that closure maps each subset to its candidate, and the distinct candidates are its closed sets. They are now enumerated directly (Next Closure: one closure per step and group), then put in the same binary-counter order, so the output is unchanged. Twenty nested groups give their 20 candidates quickly; a brute-force test checks the same candidates on small cases. Sharing groups are computed once per candidate, and a Pass 2 candidate names its shared groups by kind and number (Scan #1, TimeRange #2), so repeated kinds stay distinguishable. The viewer fixtures are regenerated. --- .../asap-aware-mapping/src/asap_aware_cse.rs | 195 ++++++++++++++---- .../src/pass/stage1_to_stage3.rs | 11 +- .../examples/planner-layering-example1.json | 24 +-- 3 files changed, 172 insertions(+), 58 deletions(-) diff --git a/crates/asap-aware-mapping/src/asap_aware_cse.rs b/crates/asap-aware-mapping/src/asap_aware_cse.rs index 4945e93e5..5bbbb0a13 100644 --- a/crates/asap-aware-mapping/src/asap_aware_cse.rs +++ b/crates/asap-aware-mapping/src/asap_aware_cse.rs @@ -92,53 +92,91 @@ fn pair( } /// Every Pass 2 candidate of the identical-expression rule for the -/// candidate with query roots `roots`: one per non-empty subset of its -/// [`sharing_groups`] that gives a distinct candidate, in binary-counter -/// order of the subsets (the first group varying fastest). Empty when -/// nothing is identical. -pub fn identical_expression_variants(roots: &[QueryRoot]) -> Vec { - let groups = sharing_groups(roots); +/// candidate with query roots `roots` and sharing groups `groups` +/// ([`sharing_groups`]): one per non-empty subset of the groups that gives a +/// distinct candidate, in binary-counter order of the subsets (the first +/// group the lowest bit). Empty when nothing is identical. +/// +/// Sharing a set of groups also shares every group whose remaining nodes it +/// leaves as one; that closure maps each subset to the candidate it gives, +/// and the distinct candidates are the closed sets. They are enumerated +/// directly (Ganter's Next Closure, one closure per step and group) rather +/// than by trying all `2^g` subsets, which repeat when groups nest. +pub fn identical_expression_variants( + roots: &[QueryRoot], + groups: &[SharingGroup], +) -> Vec { + let n = groups.len(); + let mut variants = Vec::new(); + // The closed sets in lectic order, from the closure of the empty set + // (which shares nothing). + let mut closed = vec![false; n]; + 'next: loop { + for i in (0..n).rev() { + if closed[i] { + continue; + } + let mut candidate: Vec = (0..n).map(|g| g < i && closed[g]).collect(); + candidate[i] = true; + let (shared_roots, merged) = apply(roots, groups, &candidate); + // Lectic successor: adds nothing below `i` that `closed` lacks. + if (0..i).all(|g| !merged[g] || closed[g]) { + // `merged` is closed: applying it gives the same candidate. + variants.push((merged.clone(), shared_roots)); + closed = merged; + continue 'next; + } + } + break; + } + // Binary-counter order of the subsets: compare from the highest group. + variants.sort_by(|(a, _), (b, _)| a.iter().rev().cmp(b.iter().rev())); + variants + .into_iter() + .map(|(subset, roots)| IdenticalExpressionVariant { + shared_groups: (0..n).filter(|&g| subset[g]).collect(), + roots, + }) + .collect() +} + +/// `roots` with the groups in `subset` shared, and the groups that leaves +/// shared: those whose every node still read became one node (a node below +/// a shared node other than the one kept is no longer read). +fn apply( + roots: &[QueryRoot], + groups: &[SharingGroup], + subset: &[bool], +) -> (Vec, Vec) { let group_of: HashMap<*const OperatorNode, usize> = groups .iter() .enumerate() .flat_map(|(index, group)| group.nodes.iter().map(move |n| (Rc::as_ptr(n), index))) .collect(); - let mut variants = Vec::new(); - let mut subset = vec![false; groups.len()]; - while next_subset(&mut subset) { - let mut sharing = Sharing { - subset: &subset, - group_of: &group_of, - rebuilt: HashMap::new(), - shared: HashMap::new(), - }; - let shared_roots: Vec = roots.iter().map(|r| sharing.root(r)).collect(); - // The groups this candidate actually shares: every node of the group - // that is still read became one node. A node below a shared node - // other than the one kept is no longer read. - let merged: Vec = groups - .iter() - .map(|group| { - let mut nodes = group.nodes.iter().filter_map(|node| { - let (_, rebuilt) = sharing.rebuilt.get(&Rc::as_ptr(node))?; - Some(Rc::as_ptr(rebuilt)) - }); - let first = nodes.next(); - nodes.all(|node| Some(node) == first) - }) - .collect(); - if merged == subset { - variants.push(IdenticalExpressionVariant { - shared_groups: (0..groups.len()).filter(|&g| subset[g]).collect(), - roots: shared_roots, + let mut sharing = Sharing { + subset, + group_of: &group_of, + rebuilt: HashMap::new(), + shared: HashMap::new(), + }; + let shared_roots: Vec = roots.iter().map(|r| sharing.root(r)).collect(); + let merged = groups + .iter() + .map(|group| { + let mut nodes = group.nodes.iter().filter_map(|node| { + let (_, rebuilt) = sharing.rebuilt.get(&Rc::as_ptr(node))?; + Some(Rc::as_ptr(rebuilt)) }); - } - } - variants + let first = nodes.next(); + nodes.all(|node| Some(node) == first) + }) + .collect(); + (shared_roots, merged) } /// Binary increment of `subset`, the first entry the lowest bit; `false` /// once every subset has been visited. +#[cfg(test)] fn next_subset(subset: &mut [bool]) -> bool { for bit in subset.iter_mut() { *bit = !*bit; @@ -284,7 +322,7 @@ mod tests { groups.iter().map(kinds).collect::>(), ["Scan", "TimeRange"] ); - let variants = identical_expression_variants(&roots); + let variants = identical_expression_variants(&roots, &sharing_groups(&roots)); assert_eq!( variants .iter() @@ -317,7 +355,7 @@ mod tests { sharing_groups(&roots).iter().map(kinds).collect::>(), ["Scan", "TimeRange", "Scan", "TimeRange"] ); - let variants = identical_expression_variants(&roots); + let variants = identical_expression_variants(&roots, &sharing_groups(&roots)); assert_eq!(variants.len(), 8); let shares = |v: &IdenticalExpressionVariant| { ( @@ -335,9 +373,84 @@ mod tests { /// Queries with nothing in common have no Pass 2 candidate. #[test] fn nothing_identical_gives_no_variant() { + assert!({ + let roots = roots(&["sum(rate(a[1m]))", "sum(rate(b[1m]))"]); + identical_expression_variants(&roots, &sharing_groups(&roots)) + } + .is_empty()); + } + + /// Every non-empty subset of the groups, applied, kept when it gives a + /// distinct candidate: the definition the direct enumeration follows. + fn brute_force(roots: &[QueryRoot]) -> Vec> { + let groups = sharing_groups(roots); + let mut subset = vec![false; groups.len()]; + let mut out = Vec::new(); + while next_subset(&mut subset) { + let (_, merged) = apply(roots, &groups, &subset); + if merged == subset { + out.push((0..groups.len()).filter(|&g| subset[g]).collect()); + } + } + out + } + + /// The direct enumeration gives exactly the brute force's candidates, in + /// the same order, on nested and independent groups. + #[test] + fn direct_enumeration_matches_the_brute_force() { + for queries in [ + &[ + "sum by (job) (rate(m[1m]))", + "topk by (job) (10, sum_over_time(m[1m]))", + ][..], + &["sum(a)", "max(a)", "sum(b)", "max(b)"], + &["sum(abs(a))", "max(abs(a))", "min(a)", "sum(b)", "max(b)"], + &["sum(abs(ceil(a)))", "max(abs(ceil(a)))", "min(ceil(a))"], + ] { + let roots = roots(queries); + let groups = sharing_groups(&roots); + let direct: Vec<_> = identical_expression_variants(&roots, &groups) + .into_iter() + .map(|v| v.shared_groups) + .collect(); + assert_eq!(direct, brute_force(&roots), "{queries:?}"); + } + } + + /// Twenty nested groups (two identical chains of a keyed scan and 19 + /// filters) have 20 distinct candidates, one per depth shared, not 2^20 + /// subsets to try. + #[test] + fn twenty_nested_groups_enumerate_quickly() { + use crate::test_support::{filter, metric_scan_with_keys}; + use asap_types::ir::ScalarExpr; + use asap_types::pre_asap::expr_ir::CompareOpKind; + let chain = || { + let mut node = metric_scan_with_keys(&["job"], vec![vec![0, 2]]); + for i in 0..19 { + node = filter( + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(1)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::literal_f64(i as f64)), + semantics: asap_types::ir::ExprSemantics::Promql, + }, + node, + ); + } + QueryRoot::Operator(node) + }; + let roots = [chain(), chain()]; + let groups = sharing_groups(&roots); + assert_eq!(groups.len(), 20); + let start = std::time::Instant::now(); + let variants = identical_expression_variants(&roots, &groups); + assert_eq!(variants.len(), 20); assert!( - identical_expression_variants(&roots(&["sum(rate(a[1m]))", "sum(rate(b[1m]))"])) - .is_empty() + start.elapsed() < std::time::Duration::from_secs(5), + "{:?}", + start.elapsed() ); } } diff --git a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs index 60abb16a7..2e60ce1bb 100644 --- a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs +++ b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs @@ -193,10 +193,11 @@ pub fn stage1_logical_asap_optimization( let roots = compose_logical_candidate(per_computation, &this) .map(|roots| roots.into_iter().map(|(_, root)| root).collect::>()); let (groups, variants) = match &roots { - Ok(roots) => ( - crate::asap_aware_cse::sharing_groups(roots), - identical_expression_variants(roots), - ), + Ok(roots) => { + let groups = crate::asap_aware_cse::sharing_groups(roots); + let variants = identical_expression_variants(roots, &groups); + (groups, variants) + } Err(_) => (Vec::new(), Vec::new()), }; let from = position + queue.len(); @@ -210,7 +211,7 @@ pub fn stage1_logical_asap_optimization( let shared: Vec = variant .shared_groups .iter() - .map(|&g| groups[g].nodes[0].operator.kind_name().to_string()) + .map(|&g| format!("{} #{}", groups[g].nodes[0].operator.kind_name(), g + 1)) .collect(); queue.push_back(Stage1Candidate { display_name: format!("{display_name} · shared {}", shared.join(", ")), diff --git a/tools/dag-viewer/examples/planner-layering-example1.json b/tools/dag-viewer/examples/planner-layering-example1.json index bc7878e9c..2c1cef72d 100644 --- a/tools/dag-viewer/examples/planner-layering-example1.json +++ b/tools/dag-viewer/examples/planner-layering-example1.json @@ -2280,7 +2280,7 @@ } ] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 exact (Sum acc) · shared Scan", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 exact (Sum acc) · shared Scan #1", "id": "L2" }, { @@ -2983,7 +2983,7 @@ } ] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 exact (Sum acc) · shared Scan, TimeRange", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 exact (Sum acc) · shared Scan #1, TimeRange #2", "id": "L3" }, { @@ -4595,7 +4595,7 @@ } ] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap · shared Scan", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap · shared Scan #1", "id": "L5" }, { @@ -5265,7 +5265,7 @@ } ] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap · shared Scan, TimeRange", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap · shared Scan #1, TimeRange #2", "id": "L6" }, { @@ -6877,7 +6877,7 @@ } ] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap · shared Scan", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap · shared Scan #1", "id": "L8" }, { @@ -7547,7 +7547,7 @@ } ] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap · shared Scan, TimeRange", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap · shared Scan #1, TimeRange #2", "id": "L9" } ], @@ -10799,7 +10799,7 @@ ], "scalar_roots": [] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 exact (Sum acc) · shared Scan", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 exact (Sum acc) · shared Scan #1", "from_logical": "L2", "id": "P2" }, @@ -12227,7 +12227,7 @@ ], "scalar_roots": [] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 exact (Sum acc) · shared Scan, TimeRange", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 exact (Sum acc) · shared Scan #1, TimeRange #2", "from_logical": "L3", "id": "P3" }, @@ -14828,7 +14828,7 @@ ], "scalar_roots": [] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap · shared Scan", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap · shared Scan #1", "from_logical": "L5", "id": "P5" }, @@ -15933,7 +15933,7 @@ ], "scalar_roots": [] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap · shared Scan, TimeRange", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CMS+heap · shared Scan #1, TimeRange #2", "from_logical": "L6", "id": "P6" }, @@ -18534,7 +18534,7 @@ ], "scalar_roots": [] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap · shared Scan", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap · shared Scan #1", "from_logical": "L8", "id": "P8" }, @@ -19639,7 +19639,7 @@ ], "scalar_roots": [] }, - "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap · shared Scan, TimeRange", + "display_name": "Q1 exact (Sum acc, Rate acc) · Q2 CountSketch+heap · shared Scan #1, TimeRange #2", "from_logical": "L9", "id": "P9" } From e9cac86b15b14eed9e81b7ed91093e7e2e3b7496 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:32:57 +0000 Subject: [PATCH 10/15] fix(plan-selection): every consumer gets the selected plan with confirmed weights The counter-sample requirement Stage 3 confirms was made a proof only in the planner pass, so run_stage1_to_stage3's selected_logical and the selected candidate's roots still carried the open requirement, which the executor rejects. Stage 3 now puts the plan on its output: Selection::plan is the selected candidate's roots with confirmed weights, and Stage1ToStage3Run::selected_logical returns the selected logical roots the same way. The pass reads Selection::plan. --- .../src/pass/stage1_to_stage3.rs | 44 +++++++------ .../src/stage3_plan_selection.rs | 64 +++++++++++++++++++ 2 files changed, 88 insertions(+), 20 deletions(-) diff --git a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs index 2e60ce1bb..4292b703e 100644 --- a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs +++ b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs @@ -105,6 +105,9 @@ pub struct Stage1ToStage3Run { pub stage2_pruned: Vec, /// Stage 3: plan selection over the Stage 2 candidates. pub stage3_plan_selection: Result, + /// Stage 3: the selected candidate's logical roots, with the weight + /// requirements it confirmed made proofs ([`Self::selected_logical`]). + pub stage3_selected_logical: Option>, } impl Stage1ToStage3Run { @@ -118,14 +121,10 @@ impl Stage1ToStage3Run { /// The composed roots of the logical candidate the selected physical /// candidate implements: the plan before physical operator - /// implementation and materialization. + /// implementation and materialization, with the weight requirements + /// Stage 3 confirmed made proofs, as in [`Selection::plan`]. pub fn selected_logical(&self) -> Option<&[QueryRoot]> { - let from = &self.selected()?.from_logical; - let logical = self - .stage1_logical_asap_optimization - .iter() - .find(|l| &l.id == from)?; - logical.roots.as_deref().ok() + self.stage3_selected_logical.as_deref() } } @@ -291,7 +290,23 @@ pub fn run_stage2_and_stage3( }); } let selection = select_plan(&physical, targets, recurrence, data, models); + let selected_logical = selection.as_ref().ok().and_then(|selection| { + let from = &physical + .iter() + .find(|p| p.id == selection.selected)? + .from_logical; + let roots = logical + .iter() + .find(|l| &l.id == from)? + .roots + .as_ref() + .ok()?; + Some(crate::stage3_plan_selection::with_confirmed_weights( + roots, data, + )) + }); Stage1ToStage3Run { + stage3_selected_logical: selected_logical, stage1_pass1_per_computation: per_computation, combinations, stage1_pass2_identical_expression: pass2, @@ -461,19 +476,8 @@ impl OptimizationPass for Stage1ToStage3Pass { .collect(); let data = workload.data_workload().cloned().unwrap_or_default(); let run = run_stage1_to_stage3(roots, &targets, &recurrence, &data, input.models)?; - let Stage1ToStage3Run { - stage2_physical_asap_optimization: physical, - stage3_plan_selection: selection, - .. - } = run; - let selection = selection?; - let selected = physical - .into_iter() - .find(|p| p.id == selection.selected) - .expect("the selected candidate is a physical candidate"); - // The executor reads no data workload: answer every open weight - // requirement plan selection confirmed. - let roots = crate::stage3_plan_selection::with_confirmed_weights(&selected.roots, &data); + let selection = run.stage3_plan_selection?; + let roots = selection.plan.clone(); let mut output = PlanOutput::new(Vec::new()); for (entry_index, root) in roots.iter().enumerate() { match root { diff --git a/crates/asap-aware-mapping/src/stage3_plan_selection.rs b/crates/asap-aware-mapping/src/stage3_plan_selection.rs index 4581cf49e..054ed70b4 100644 --- a/crates/asap-aware-mapping/src/stage3_plan_selection.rs +++ b/crates/asap-aware-mapping/src/stage3_plan_selection.rs @@ -166,6 +166,10 @@ pub struct Rejection { #[derive(Debug, Clone, PartialEq)] pub struct Selection { pub selected: String, + /// The selected candidate's query roots with every weight requirement + /// Stage 3 confirmed made a proof ([`with_confirmed_weights`]): the plan + /// to run. The Stage 2 candidate keeps the open requirement. + pub plan: Vec, pub costs: BTreeMap, pub rejected: Vec, } @@ -258,8 +262,14 @@ pub fn select_plan( }); } } + let plan = cands + .iter() + .find(|c| c.id == selected) + .map(|c| with_confirmed_weights(&c.roots, data)) + .unwrap_or_default(); Ok(Selection { selected: selected.to_string(), + plan, costs, rejected, }) @@ -1521,6 +1531,60 @@ mod tests { } } + /// Every consumer of a run sees the selected plan with its weight + /// requirements answered: the logical plan (`selected_logical`) and the + /// executable roots (`Selection::plan`) carry the counter proof, not the + /// open requirement the executor rejects. + #[test] + fn the_selected_plan_carries_confirmed_weights() { + use asap_types::workload::MetricType; + let target = AccuracyTarget::EpsilonDelta { + epsilon: 0.01, + delta: 0.001, + }; + let root = lower_promql("topk by (job) (10, sum_over_time(m[1m]))", target.clone()); + let root = asap_types::ir::schema_support::with_promql_series_identity(&root).unwrap(); + let data = DataWorkload { + metric_types: [("m".to_string(), MetricType::Counter)].into(), + input_cardinality: Evidence { + value: Some(10_000_000), + ..Default::default() + }, + ..data() + }; + let run = crate::pass::run_stage1_to_stage3( + vec![(0, QueryRoot::Operator(root))], + &[Some(target)], + &every_10s(1), + &data, + PlanningModels::builtin().with_evidence(&EveryOutputMeasuredExact), + ) + .unwrap(); + let domains = |roots: &[QueryRoot]| -> Vec { + roots + .iter() + .flat_map(root_operators) + .flat_map(OperatorNode::reachable) + .filter_map(|n| match n.asap() { + Some(ASAPOp::SummaryAgg { + input, + family: FieldDataType::Sketch(kind, _), + .. + }) if *kind.algorithm() == SketchAlgorithm::CmsWithHeap => { + Some(input.weight_domain.clone()) + } + _ => None, + }) + .collect() + }; + let proven = [WeightDomain::NonNegative { + proof: asap_types::post_asap::NonNegativeWeightProof::CounterSamples, + }]; + let selection = run.stage3_plan_selection.as_ref().unwrap(); + assert_eq!(domains(&selection.plan), proven, "{selection:?}"); + assert_eq!(domains(run.selected_logical().unwrap()), proven); + } + /// The selected plan's open requirement becomes a proof when confirmed /// and stays unproven otherwise; the executor reads no data workload. #[test] From 3c4ee9427ce356ba693738979da0f70d4d0aafc3 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:34:22 +0000 Subject: [PATCH 11/15] fix(logical-asap-optimization): an open weight requirement does not certify Count-Min estimate_has_bound treated any weight domain but UnknownOrSigned as proven, so a Count-Min estimate over weights with the open SamplesOfMetric requirement got its bound as if the weights were proven non-negative. Only NonNegative now counts. Such an estimate instead gets its local bound gated on the metric's sign, values_non_negative[M], which plan selection binds to 1 when M is declared a counter and leaves unbound otherwise. --- .../src/per_computation_candidates.rs | 130 +++++++++++++++++- 1 file changed, 126 insertions(+), 4 deletions(-) diff --git a/crates/asap-aware-mapping/src/per_computation_candidates.rs b/crates/asap-aware-mapping/src/per_computation_candidates.rs index 681addda6..effd46804 100644 --- a/crates/asap-aware-mapping/src/per_computation_candidates.rs +++ b/crates/asap-aware-mapping/src/per_computation_candidates.rs @@ -916,9 +916,14 @@ fn composed_guarantee(node: &OperatorNode) -> Option { child: input, reduction, .. - }) if estimate_has_bound(family, update, query) => { - let local = classic_hll_guarantee(family, query, input) - .or_else(|| DefaultAccuracyModel.local_guarantee(family, query))?; + }) if estimate_has_bound(family, update, query) + || estimate_bound_over_samples(family, update, query).is_some() => + { + let local = match estimate_bound_over_samples(family, update, query) { + Some(gated) => gated, + None => classic_hll_guarantee(family, query, input) + .or_else(|| DefaultAccuracyModel.local_guarantee(family, query))?, + }; if exact_inputs(summary_input) { Some(local) } else if exact_filter(summary_input) && !groups_by_estimate(reduction, input) { @@ -1682,7 +1687,43 @@ fn estimate_has_bound( if matches!(kind.algorithm(), SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap) ); !matches!(query, SketchStatistic::TopK { .. }) - && !(count_min && update.weight_domain == WeightDomain::UnknownOrSigned) + && !(count_min && !matches!(update.weight_domain, WeightDomain::NonNegative { .. })) +} + +/// A Count-Min estimate whose weights carry the open requirement that they +/// are samples of one metric ([`WeightDomain::SamplesOfMetric`]): its local +/// guarantee with the bound gated on that metric's sign +/// ([`samples_sign_statistic`]), which plan selection binds to 1 when the +/// metric is declared a counter. `None` for any other estimate. +fn estimate_bound_over_samples( + family: &FieldDataType, + update: &SummaryUpdate, + query: &SketchStatistic, +) -> Option { + let WeightDomain::SamplesOfMetric { metric } = &update.weight_domain else { + return None; + }; + let proven = SummaryUpdate { + weight_domain: WeightDomain::NonNegative { + proof: NonNegativeWeightProof::CounterSamples, + }, + ..update.clone() + }; + if !estimate_has_bound(family, &proven, query) { + return None; + } + let local = DefaultAccuracyModel.local_guarantee(family, query)?; + Some(AccuracyGuarantee { + bound: asap_types::post_asap::BoundExpr::Product { + factors: vec![ + local.bound, + asap_types::post_asap::BoundExpr::Unknown { + statistic: samples_sign_statistic(metric), + }, + ], + }, + ..local + }) } /// Replace an aggregate whose measures are `targets`' (or exact, for a @@ -3076,4 +3117,85 @@ mod tests { }; root.clone() } + + /// A Count-Min estimate has a bound only over weights proven + /// non-negative: an open requirement (`SamplesOfMetric`) is not a proof. + #[test] + fn count_min_estimate_has_no_bound_over_an_open_requirement() { + let family = FieldDataType::Sketch( + SketchKind::new( + SketchAlgorithm::Cms, + asap_types::post_asap::SketchParams::Cms { width: 8, depth: 2 }, + ), + GroupingStrategy::default(), + ); + let query = SketchStatistic::PointCount { + key: ColumnRef::SampleValue, + value: None, + }; + let with = |weight_domain| SummaryUpdate { + item: Some(SummaryInputExpr::Column(ColumnRef::SampleValue)), + weight: SummaryInputExpr::Column(ColumnRef::SampleValue), + weight_domain, + }; + assert!(!estimate_has_bound( + &family, + &with(WeightDomain::SamplesOfMetric { metric: "m".into() }), + &query + )); + assert!(!estimate_has_bound( + &family, + &with(WeightDomain::UnknownOrSigned), + &query + )); + assert!(estimate_has_bound( + &family, + &with(WeightDomain::NonNegative { + proof: NonNegativeWeightProof::CounterSamples + }), + &query + )); + } + + /// A Count-Min estimate over samples of `m` gets its bound gated on + /// `m`'s sign: unknown until plan selection binds it, the local bound + /// once bound to 1. + #[test] + fn count_min_estimate_over_samples_is_gated_on_the_metric() { + let family = FieldDataType::Sketch( + SketchKind::new( + SketchAlgorithm::Cms, + asap_types::post_asap::SketchParams::Cms { + width: 272, + depth: 5, + }, + ), + GroupingStrategy::default(), + ); + let query = SketchStatistic::PointCount { + key: ColumnRef::SampleValue, + value: None, + }; + let update = SummaryUpdate { + item: Some(SummaryInputExpr::Column(ColumnRef::SampleValue)), + weight: SummaryInputExpr::Column(ColumnRef::SampleValue), + weight_domain: WeightDomain::SamplesOfMetric { metric: "m".into() }, + }; + let gated = estimate_bound_over_samples(&family, &update, &query).expect("gated"); + assert_eq!(gated.bound.evaluate(), None); + let local = DefaultAccuracyModel + .local_guarantee(&family, &query) + .unwrap(); + let statistic = samples_sign_statistic("m"); + // Every other statistic (a stream norm) bound alike. + let others = |s: &str| Some(if s == statistic { 1.0 } else { 1_000.0 }); + let bound = gated.bind(&others).bound.evaluate(); + assert!(bound.is_some()); + assert_eq!(bound, local.bind(&others).bound.evaluate()); + let unconfirmed = gated + .bind(&|s| (s != statistic).then_some(1_000.0)) + .bound + .evaluate(); + assert_eq!(unconfirmed, None); + } } From 8176270430e8c8725e302b545e36578f9fbd218e Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:35:08 +0000 Subject: [PATCH 12/15] test: Example 1's shared-input totals tie with the independent one up to rounding Shows that the tie tolerance is needed: the shared-input candidates' totals differ from the independent candidate's in the last bits, and plan selection reports them as tied and keeps the independent one. Also drops stale wording in the acceptance helpers. --- .../tests/planner_layering_common/mod.rs | 4 +-- .../tests/planner_layering_example1.rs | 34 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/crates/integration-tests/tests/planner_layering_common/mod.rs b/crates/integration-tests/tests/planner_layering_common/mod.rs index 94f82428a..7f5712d55 100644 --- a/crates/integration-tests/tests/planner_layering_common/mod.rs +++ b/crates/integration-tests/tests/planner_layering_common/mod.rs @@ -86,7 +86,7 @@ pub struct Logical { impl Logical { /// Whether two queries read a common node. Only Pass 2's - /// identical-expression rule (#588) shares a node across queries. + /// identical-expression rule shares a node across queries. pub fn shared_input(&self) -> bool { !cross_query_nodes(&self.dag, &self.query_roots).is_empty() } @@ -477,7 +477,7 @@ pub fn assert_each_node_charged_per_consuming_query(run: &Run) { } } -/// How many times each node is computed per workload evaluation (#509; the +/// How many times each node is computed per evaluation of its consumers (#509; the /// rule of plan selection's private `computations`): 1 for a materialized /// node; for a `NotMaterialized` one, the number of consumers that reach /// it. A consumer is a query (operator or scalar root) or a materialized diff --git a/crates/integration-tests/tests/planner_layering_example1.rs b/crates/integration-tests/tests/planner_layering_example1.rs index bd2265cb3..abb9c331d 100644 --- a/crates/integration-tests/tests/planner_layering_example1.rs +++ b/crates/integration-tests/tests/planner_layering_example1.rs @@ -530,6 +530,40 @@ fn stage3_plan_selection_count_min_needs_the_counter_declaration() { ); } +/// A shared NotMaterialized input is charged once per consumer, so the +/// shared-input candidates cost what the independent one does; their totals +/// differ from it only by floating-point rounding (the same charges summed +/// in another grouping), which plan selection treats as a tie, so the +/// independent candidate, enumerated first, is selected. +#[test] +fn stage3_plan_selection_shared_input_ties_up_to_rounding() { + let run = pipeline(); + let independent = physical_with(&run, Q2Option::Exact); + let base = run.cost(&independent.id).unwrap(); + let shared: Vec<_> = run + .physical + .iter() + .filter(|p| classify(p.dag(), &p.query_roots) == (Q2Option::Exact, true)) + .map(|p| (p.id.clone(), run.cost(&p.id).unwrap())) + .collect(); + assert!(!shared.is_empty()); + assert!( + shared + .iter() + .any(|(_, total)| total.to_bits() != base.to_bits()), + "the rounding difference the tie tolerance absorbs: {base} vs {shared:?}" + ); + for (id, total) in &shared { + assert!( + (total - base).abs() <= 1e-9 * base, + "{id}: {total} vs {base}" + ); + let rejection = run.selection.rejected.iter().find(|r| &r.id == id).unwrap(); + assert!(rejection.reason.contains("tied"), "{}", rejection.reason); + } + assert_eq!(run.selection.selected, independent.id); +} + /// The selected plan is the cheapest valid candidate for the whole workload. #[test] fn stage3_plan_selection_selects_cheapest_valid() { From ba75c7bf60408d7945085dbc417ad9ce767e1393 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 14 Oct 2026 22:19:55 +0000 Subject: [PATCH 13/15] test(logical-asap-optimization): Pass 2 candidate labels and roots agree Applying a candidate's label (the groups it shares) gives the same roots the direct enumeration returned, sharing exactly those groups. --- .../asap-aware-mapping/src/asap_aware_cse.rs | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/crates/asap-aware-mapping/src/asap_aware_cse.rs b/crates/asap-aware-mapping/src/asap_aware_cse.rs index 5bbbb0a13..5278ccb1d 100644 --- a/crates/asap-aware-mapping/src/asap_aware_cse.rs +++ b/crates/asap-aware-mapping/src/asap_aware_cse.rs @@ -453,4 +453,44 @@ mod tests { start.elapsed() ); } + + /// Each candidate's label (the groups it shares) and its roots agree: + /// applying the label gives the same roots, sharing exactly the + /// labelled groups. + #[test] + fn labels_and_roots_agree() { + for queries in [ + &[ + "sum by (job) (rate(m[1m]))", + "topk by (job) (10, sum_over_time(m[1m]))", + ][..], + &["sum(a)", "max(a)", "sum(b)", "max(b)"], + &["sum(abs(a))", "max(abs(a))", "min(a)", "sum(b)", "max(b)"], + ] { + let roots = roots(queries); + let groups = sharing_groups(&roots); + for variant in identical_expression_variants(&roots, &groups) { + let label: Vec = (0..groups.len()) + .map(|g| variant.shared_groups.contains(&g)) + .collect(); + let (applied, merged) = apply(&roots, &groups, &label); + assert_eq!(merged, label, "{queries:?}"); + assert_eq!(applied, variant.roots, "{queries:?}"); + assert_eq!( + distinct_nodes(&applied), + distinct_nodes(&variant.roots), + "{queries:?} {:?}", + variant.shared_groups + ); + for (a, b) in [(0, 1), (2, 3)] { + if b < roots.len() { + assert_eq!( + common_nodes(&applied[a], &applied[b]).len(), + common_nodes(&variant.roots[a], &variant.roots[b]).len() + ); + } + } + } + } + } } From b9130aa9a5313038fc307d09d09e9a923064c90f Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 14 Oct 2026 22:19:55 +0000 Subject: [PATCH 14/15] fix(plan-selection): the sign gate opens only at 1 The sign of one metric's samples gates a relative bound as a factor; a binder supplying 0 would have read as a perfect bound. Plan selection now binds a sign statistic only to 1 (proven) and leaves any other value unbound. --- .../src/stage3_plan_selection.rs | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/crates/asap-aware-mapping/src/stage3_plan_selection.rs b/crates/asap-aware-mapping/src/stage3_plan_selection.rs index 054ed70b4..2210f3755 100644 --- a/crates/asap-aware-mapping/src/stage3_plan_selection.rs +++ b/crates/asap-aware-mapping/src/stage3_plan_selection.rs @@ -454,11 +454,28 @@ fn bound_root_guarantee( ) -> Option { let guarantee = root_guarantee(root)?; let domains = operand_domain_bounds(root, data, models); - Some(guarantee.bind(&|statistic| { + Some(bind_statistics(&guarantee, &|statistic| { workload_statistic(statistic, data).or_else(|| domains.get(statistic).copied().flatten()) })) } +/// `guarantee` with the statistics `value` supplies bound. A sign gate +/// ([`samples_sign_statistic`](crate::accuracy::composition::samples_sign_statistic)) +/// is an indicator that only 1 (proven) opens: any other value leaves it +/// unbound, so it can never read as a perfect bound. +fn bind_statistics( + guarantee: &AccuracyGuarantee, + value: &dyn Fn(&str) -> Option, +) -> AccuracyGuarantee { + guarantee.bind(&|statistic| { + let supplied = value(statistic); + match parse_samples_sign_statistic(statistic) { + Some(_) => supplied.filter(|v| *v == 1.0), + None => supplied, + } + }) +} + /// A data-workload statistic Stage 1 left symbolic: the input row count, /// one source's stream norm or classic HLL's failure probability given one /// source's distinct bound, from evidence that holds at any planning time, @@ -1489,6 +1506,48 @@ mod tests { } } + /// The sign gate is an indicator that only 1 (proven) opens: any other + /// value a binder supplies, 0 included, leaves it unbound rather than + /// reading as a perfect bound. Other statistics pass through. + #[test] + fn the_sign_gate_opens_only_at_one() { + use crate::accuracy::composition::samples_sign_statistic; + let gated = AccuracyGuarantee { + metric: ErrorMetric::RelativeValue, + bound: BoundExpr::Product { + factors: vec![ + BoundExpr::Constant { value: 0.01 }, + BoundExpr::Unknown { + statistic: samples_sign_statistic("m"), + }, + ], + }, + failure_probability: ProbabilityExpr::Constant { value: 0.001 }, + provenance: vec![], + }; + for value in [0.0, 0.5, 2.0, -1.0] { + assert_eq!( + bind_statistics(&gated, &|_| Some(value)).bound.evaluate(), + None, + "{value}" + ); + } + assert_eq!( + bind_statistics(&gated, &|_| Some(1.0)).bound.evaluate(), + Some(0.01) + ); + let norm = AccuracyGuarantee { + bound: BoundExpr::Unknown { + statistic: INPUT_ROW_COUNT.into(), + }, + ..gated + }; + assert_eq!( + bind_statistics(&norm, &|_| Some(0.0)).bound.evaluate(), + Some(0.0) + ); + } + /// A relative bound gated on the sign of one metric's samples is bound /// when the data workload declares that metric a counter, and stays /// unknown for a gauge, an undeclared metric or a counter by another From d542c30ac3805da87398a4a5c64c8eba9304d02f Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 14 Oct 2026 22:24:39 +0000 Subject: [PATCH 15/15] docs: update the Example 1, 2 and 3 selections this PR changes The Example 3 acceptance spec and the stage viewer guide now give 3a's 972 candidates and its all-KLL selection, 3b's KLL selection tied with DDSketch, the per-second totals, and the shared-input plans that tie with the independent one. --- .../proposals/planner-layering-example3-acceptance.md | 9 +++++---- docs/user_guide_docs/stage-viewer.md | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/design_docs/proposals/planner-layering-example3-acceptance.md b/docs/design_docs/proposals/planner-layering-example3-acceptance.md index 6ef0e8a01..a09a29ac2 100644 --- a/docs/design_docs/proposals/planner-layering-example3-acceptance.md +++ b/docs/design_docs/proposals/planner-layering-example3-acceptance.md @@ -8,9 +8,10 @@ windows" and the Pass 2 section. Tests: `tests/planner_layering_common/`; the Example 3/4 workloads and the stage pipeline in `asap_devtools::{planner_layering_examples, planning_stages}`). Devtool: `stage_pipeline --example planner-layering-3a` and -`planner-layering-3b`. Both run every candidate (243 for 3a) through plan +`planner-layering-3b`. Both run every candidate (972 for 3a: 243 from +Pass 1, each with 3 Pass 2 shared-input candidates) through plan selection; `--max-displayed` (default 64) limits only how many the document -shows, so 3a selects P109: KLL for q1 and q2, exact for q3–q5. +shows, so 3a selects P485: KLL for all five queries. This spec was written by a test designer who does not implement the stages. Tests run the stages through `planner_layering_common::run_stages`, which @@ -41,9 +42,9 @@ Shared data workload, except Pattern A's `arrival` is `mixed`. |---|---|---| | Query-language frontends | A: five scan → range → quantile chains. B: one. | Same; q3–q5 also have a `time_shift`. | | Logical ASAP-aware optimization, Pass 1 | A: exact or KLL over its own interval per query, 2⁵ = 32. B: exact, KLL. | A: exact, KLL (k = 547), DDSketch per query, 3⁵ = **243**. B: exact, KLL (k = 269), DDSketch, **3**. | -| Logical ASAP-aware optimization, Pass 2 | A: window composition adds one EH of KLLs over [T − 5 y, T], one merge + p99 estimate per query, plus a candidate for every grouping of two or more queries onto shared window summaries ("hundreds"). B: each Pass 1 option × {none, sliding L = 5 min s = 1 min (5 active windows), 1-min tumbling (merge 5)} = 6. Independent kept. | Not implemented: no shared input, no window summaries. | +| Logical ASAP-aware optimization, Pass 2 | A: window composition adds one EH of KLLs over [T − 5 y, T], one merge + p99 estimate per query, plus a candidate for every grouping of two or more queries onto shared window summaries ("hundreds"). B: each Pass 1 option × {none, sliding L = 5 min s = 1 min (5 active windows), 1-min tumbling (merge 5)} = 6. Independent kept. | Identical-expression rule only: each Pass 1 candidate also gets its shared-input candidates (A: 3 per Pass 1 candidate; B: none). No window summaries. | | Physical ASAP-aware optimization | See Example 4. | One physical candidate per logical candidate, every sub-DAG `NotMaterialized`. | -| Plan selection | Not stated for Example 3. | Every candidate is valid. A: KLL for q1 and q2, exact for q3–q5, whose rows are underestimated (#632); DDSketch costs slightly more than KLL. B: exact, since each window holds 20 samples per series; DDSketch and then KLL cost more. | +| Plan selection | Not stated for Example 3. | Every candidate is valid. A: KLL for all five queries (P485); DDSketch in place of KLL ties, and the tie goes to enumeration order. B: KLL (P2, 2.100 per second), tied with DDSketch and selected on enumeration order; exact costs more (2.400). | ## Invariants and tests diff --git a/docs/user_guide_docs/stage-viewer.md b/docs/user_guide_docs/stage-viewer.md index debf3b8cf..14024c8f0 100644 --- a/docs/user_guide_docs/stage-viewer.md +++ b/docs/user_guide_docs/stage-viewer.md @@ -68,10 +68,10 @@ plan wins. | Tab | Workload | What to look for | | --- | --- | --- | -| 1 · dashboard panels | Two PromQL panels over 1M counter series every 10 s: an exact per-job rate, and the approximate top-10 series per job | The exact query stays exact; the CMS+heap and CountSketch+heap top-k plans are rejected with their reasons, so the all-exact plan wins; each query reads its own scan until cross-query sharing lands | -| 2 · SQL flow statistics | Distinct sources, entropy and L2 of per-source counts over the last minute of flows | All 64 plans compose and are priced; none is pruned. The SQL frontend does not yet recognize entropy and L2, so Q2 and Q3 are per-source counts with exact, Count-Min, CountSketch or UnivMon options. 56 plans are invalid: HLL for Q1 because its bound has "failure probability None", and UnivMon for Q1 or Count-Min or CountSketch for Q2 and Q3 because "the output has no guarantee, and the deployment has no evidence that it meets target". 8 are valid: Q1 exact or KMV with Q2 and Q3 each exact or UnivMon. Theta is not offered: `asap_sketchlib` has no Theta sketch. P1 (all exact) is selected as the first of four plans tied at 52.80111 ms (with UnivMon on Q2, Q3 or both); KMV on Q1 costs more, because its state is sized from its k | -| 3a · historical p99 batch | Five p99 reports over 1–5 years, run once | All 243 plans are priced. Nothing is shared yet: each query scans its own interval and builds its own sketch at query time. P109 is selected: KLL for the 5-year and 1-year queries, and exact for the three offset queries, whose rows are underestimated (#632). DDSketch in place of KLL costs slightly more, because its state is larger. The all-exact plan costs about 29% more | -| 3b · live p99 panel | p99 over the last 5 min every minute, 1M series | P1, exact over the window's raw samples at each evaluation, is selected: each series' 5-min window holds 20 samples, less state than any sketch. DDSketch costs about 5.9 times as much and KLL about 6.7 times. Everything runs at query time and nothing is materialized. KLL panes kept from ingestion time and the 200 ms latency check come with the window-summary PRs | +| 1 · dashboard panels | Two PromQL panels over 1M counter series every 10 s: an exact per-job rate, and the approximate top-10 series per job | The exact query stays exact; the CMS+heap and CountSketch+heap top-k plans are rejected with their reasons, so the all-exact plan wins; the plans that share the two queries' scan and range selector tie with it, and the independent plan is kept on enumeration order | +| 2 · SQL flow statistics | Distinct sources, entropy and L2 of per-source counts over the last minute of flows | All 64 plans compose and are priced; none is pruned. The SQL frontend does not yet recognize entropy and L2, so Q2 and Q3 are per-source counts with exact, Count-Min, CountSketch or UnivMon options. 56 plans are invalid: HLL for Q1 because its bound has "failure probability None", and UnivMon for Q1 or Count-Min or CountSketch for Q2 and Q3 because "the output has no guarantee, and the deployment has no evidence that it meets target". 8 are valid: Q1 exact or KMV with Q2 and Q3 each exact or UnivMon. Theta is not offered: `asap_sketchlib` has no Theta sketch. P1 (all exact) is selected at 8.64 per second, tied with its shared-input plans; KMV on Q1 costs more | +| 3a · historical p99 batch | Five p99 reports over 1–5 years, run once | All 972 plans are priced: 243 Pass 1 plans, each with 3 shared-input plans. Each query builds its own sketch at query time. P485 is selected: KLL for all five queries. DDSketch in place of KLL ties, and the tie goes to enumeration order. The all-exact plan costs about 16% more | +| 3b · live p99 panel | p99 over the last 5 min every minute, 1M series | P2, KLL, is selected at 2.100 per second, tied with DDSketch and chosen on enumeration order; exact over the window's raw samples costs 2.400. Memory a query holds only while it runs is not charged. Everything runs at query time and nothing is materialized. KLL panes kept from ingestion time and the 200 ms latency check come with the window-summary PRs | | 4a · monthly p99 reports | Example 3a repeated monthly | The same plan, with its cost amortized over a month | | 4b · panes that pay off | p99 over the last hour every 10 min, 1k series sampled every second, raw data not kept by the deployment | Maintaining six 10-min KLL panes at ingestion time wins |