From 2bcce92beff76aeecdfd672447c3dad4661dd5db Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:53:56 +0000 Subject: [PATCH 1/2] feat(asap-aware-mapping)!: deployment capabilities for Stages 1-3 DeploymentCapabilities states which sketch families and readouts the executor supports, whether raw data is retained and whether a query-time result may be kept across evaluations. It sits in PlanningModels, and stage_pipeline reads it from --deployment and shows it read-only under the document's top-level deployment. - Stage 1: Pass 1 offers no unsupported summary (PruneReason::UnsupportedByDeployment); Pass 2 adds none. - Stage 2: with query_time_retention, a tumbling-pane chain whose readers repeat at its pane width is also offered MatAtQuery (Example 4's B3). - Stage 3: such a chain is priced as kept across evaluations (one pane scanned and built per evaluation, N - 1 panes held); without raw data retained, a query-time scan makes a candidate invalid. - Remove CostModel::summary_support_evidence, replaced by the supported summaries. - Example 4's B3 tests run with query-time retention and are no longer ignored. The default deployment plans exactly as before. --- crates/asap-aware-mapping/src/cost_model.rs | 7 - .../src/deployment_capabilities.rs | 105 +++++ crates/asap-aware-mapping/src/lib.rs | 3 + crates/asap-aware-mapping/src/pass/mod.rs | 26 +- .../src/pass/stage1_to_stage3.rs | 28 +- .../src/per_computation_candidates.rs | 150 ++++++ crates/asap-aware-mapping/src/replacement.rs | 38 +- .../src/stage2_physical_asap_optimization.rs | 187 ++++++-- .../src/stage3_plan_selection.rs | 443 ++++++++++++++++-- crates/devtools/src/bin/stage_pipeline.rs | 26 +- crates/devtools/src/planning_stages.rs | 15 +- crates/devtools/tests/stage_pipeline.rs | 76 +++ .../tests/deployment_capabilities.rs | 131 ++++++ .../tests/planner_layering_common/mod.rs | 30 +- .../tests/planner_layering_example4.rs | 41 +- .../stage2_physical_asap_optimization_sql.rs | 1 + .../types/src/post_asap/family_capability.rs | 6 +- .../examples/planner-layering-example1.json | 5 + 18 files changed, 1146 insertions(+), 172 deletions(-) create mode 100644 crates/asap-aware-mapping/src/deployment_capabilities.rs create mode 100644 crates/integration-tests/tests/deployment_capabilities.rs diff --git a/crates/asap-aware-mapping/src/cost_model.rs b/crates/asap-aware-mapping/src/cost_model.rs index 944aa40d2..51fbc86e7 100644 --- a/crates/asap-aware-mapping/src/cost_model.rs +++ b/crates/asap-aware-mapping/src/cost_model.rs @@ -761,13 +761,6 @@ pub trait CostModel { f64::NAN } - /// Physical feasibility evidence for a complete summary candidate. - /// `None` defers admission to physical/deployment compilation; `Some(false)` - /// excludes the candidate without changing its computation or parameters. - fn summary_support_evidence(&self, _summary: &OperatorNode) -> Option { - None - } - /// Which mixed exact/summary execution shapes the downstream runtime /// advertises (issue #171). Gates candidate *generation* in /// [`crate::exact_composition::ExactCompositionStrategy`]: a shape the diff --git a/crates/asap-aware-mapping/src/deployment_capabilities.rs b/crates/asap-aware-mapping/src/deployment_capabilities.rs new file mode 100644 index 000000000..1cce407b1 --- /dev/null +++ b/crates/asap-aware-mapping/src/deployment_capabilities.rs @@ -0,0 +1,105 @@ +//! What the deployment's executor can run and keep (#509's third +//! deployment input, beside the cost and accuracy models). +//! +//! [`DeploymentCapabilities`] says three things: +//! +//! * which sketch families the executor supports, and which readouts it +//! supports for each. A family is supported when the executor can build +//! its state, and a readout of it when the executor can answer that +//! statistic ([`StatisticKind`]) from the state. Stage 1 (Pass 1) offers +//! no summary candidate whose family or readout is unsupported, so no +//! Pass 2 candidate has one either (Pass 2 rules share, split into panes +//! or reuse existing builds; none adds a family or a readout). Exact +//! accumulators and plain exact execution are always supported; +//! * whether raw samples are retained. If they are, Stage 3 counts them on +//! disk once per source, at the longest window a query-time scan of it +//! reads. If they are not, raw samples are not kept after they arrive, so +//! a candidate that scans raw data at query time is invalid: only +//! ingestion-time work reads raw rows; +//! * whether a query-time result (`MatAtQuery`) may be kept across +//! evaluations. Its retention is derived from its consumers: a +//! tumbling-pane chain whose every reader repeats at the pane width +//! builds one new pane per evaluation and keeps the others for the next +//! ones, covering the readers' longest look-back. Without it, a +//! `MatAtQuery` result is reused only within one evaluation. +//! +//! The default supports everything the planner can build, retains raw data +//! and keeps no query-time result across evaluations, which is the +//! planner's behavior without deployment inputs. + +use asap_types::post_asap::{SketchAlgorithm, StatisticKind}; +use serde::{Deserialize, Serialize}; + +/// What the deployment's executor can run and keep. Planning reads it in +/// Stage 1 (`supported_summaries`), Stage 2 and Stage 3 (the two +/// retention switches). It is data only; `stage_pipeline` reads it from +/// `--deployment ` and shows it in the stage document. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct DeploymentCapabilities { + /// The sketch families the executor can build, each with the readouts + /// it can answer from one; `None` (the default) supports every family + /// and readout the planner can build. + pub supported_summaries: Option>, + /// Whether raw samples are kept for query-time scans; default `true`. + pub raw_data_retained: bool, + /// Whether a query-time (`MatAtQuery`) result may be kept across + /// evaluations; default `false`. + pub query_time_retention: bool, +} + +/// One sketch family the executor can build, with the readouts it can +/// answer from the family's state. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SupportedSummary { + /// The family the executor can build. + pub family: SketchAlgorithm, + /// The statistics it can answer from one state of the family. + pub readouts: Vec, +} + +impl Default for DeploymentCapabilities { + fn default() -> Self { + Self { + supported_summaries: None, + raw_data_retained: true, + query_time_retention: false, + } + } +} + +impl DeploymentCapabilities { + /// Whether the executor can build `family` and answer `readout` from it. + pub fn supports(&self, family: &SketchAlgorithm, readout: StatisticKind) -> bool { + self.supported_summaries.as_ref().is_none_or(|supported| { + supported + .iter() + .any(|s| s.family == *family && s.readouts.contains(&readout)) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The default supports every family and readout, retains raw data and + /// keeps no query-time result across evaluations; a JSON object with + /// only some fields keeps the defaults for the rest. + #[test] + fn default_supports_everything_and_json_fills_the_rest() { + let default = DeploymentCapabilities::default(); + assert!(default.supports(&SketchAlgorithm::Theta, StatisticKind::TopK)); + assert!(default.raw_data_retained && !default.query_time_retention); + let parsed: DeploymentCapabilities = serde_json::from_str( + r#"{"supported_summaries": [{"family": "Kll", "readouts": ["Quantile"]}]}"#, + ) + .unwrap(); + assert!(parsed.supports(&SketchAlgorithm::Kll, StatisticKind::Quantile)); + assert!(!parsed.supports(&SketchAlgorithm::Kll, StatisticKind::Total)); + assert!(!parsed.supports(&SketchAlgorithm::DDSketch, StatisticKind::Quantile)); + assert!(parsed.raw_data_retained && !parsed.query_time_retention); + assert!(serde_json::from_str::(r#"{"raw_data": false}"#).is_err()); + } +} diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 2d59269e3..6efbf7269 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -147,6 +147,8 @@ pub mod accuracy; pub mod analytical_cost; pub mod cost_model; +/// The deployment's capabilities: supported summaries and what it retains. +pub mod deployment_capabilities; pub mod empirical_cost; pub mod empirical_resources; pub mod exact_composition; @@ -179,6 +181,7 @@ pub use cost_model::{ CostModel, CostProvenance, CostUnit, DefaultCostModel, ExactCompositionCostInputs, ExactCompositionCostRequest, ValueOperationCapabilities, }; +pub use deployment_capabilities::{DeploymentCapabilities, SupportedSummary}; pub use exact_composition::{ExactComposition, ExactCompositionStrategy, OperationPlacement}; pub use explanation::{ explain_replacements, explain_replacements_with, ExplanationKind, ReplacementExplanation, diff --git a/crates/asap-aware-mapping/src/pass/mod.rs b/crates/asap-aware-mapping/src/pass/mod.rs index 2760cc823..464a27fc1 100644 --- a/crates/asap-aware-mapping/src/pass/mod.rs +++ b/crates/asap-aware-mapping/src/pass/mod.rs @@ -31,6 +31,7 @@ use crate::accuracy::{ AccuracyEvidenceProvider, AccuracyModel, DefaultAccuracyModel, NoAccuracyEvidence, }; use crate::cost_model::{CostModel, DefaultCostModel}; +use crate::deployment_capabilities::DeploymentCapabilities; use crate::per_computation_candidates::PerComputationCandidateError; use crate::stage3_plan_selection::{PlanSelectionSettings, Selection, SelectionError}; @@ -45,16 +46,19 @@ static DEFAULT_ACCURACY_MODEL: DefaultAccuracyModel = DefaultAccuracyModel; static NO_ACCURACY_EVIDENCE: NoAccuracyEvidence = NoAccuracyEvidence; static DEFAULT_PLAN_SELECTION: std::sync::LazyLock = std::sync::LazyLock::new(PlanSelectionSettings::default); +static DEFAULT_DEPLOYMENT: std::sync::LazyLock = + std::sync::LazyLock::new(DeploymentCapabilities::default); // ── Input ──────────────────────────────────────────────────────────────── /// Planning logic, as opposed to the scoped facts it consumes: a model can have /// a built-in default, evidence about a particular deployment cannot. /// -/// Plan selection reads `accuracy`, `evidence` and `plan_selection`. It -/// measures candidates with its own analytical model and compares them -/// under `plan_selection`'s cost model, so [`Stage1ToStage3Pass`] does not -/// read `cost`. +/// Plan selection reads `accuracy`, `evidence`, `plan_selection` and +/// `deployment`. It measures candidates with its own analytical model and +/// compares them under `plan_selection`'s cost model, so +/// [`Stage1ToStage3Pass`] does not read `cost`. Stage 1 and Stage 2 read +/// `deployment` too. #[derive(Clone, Copy)] #[non_exhaustive] pub struct PlanningModels<'a> { @@ -65,6 +69,11 @@ pub struct PlanningModels<'a> { /// cost model 1 with illustrative unit costs), the resource speed and /// the non-repeating query horizon. pub plan_selection: &'a PlanSelectionSettings, + /// What the deployment's executor can run and keep: the summaries it + /// supports (Stage 1), whether it retains raw data (Stage 3) and + /// whether it keeps query-time results across evaluations (Stages 2 + /// and 3); default [`DeploymentCapabilities::default`]. + pub deployment: &'a DeploymentCapabilities, } impl<'a> PlanningModels<'a> { @@ -78,6 +87,7 @@ impl<'a> PlanningModels<'a> { accuracy, evidence, plan_selection: &DEFAULT_PLAN_SELECTION, + deployment: &DEFAULT_DEPLOYMENT, } } @@ -88,6 +98,7 @@ impl<'a> PlanningModels<'a> { accuracy: &DEFAULT_ACCURACY_MODEL, evidence: &NO_ACCURACY_EVIDENCE, plan_selection: &DEFAULT_PLAN_SELECTION, + deployment: &DEFAULT_DEPLOYMENT, } } @@ -112,6 +123,13 @@ impl<'a> PlanningModels<'a> { self.plan_selection = plan_selection; self } + + /// These models with the deployment's capabilities (supported + /// summaries, raw-data and query-time retention). + pub fn with_deployment(mut self, deployment: &'a DeploymentCapabilities) -> Self { + self.deployment = deployment; + self + } } #[derive(Clone, Copy)] diff --git a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs index 3df6cecb3..184ccae15 100644 --- a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs +++ b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs @@ -45,9 +45,7 @@ use std::rc::Rc; use asap_types::ir::flat::flatten; use asap_types::ir::schema_support::with_promql_series_identity; use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::workload::{ - DataWorkload, QueryLanguage, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, -}; +use asap_types::workload::{DataWorkload, QueryLanguage, QueryRecurrence, QueryWorkloadEntry}; use super::{OptimizationInput, OptimizationPass, OptimizeError, PlanOutput, QueryPlan}; use crate::asap_aware_cse::{ @@ -61,7 +59,9 @@ use crate::per_computation_candidates::{ compose_logical_candidate, generate_per_computation_candidates, PerComputationCandidateError, PerComputationCandidates, }; -use crate::stage2_physical_asap_optimization::{optimize_physical_asap, PhysicalCandidate}; +use crate::stage2_physical_asap_optimization::{ + fixed_interval_ms, optimize_physical_asap, PhysicalCandidate, +}; use crate::stage3_plan_selection::{select_plan, Selection, SelectionError}; use crate::PlanningModels; @@ -295,17 +295,6 @@ impl Pass2 { } } -/// The fixed interval at which a query repeats, in milliseconds. -fn fixed_interval_ms(recurrence: &QueryRecurrence) -> Option { - match recurrence { - QueryRecurrence::Repeated( - RepeatedDemand::FixedInterval(interval) - | RepeatedDemand::FixedIntervalAt { interval, .. }, - ) => Some(u64::from(interval.0)).filter(|&ms| ms > 0), - _ => None, - } -} - /// The Pass 2 candidates of the composed candidate with roots `roots`, in /// enumeration order, each distinct candidate once. `interval_ms[q]` is the /// fixed interval of query `q`, if it repeats at one. @@ -488,8 +477,10 @@ impl Pass2List { /// [`run_stage1_to_stage3`] after Pass 1: enumerates every Stage 1 /// candidate of `per_computation` (the Pass 1 inventory of the frontends' -/// roots, [`stage1_logical_asap_optimization`]), so a caller can read the -/// inventory, e.g. its +/// roots, [`stage1_logical_asap_optimization`]) without the summaries the +/// deployment does not support +/// ([`PerComputationCandidates::without_unsupported_summaries`]), so a +/// caller can read the inventory, e.g. its /// [`combinations`](PerComputationCandidates::combinations), before /// enumerating. pub fn run_stage1_to_stage3_from_pass1( @@ -498,6 +489,7 @@ pub fn run_stage1_to_stage3_from_pass1( data: &DataWorkload, models: PlanningModels<'_>, ) -> Stage1ToStage3Run { + let per_computation = per_computation.without_unsupported_summaries(models.deployment); let recurrence: Vec<_> = entries.iter().map(|e| e.recurrence.clone()).collect(); let stage1 = stage1_logical_asap_optimization(&per_computation, &recurrence).collect(); run_stage2_and_stage3(per_computation, stage1, entries, data, models) @@ -523,7 +515,7 @@ pub fn run_stage2_and_stage3( let number = index + 1; let id = format!("L{number}"); if let Ok(roots) = &candidate.roots { - match optimize_physical_asap(&id, roots, entries, data) { + match optimize_physical_asap(&id, roots, entries, data, models.deployment) { Ok(implemented) => { for (k, mut p) in implemented.into_iter().enumerate() { p.id = match k { diff --git a/crates/asap-aware-mapping/src/per_computation_candidates.rs b/crates/asap-aware-mapping/src/per_computation_candidates.rs index bbd26fb79..19934a288 100644 --- a/crates/asap-aware-mapping/src/per_computation_candidates.rs +++ b/crates/asap-aware-mapping/src/per_computation_candidates.rs @@ -119,6 +119,7 @@ use crate::accuracy::composition::{ }; use crate::accuracy::estimators::{hll, univmon}; use crate::accuracy::{per_input_target, AccuracyModel, DefaultAccuracyModel, PropagationStats}; +use crate::deployment_capabilities::DeploymentCapabilities; use crate::function_rules::function_rules; use crate::realization::{ accuracy_budget, accuracy_target, default_size_params, summary_candidates, @@ -168,6 +169,10 @@ pub enum PruneReason { RequiresExact, /// `count` counts rows, so a possibly-null `avg` input changes the result. NullableAvgInput, + /// The deployment's executor cannot build the summary's family or + /// answer the measure's readout from it + /// ([`DeploymentCapabilities::supports`]). + UnsupportedByDeployment, } /// Unranked candidates of one measure and the candidates pruned for it. @@ -347,6 +352,37 @@ impl PerComputationCandidates { .is_some_and(|inner| choice[inner] != 0) }) } + + /// This inventory without the summary candidates `deployment` does not + /// support: a target's own are moved to its pruned candidates with + /// [`PruneReason::UnsupportedByDeployment`], and a fused computation's + /// are dropped (with all of them, the fusion is). The readout is the + /// statistic the measure's estimate reads (the outer measure's for a + /// fused computation). With every summary supported it is unchanged. + pub fn without_unsupported_summaries(mut self, deployment: &DeploymentCapabilities) -> Self { + for target in &mut self.targets { + let Some(NonASAPOp::Aggregate { measures, .. }) = target.aggregate.non_asap() else { + continue; + }; + let Ok(readout) = statistic(&measures[target.measure]).map(|s| s.kind()) else { + continue; + }; + let unsupported = |candidate: &ComputationCandidate| { + matches!(candidate, ComputationCandidate::Summary(family) + if !deployment.supports(family, readout)) + }; + target + .choices + .prune(PruneReason::UnsupportedByDeployment, unsupported); + if let Some(fused) = &mut target.fused { + fused.candidates.retain(|candidate| !unsupported(candidate)); + if fused.candidates.is_empty() { + target.fused = None; + } + } + } + self + } } /// The indices of `targets` some target's fused computation absorbs. @@ -3214,6 +3250,7 @@ fn statistic(intent: &AggIntent) -> Result DeploymentCapabilities { + DeploymentCapabilities { + supported_summaries: Some( + supported + .iter() + .map(|(family, readouts)| crate::SupportedSummary { + family: family.clone(), + readouts: readouts.to_vec(), + }) + .collect(), + ), + ..DeploymentCapabilities::default() + } + } + + /// A family the deployment cannot build is pruned from Pass 1 with + /// `UnsupportedByDeployment`; the supported family and the exact option + /// stay. With every summary supported the inventory is unchanged. + #[test] + fn a_family_the_deployment_cannot_build_is_pruned() { + let query = "quantile_over_time(0.99, m[5m])"; + let unrestricted = inventory(query); + let restricted = inventory(query).without_unsupported_summaries(&supporting(&[( + SketchAlgorithm::Kll, + &[StatisticKind::Quantile], + )])); + let target = &restricted.targets[0]; + assert_eq!( + target.choices.candidates, + [ + ComputationCandidate::Exact, + ComputationCandidate::Summary(SketchAlgorithm::Kll) + ] + ); + assert_eq!( + target.choices.pruned, + [( + ComputationCandidate::Summary(SketchAlgorithm::DDSketch), + PruneReason::UnsupportedByDeployment + )] + ); + let default = + inventory(query).without_unsupported_summaries(&DeploymentCapabilities::default()); + assert_eq!(default.targets[0].choices, unrestricted.targets[0].choices); + } + + /// A family the deployment builds but cannot answer the measure's + /// readout from is pruned: UnivMon, supported for distinct counts and + /// L2 only, is no count candidate; Count-Min and CountSketch, supported + /// for the total, stay. + #[test] + fn a_readout_the_deployment_cannot_answer_prunes_its_family() { + let restricted = + inventory("count_over_time(m[1m])").without_unsupported_summaries(&supporting(&[ + (SketchAlgorithm::Cms, &[StatisticKind::Total]), + (SketchAlgorithm::CountSketch, &[StatisticKind::Total]), + ( + SketchAlgorithm::UnivMon, + &[StatisticKind::Distinct, StatisticKind::FrequencyL2], + ), + ])); + let target = &restricted.targets[0]; + assert_eq!( + target.choices.candidates, + [ + ComputationCandidate::ExactAccumulator(vec![ExactKind::Count]), + ComputationCandidate::Summary(SketchAlgorithm::Cms), + ComputationCandidate::Summary(SketchAlgorithm::CountSketch), + ] + ); + assert_eq!( + target.choices.pruned, + [( + ComputationCandidate::Summary(SketchAlgorithm::UnivMon), + PruneReason::UnsupportedByDeployment + )] + ); + } + + /// A fused whole-expression top-k loses the heap families the + /// deployment cannot answer top-k from, and the fusion goes with the + /// last of them: the target and the inner sum keep their exact options. + #[test] + fn a_fused_computation_loses_its_unsupported_heaps() { + let query = "topk by (job) (10, sum_over_time(m[1m]))"; + let one_heap = inventory(query).without_unsupported_summaries(&supporting(&[( + SketchAlgorithm::CountSketchWithHeap, + &[StatisticKind::TopK], + )])); + let topk = topk_target(&one_heap); + assert_eq!( + one_heap.targets[topk].fused.as_ref().unwrap().candidates, + [ComputationCandidate::Summary( + SketchAlgorithm::CountSketchWithHeap + )] + ); + assert_eq!(one_heap.combinations(), 2); + // Count-Min with a heap answers item frequencies, not top-k. + let none = inventory(query).without_unsupported_summaries(&supporting(&[( + SketchAlgorithm::CmsWithHeap, + &[StatisticKind::ItemFrequency], + )])); + let topk = topk_target(&none); + assert!(none.targets[topk].fused.is_none()); + assert_eq!( + none.targets[topk].choices.candidates, + [ComputationCandidate::Exact] + ); + assert_eq!(none.combinations(), 1); + } } diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index d557ae0f3..3c4370240 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -525,11 +525,10 @@ impl ReplacementSubDAG { Replacement::ExactComposition(composition) => { cost_model.value_operation_support_evidence(&composition.op, composition.placement) } - // Any summary decision, including one rooted in a relational - // operator above its evaluations, asks the deployment for support. - Replacement::SubDAG(node) if !self.is_logical_rewrite() => { - cost_model.summary_support_evidence(node) - } + // Summary support is the deployment's capabilities, which the + // planner pipeline reads (`DeploymentCapabilities`); this legacy + // search defers it to physical compilation. + Replacement::SubDAG(_) if !self.is_logical_rewrite() => None, Replacement::SubDAG(_) => Some(true), } } @@ -9277,35 +9276,6 @@ mod tests { ); } - // A cheap but physically infeasible candidate must not be selected. - #[test] - fn explicit_summary_infeasibility_prevents_selection() { - struct Unsupported; - impl CostModel for Unsupported { - fn rank_candidates( - &self, - _: &AggIntent, - candidates: &[SketchAlgorithm], - ) -> Vec { - candidates.to_vec() - } - fn candidate_cost(&self, _: &ReplacementSubDAG, _: &TargetSubDAG<'_>) -> Option { - Some(Cost(1.0)) - } - fn summary_support_evidence(&self, _: &OperatorNode) -> Option { - Some(false) - } - } - let root = lower_promql("sum_over_time(a[1m])", AccuracyTarget::exact()); - let space = search_workload(vec![("q", root)]).unwrap(); - let selected = space.global_selection(&Unsupported); - assert!(selected - .for_target(&space.roots[0].1) - .unwrap() - .chosen - .is_none()); - } - // Composable temporal/grouped Sum must be executable as one producer. #[test] fn grouped_temporal_sum_has_one_summary_producer_candidate() { diff --git a/crates/asap-aware-mapping/src/stage2_physical_asap_optimization.rs b/crates/asap-aware-mapping/src/stage2_physical_asap_optimization.rs index b186e37d6..35acaae63 100644 --- a/crates/asap-aware-mapping/src/stage2_physical_asap_optimization.rs +++ b/crates/asap-aware-mapping/src/stage2_physical_asap_optimization.rs @@ -12,9 +12,16 @@ //! * `NotMaterialized`: rebuilt at query time by every query that reads it; //! * `MatAtIngestion(Memory)`: maintained as data arrives and kept for the //! queries, when the build is eligible ([`IngestionPruneReason`]); -//! * `MatAtQuery(Memory)`: built once per evaluation for every query that -//! reads it, offered for a build two or more queries read. State is reused -//! only within one evaluation; keeping it across evaluations comes later. +//! * `MatAtQuery(Memory)`: built at query time for every query that reads +//! it, offered for a build two or more queries read, and, when the +//! deployment keeps query-time results across evaluations +//! ([`DeploymentCapabilities::query_time_retention`]), for a +//! tumbling-pane chain that builds one new pane per evaluation +//! ([`builds_one_pane_per_evaluation`]): every reader repeats at the pane +//! width. Such a chain is kept across evaluations (#509 Example 4's B3): +//! each evaluation builds the newest pane and reuses the others. Any +//! other `MatAtQuery` state is reused only within one evaluation. The IR +//! records no retention: Stage 3 derives it from the readers the same way. //! //! A build at ingestion time needs every build below it at ingestion time too //! (#509 Algorithm 3, line 13); such assignments are not generated. Every @@ -83,6 +90,8 @@ use asap_types::workload::{ }; use thiserror::Error; +use crate::deployment_capabilities::DeploymentCapabilities; + /// One CandidatePhysicalASAPDAG (Stage 2 output), derived from exactly one /// CandidateLogicalASAPDAG (Stage 1 output). It holds planning data only; /// display labels are derived from it by the devtools. @@ -180,23 +189,54 @@ pub(crate) struct MaterializationCandidate { pub(crate) readers: Vec, /// `Ok` when `MatAtIngestion` is an option. pub(crate) at_ingestion: Result<(), IngestionPruneReason>, + /// Whether it may be kept from query time across evaluations: the + /// deployment keeps query-time results and it is a tumbling-pane chain + /// that builds one new pane per evaluation. + pub(crate) kept_across_evaluations: bool, } impl MaterializationCandidate { /// Its options, `NotMaterialized` first: `MatAtIngestion` when eligible, - /// `MatAtQuery` when two or more queries read it. + /// `MatAtQuery` when two or more queries read it or it may be kept + /// across evaluations. pub(crate) fn options(&self) -> Vec { let mut options = vec![Materialization::NotMaterialized]; if self.at_ingestion.is_ok() { options.push(Materialization::MatAtIngestion(Medium::Memory)); } - if self.readers.len() > 1 { + if self.readers.len() > 1 || self.kept_across_evaluations { options.push(Materialization::MatAtQuery(Medium::Memory)); } options } } +/// The fixed interval at which a query repeats, in milliseconds. +pub(crate) fn fixed_interval_ms(recurrence: &QueryRecurrence) -> Option { + match recurrence { + QueryRecurrence::Repeated( + RepeatedDemand::FixedInterval(interval) + | RepeatedDemand::FixedIntervalAt { interval, .. }, + ) => Some(u64::from(interval.0)).filter(|&ms| ms > 0), + _ => None, + } +} + +/// Whether a tumbling-pane chain of `pane_width_ms`-wide panes read by +/// queries repeating as `readers` builds exactly one new pane per +/// evaluation: there is a reader, and every reader repeats at a fixed +/// interval equal to the pane width. Its other panes are the previous +/// evaluations' newest, so a deployment that keeps query-time results +/// reuses them (#509 Example 4's B3). +pub(crate) fn builds_one_pane_per_evaluation<'a>( + pane_width_ms: u64, + readers: impl IntoIterator, +) -> bool { + let mut readers = readers.into_iter().peekable(); + readers.peek().is_some() + && readers.all(|recurrence| fixed_interval_ms(recurrence) == Some(pane_width_ms)) +} + /// Stage 2 for one CandidateLogicalASAPDAG: implement every operator /// (physical operator implementation), then one physical candidate per /// materialization assignment of its mat_candidates, the all-`NotMaterialized` one @@ -204,16 +244,18 @@ impl MaterializationCandidate { /// its recurrence and predictability decide whether state may be kept from /// ingestion time; a missing entry is unknown. `id` is left empty for the /// caller to name. Sharing between `roots` is preserved: one memo serves the -/// whole workload. +/// whole workload. `deployment` says whether a query-time result may be +/// kept across evaluations. pub fn optimize_physical_asap( from_logical: &str, roots: &[QueryRoot], entries: &[QueryWorkloadEntry], data: &DataWorkload, + deployment: &DeploymentCapabilities, ) -> Result, PhysicalASAPOptimizationError> { let mut memo = HashMap::new(); let implemented = map_roots(roots, &mut |node| implement(node, &mut memo))?; - let mat_candidates = materialization_candidates(&implemented, entries, data); + let mat_candidates = materialization_candidates(&implemented, entries, data, deployment); let mut candidates = Vec::new(); for assignment in materialization_assignments(&mat_candidates) { let materialized: Vec<_> = mat_candidates @@ -273,13 +315,12 @@ pub fn physical_candidate_count( roots: &[QueryRoot], entries: &[QueryWorkloadEntry], data: &DataWorkload, + deployment: &DeploymentCapabilities, ) -> Result { let mut memo = HashMap::new(); let implemented = map_roots(roots, &mut |node| implement(node, &mut memo))?; - Ok( - materialization_assignments(&materialization_candidates(&implemented, entries, data)) - .count(), - ) + let mat_candidates = materialization_candidates(&implemented, entries, data, deployment); + Ok(materialization_assignments(&mat_candidates).count()) } /// The operator nodes `root` reads. @@ -291,11 +332,13 @@ fn root_operators(root: &QueryRoot) -> Vec<&Rc> { } /// Every summary build of `roots` (implemented), in discovery order, with -/// its readers and whether it may run at ingestion time. +/// its readers, whether it may run at ingestion time and whether it may be +/// kept from query time across evaluations. pub(crate) fn materialization_candidates( roots: &[QueryRoot], entries: &[QueryWorkloadEntry], data: &DataWorkload, + deployment: &DeploymentCapabilities, ) -> Vec { let mut nodes: Vec> = Vec::new(); let mut readers: HashMap<*const OperatorNode, Vec> = HashMap::new(); @@ -336,10 +379,23 @@ pub(crate) fn materialization_candidates( .map(|_| ()) .map_err(|e| IngestionPruneReason::UpstreamAtQueryTime(e.to_string())) }); + // A reader without a workload entry has an unknown recurrence. + let recurrences: Option> = readers + .iter() + .map(|&q| entries.get(q).map(|e| &e.recurrence)) + .collect(); + let kept_across_evaluations = deployment.query_time_retention + && match (node.asap(), recurrences) { + (Some(ASAPOp::PaneChain { pane_width_ms, .. }), Some(recurrences)) => { + builds_one_pane_per_evaluation(*pane_width_ms, recurrences) + } + _ => false, + }; MaterializationCandidate { node: Rc::clone(node), readers, at_ingestion, + kept_across_evaluations, } }) .collect() @@ -659,9 +715,15 @@ mod tests { /// The all-`NotMaterialized` candidate, with no workload declared. fn first_candidate(roots: &[QueryRoot]) -> PhysicalCandidate { - optimize_physical_asap("L1", roots, &[], &DataWorkload::default()) - .unwrap() - .remove(0) + optimize_physical_asap( + "L1", + roots, + &[], + &DataWorkload::default(), + &DeploymentCapabilities::default(), + ) + .unwrap() + .remove(0) } fn operator_root(candidate: &PhysicalCandidate, i: usize) -> Rc { @@ -800,8 +862,14 @@ mod tests { ), ]; for root in cases { - let error = optimize_physical_asap("L1", &[root.into()], &[], &DataWorkload::default()) - .unwrap_err(); + let error = optimize_physical_asap( + "L1", + &[root.into()], + &[], + &DataWorkload::default(), + &DeploymentCapabilities::default(), + ) + .unwrap_err(); assert!( matches!(error, PhysicalASAPOptimizationError::ExactTopK(_)), "{error}" @@ -1010,8 +1078,14 @@ mod tests { #[test] fn a_pane_chain_over_arriving_data_may_be_maintained_at_ingestion_time() { let root = in_panes(summary_root("quantile_over_time(0.99, m[5m])")); - let candidates = - optimize_physical_asap("L1", &[root], &[every_minute()], &arriving()).unwrap(); + let candidates = optimize_physical_asap( + "L1", + &[root], + &[every_minute()], + &arriving(), + &DeploymentCapabilities::default(), + ) + .unwrap(); assert_eq!(candidates.len(), 2); let [b2, b1] = &candidates[..] else { unreachable!() @@ -1069,12 +1143,23 @@ mod tests { (once, arriving(), Why::NotRepeated), ] { let roots = [root.clone()]; - let candidates = - optimize_physical_asap("L1", &roots, std::slice::from_ref(&demand), &data).unwrap(); + let candidates = optimize_physical_asap( + "L1", + &roots, + std::slice::from_ref(&demand), + &data, + &DeploymentCapabilities::default(), + ) + .unwrap(); assert_eq!(candidates.len(), 1, "{why:?}"); let mut memo = HashMap::new(); let implemented = map_roots(&roots, &mut |n| implement(n, &mut memo)).unwrap(); - let mat_candidates = materialization_candidates(&implemented, &[demand], &data); + let mat_candidates = materialization_candidates( + &implemented, + &[demand], + &data, + &DeploymentCapabilities::default(), + ); assert_eq!(mat_candidates.len(), 1); assert_eq!(mat_candidates[0].at_ingestion, Err(why)); } @@ -1090,11 +1175,21 @@ mod tests { let roots = [in_panes(summary_root(query))]; let mut memo = HashMap::new(); let implemented = map_roots(&roots, &mut |n| implement(n, &mut memo)).unwrap(); - let mat_candidates = - materialization_candidates(&implemented, &[every_minute()], &arriving()); + let mat_candidates = materialization_candidates( + &implemented, + &[every_minute()], + &arriving(), + &DeploymentCapabilities::default(), + ); assert_eq!(mat_candidates[0].at_ingestion, Ok(()), "{query}"); - let candidates = - optimize_physical_asap("L1", &roots, &[every_minute()], &arriving()).unwrap(); + let candidates = optimize_physical_asap( + "L1", + &roots, + &[every_minute()], + &arriving(), + &DeploymentCapabilities::default(), + ) + .unwrap(); assert_eq!(candidates.len(), 2, "{query}"); } } @@ -1120,7 +1215,12 @@ mod tests { let roots = [root]; let mut memo = HashMap::new(); let implemented = map_roots(&roots, &mut |n| implement(n, &mut memo)).unwrap(); - let mat_candidates = materialization_candidates(&implemented, &[every_minute()], &data); + let mat_candidates = materialization_candidates( + &implemented, + &[every_minute()], + &data, + &DeploymentCapabilities::default(), + ); assert_eq!( mat_candidates[0].at_ingestion, Err(Why::CounterNeedsTimeOrder) @@ -1161,9 +1261,14 @@ mod tests { let mut memo = HashMap::new(); let implemented = map_roots(std::slice::from_ref(root), &mut |n| implement(n, &mut memo)).unwrap(); - materialization_candidates(&implemented, &[demand], &arriving()) - .remove(0) - .at_ingestion + materialization_candidates( + &implemented, + &[demand], + &arriving(), + &DeploymentCapabilities::default(), + ) + .remove(0) + .at_ingestion }; assert_eq!(mat_candidates(&root, at(MINUTE as u32)), Ok(())); assert_eq!( @@ -1176,6 +1281,7 @@ mod tests { std::slice::from_ref(&root), &[at(MINUTE as u32)], &arriving(), + &DeploymentCapabilities::default(), ) .unwrap(); assert_eq!(materialized(&candidates[1]), [AT_INGESTION]); @@ -1240,8 +1346,14 @@ mod tests { .unwrap(), ); let roots = [QueryRoot::Operator(p99), QueryRoot::Operator(p50)]; - let candidates = - optimize_physical_asap("L1", &roots, &[], &DataWorkload::default()).unwrap(); + let candidates = optimize_physical_asap( + "L1", + &roots, + &[], + &DataWorkload::default(), + &DeploymentCapabilities::default(), + ) + .unwrap(); assert_eq!(candidates.len(), 2); assert_eq!( materialized(&candidates[1]), @@ -1283,9 +1395,14 @@ mod tests { ranged, ); let roots = [QueryRoot::Operator(panes), QueryRoot::Operator(exact)]; - let candidates = - optimize_physical_asap("L1", &roots, &[every_minute(), every_minute()], &arriving()) - .unwrap(); + let candidates = optimize_physical_asap( + "L1", + &roots, + &[every_minute(), every_minute()], + &arriving(), + &DeploymentCapabilities::default(), + ) + .unwrap(); assert_eq!(candidates.len(), 2); let scans = |c: &PhysicalCandidate| { let mut timings: Vec<_> = c diff --git a/crates/asap-aware-mapping/src/stage3_plan_selection.rs b/crates/asap-aware-mapping/src/stage3_plan_selection.rs index 2e4163371..ffea9ebf5 100644 --- a/crates/asap-aware-mapping/src/stage3_plan_selection.rs +++ b/crates/asap-aware-mapping/src/stage3_plan_selection.rs @@ -58,7 +58,13 @@ //! `MatAtQuery` node's state is reused only within one evaluation, so it is //! held only while that evaluation runs: its bytes times the latency of the //! longest query reading it, once per evaluation, which is close to nothing. -//! Keeping it across evaluations comes later, charged by its retention. +//! When the deployment keeps query-time results +//! ([`DeploymentCapabilities::query_time_retention`]), a `MatAtQuery` +//! tumbling-pane chain whose every reader repeats at its pane width is kept +//! across evaluations (#509 Example 4's B3): each evaluation scans one pane +//! of raw data and builds the newest pane, and the `N − 1` older panes, +//! which complete the readers' look-back, are kept in memory every second +//! (all `N` at the peak). //! Memory a computation //! holds only while it runs is not charged under cost model 1: it counts in //! [`ResourceUsage::peak_memory_bytes`], which cost models 2 and 3 read. A @@ -107,7 +113,10 @@ //! //! Disk holds state materialized on `Medium::Disk` (Stage 2 places none //! there yet) and the raw data query-time scans read: per source, the -//! longest window a scan reads. +//! longest window a scan reads. A deployment that does not retain raw data +//! ([`DeploymentCapabilities::raw_data_retained`]) keeps no raw samples +//! after they arrive, so a candidate with a query-time scan is invalid, +//! with the window it would read as the reason. //! //! ## Latency //! @@ -161,6 +170,7 @@ use crate::accuracy::{OperandDomain, PropagationStats}; use crate::analytical_cost::{ estimate_operator, AnalyticalCostError, PhysicalOperator, ResourceEstimate, }; +use crate::deployment_capabilities::DeploymentCapabilities; use crate::per_computation_candidates::{ approximate_inputs, distinct_bound_source, operand_domain_statistic, operand_guarantee, operand_of, project_column_guarantee, root_guarantee, with_resized_summaries, @@ -168,7 +178,7 @@ use crate::per_computation_candidates::{ use crate::physical_operator_statistics::{ EdgeStatistics, OperatorStatistics, PartitionStatistics, UnaryEdgeStatistics, }; -use crate::stage2_physical_asap_optimization::PhysicalCandidate; +use crate::stage2_physical_asap_optimization::{builds_one_pane_per_evaluation, PhysicalCandidate}; use crate::PlanningModels; /// Groups assumed for every `by (...)` reduction, absent group-count evidence. @@ -644,14 +654,26 @@ pub fn select_plan( continue; } }; - let cost = - price(&candidate.dag, &recurrence, data, settings).map_err(|(node, error)| { - SelectionError::Cost { - candidate: candidate.id.clone(), - node, - error, - } - })?; + if let Some(reason) = raw_data_violation(&candidate.dag, &recurrence, models.deployment) { + rejected.push(Rejection { + id: candidate.id.clone(), + valid: false, + reason, + }); + continue; + } + let cost = price( + &candidate.dag, + &recurrence, + data, + settings, + models.deployment, + ) + .map_err(|(node, error)| SelectionError::Cost { + candidate: candidate.id.clone(), + node, + error, + })?; let violation = latency_violation(&cost, entries).or_else(|| match &settings.cost_model { CostModel::ClusterCapacity(limits) => capacity_violation(&cost.usage, limits), _ => None, @@ -1870,12 +1892,14 @@ fn computations( /// time ranges keep, times how often it is computed per second /// ([`computations`]); an ingestion-time node over one second of arriving /// rows. A materialized node's kept state is counted in byte-seconds per -/// second. +/// second. `deployment` says whether a query-time pane chain is kept across +/// evaluations ([`kept_pane_chains`]). fn price( dag: &PhysicalASAPDAG, recurrence: &[QueryRecurrence], data: &DataWorkload, settings: &PlanSelectionSettings, + deployment: &DeploymentCapabilities, ) -> Result { let first = dag.nodes.first().map(|n| n.id).unwrap_or_default(); let speed = settings.resource_speed; @@ -1892,6 +1916,7 @@ fn price( // The sources each node reads, filled children first. let mut sources_read_by: HashMap> = HashMap::new(); let nodes: HashMap<_, _> = dag.nodes.iter().map(|n| (n.id, n)).collect(); + let kept_chains = kept_pane_chains(dag, &nodes, recurrence, deployment); let multipliers = computations( dag, recurrence, @@ -1962,11 +1987,10 @@ fn price( NonASAPOp::Scan { .. } => { // At ingestion time, one second of arriving rows; at // query time, the longest window read above it. - let lookback = windows_read(dag, &nodes, node.id) - .into_iter() - .max() - .unwrap_or(DEFAULT_LOOKBACK_MS); - let lookback = if ingestion { 1_000 } else { lookback }; + let lookback = match ingestion { + true => 1_000, + false => query_time_lookback_ms(dag, &nodes, node.id, &kept_chains), + }; let out = edge(((rows_per_ms * lookback as f64).round() as u64).max(1)); let estimate = estimate_operator( PhysicalOperator::Scan, @@ -2239,6 +2263,22 @@ fn price( usage.peak_memory_bytes = estimate.peak_memory_bytes().saturating_mul(multiplier); (0, Medium::Memory) } + // A pane chain kept across evaluations holds its N − 1 older + // panes between evaluations, for the next ones, and all N + // while one runs. + (Materialization::MatAtQuery(medium), _) if kept_chains.contains(&node.id) => { + let panes = panes_of(node); + let between = state / panes * (panes - 1); + match medium { + Medium::Memory => { + usage.average_memory_bytes = between as f64; + usage.peak_memory_bytes = state; + } + Medium::Disk => usage.disk_bytes = state, + } + detail += &format!("; keeps {between} bytes across evaluations"); + (0, medium) + } (Materialization::MatAtQuery(medium), _) => { match medium { Medium::Memory => { @@ -2521,13 +2561,95 @@ fn query_reach( .collect() } +/// The raw data the query-time scan `scan` reads, in milliseconds back from +/// the evaluation: the longest window read above it ([`windows_read`]), or +/// [`DEFAULT_LOOKBACK_MS`] without one. +fn query_time_lookback_ms( + dag: &PhysicalASAPDAG, + nodes: &HashMap, + scan: PhysicalASAPNodeId, + kept_chains: &HashSet, +) -> u64 { + windows_read(dag, nodes, scan, kept_chains) + .into_iter() + .max() + .unwrap_or(DEFAULT_LOOKBACK_MS) +} + +/// The `MatAtQuery` tumbling-pane chains of `dag` kept across evaluations: +/// none unless the deployment keeps query-time results; otherwise each one +/// that builds one new pane per evaluation, every query reaching it +/// repeating at its pane width (`builds_one_pane_per_evaluation`). Its +/// retention is derived from those readers: the newest pane is built at +/// each evaluation, and the older ones, which complete the readers' +/// longest look-back, are kept from earlier evaluations. +fn kept_pane_chains( + dag: &PhysicalASAPDAG, + nodes: &HashMap, + recurrence: &[QueryRecurrence], + deployment: &DeploymentCapabilities, +) -> HashSet { + if !deployment.query_time_retention { + return HashSet::new(); + } + let reach = query_reach(dag, nodes); + dag.nodes + .iter() + .filter_map(|node| match (&node.payload, node.materialization) { + ( + Payload::ASAP(ASAPOp::PaneChain { pane_width_ms, .. }), + Materialization::MatAtQuery(_), + ) => { + let readers = reach + .iter() + .zip(recurrence) + .filter(|(reached, _)| reached.contains(&node.id)) + .map(|(_, recurrence)| recurrence); + builds_one_pane_per_evaluation(*pane_width_ms, readers).then_some(node.id) + } + _ => None, + }) + .collect() +} + +/// When the deployment does not retain raw data, the first query-time scan +/// of `dag` as a rejection reason. Raw samples are then not kept after they +/// arrive, so no window of them is there to scan at query time: only +/// ingestion-time work reads raw rows. +fn raw_data_violation( + dag: &PhysicalASAPDAG, + recurrence: &[QueryRecurrence], + deployment: &DeploymentCapabilities, +) -> Option { + if deployment.raw_data_retained { + return None; + } + let nodes: HashMap<_, _> = dag.nodes.iter().map(|n| (n.id, n)).collect(); + let kept_chains = kept_pane_chains(dag, &nodes, recurrence, deployment); + dag.nodes.iter().find_map(|node| match &node.payload { + Payload::NonASAP(NonASAPOp::Scan { source, .. }) + if node.output_state.timing != ExecutionTiming::IngestionTime => + { + Some(format!( + "scans {} ms of raw {} at query time, but the deployment does not retain raw data", + query_time_lookback_ms(dag, &nodes, node.id, &kept_chains), + source_name(source) + )) + } + _ => None, + }) +} + /// The windows, in milliseconds, read above the scan `scan` at query time: -/// each range `TimeRange` and `PaneChain` (its whole window) reached -/// through time shifts, which move a window without widening it. +/// each range `TimeRange` and `PaneChain` (its whole window, or one pane +/// for a chain in `kept_chains`, which builds only its newest pane per +/// evaluation) reached through time shifts, which move a window without +/// widening it. fn windows_read( dag: &PhysicalASAPDAG, nodes: &HashMap, scan: PhysicalASAPNodeId, + kept_chains: &HashSet, ) -> Vec { let mut windows = Vec::new(); let mut stack = vec![scan]; @@ -2541,6 +2663,11 @@ fn windows_read( Payload::NonASAP(NonASAPOp::TimeRange { range, .. }) => { windows.push(range.as_millis() as u64) } + Payload::ASAP(ASAPOp::PaneChain { pane_width_ms, .. }) + if kept_chains.contains(&edge.consumer) => + { + windows.push(*pane_width_ms) + } Payload::ASAP(ASAPOp::PaneChain { pane_width_ms, pane_count, @@ -2702,6 +2829,22 @@ fn ddsketch_bins(alpha: f64) -> u64 { #[cfg(test)] mod tests { use super::*; + + /// [`super::price`] under the default deployment capabilities. + fn price( + dag: &PhysicalASAPDAG, + recurrence: &[QueryRecurrence], + data: &DataWorkload, + settings: &PlanSelectionSettings, + ) -> Result { + super::price( + dag, + recurrence, + data, + settings, + &DeploymentCapabilities::default(), + ) + } use crate::accuracy::AccuracyEvidenceProvider; use crate::stage2_physical_asap_optimization::optimize_physical_asap; use crate::test_support::{entry, entry_with, lower_promql}; @@ -2828,9 +2971,15 @@ mod tests { .into_iter() .map(|(_, root)| root) .collect(); - let mut candidate = optimize_physical_asap("L", &roots, &[], &data()) - .unwrap() - .remove(0); + let mut candidate = optimize_physical_asap( + "L", + &roots, + &[], + &data(), + &DeploymentCapabilities::default(), + ) + .unwrap() + .remove(0); candidate.id = format!("P{}", index + 1); candidate }) @@ -3183,6 +3332,7 @@ mod tests { &[QueryRoot::Operator(outer), QueryRoot::Operator(inner)], &[], &data(), + &DeploymentCapabilities::default(), ) .unwrap() .remove(0); @@ -3257,9 +3407,15 @@ mod tests { .into_iter() .map(|(_, root)| root) .collect(); - let mut candidate = optimize_physical_asap("L", &roots, &[], &data()) - .unwrap() - .remove(0); + let mut candidate = optimize_physical_asap( + "L", + &roots, + &[], + &data(), + &DeploymentCapabilities::default(), + ) + .unwrap() + .remove(0); candidate.id = format!("P{}", index + 1); candidate }) @@ -4611,10 +4767,16 @@ mod tests { .into_iter() .map(|(_, root)| root) .collect(); - optimize_physical_asap("L1", &roots, &[], &data()) - .unwrap() - .remove(0) - .dag + optimize_physical_asap( + "L1", + &roots, + &[], + &data(), + &DeploymentCapabilities::default(), + ) + .unwrap() + .remove(0) + .dag } /// A time shift is free and does not change what the scan below it @@ -4778,10 +4940,16 @@ mod tests { .into_iter() .map(|query| { let root = lower_promql(query, AccuracyTarget::exact()); - let dag = optimize_physical_asap("L", &[QueryRoot::Operator(root)], &[], &data()) - .unwrap() - .remove(0) - .dag; + let dag = optimize_physical_asap( + "L", + &[QueryRoot::Operator(root)], + &[], + &data(), + &DeploymentCapabilities::default(), + ) + .unwrap() + .remove(0) + .dag; (query.to_string(), dag) }) .collect(); @@ -4812,10 +4980,16 @@ mod tests { fn kept_exact_state_is_charged_by_the_values_it_keeps() { let kept = |query: &str| { let root = lower_promql(query, AccuracyTarget::exact()); - let mut dag = optimize_physical_asap("L", &[QueryRoot::Operator(root)], &[], &data()) - .unwrap() - .remove(0) - .dag; + let mut dag = optimize_physical_asap( + "L", + &[QueryRoot::Operator(root)], + &[], + &data(), + &DeploymentCapabilities::default(), + ) + .unwrap() + .remove(0) + .dag; let aggregate = dag .nodes .iter_mut() @@ -4846,15 +5020,22 @@ mod tests { #[test] fn rates_follow_per_consumer_charging() { let root = lower_promql("sum by (job) (rate(m[1m]))", AccuracyTarget::exact()); - let one = optimize_physical_asap("L", &[QueryRoot::Operator(root.clone())], &[], &data()) - .unwrap() - .remove(0) - .dag; + let one = optimize_physical_asap( + "L", + &[QueryRoot::Operator(root.clone())], + &[], + &data(), + &DeploymentCapabilities::default(), + ) + .unwrap() + .remove(0) + .dag; let mut two = optimize_physical_asap( "L", &[QueryRoot::Operator(root.clone()), QueryRoot::Operator(root)], &[], &data(), + &DeploymentCapabilities::default(), ) .unwrap() .remove(0) @@ -4883,10 +5064,16 @@ mod tests { #[test] fn one_off_recurrence_is_spread_over_the_horizon() { let root = lower_promql("sum by (job) (rate(m[1m]))", AccuracyTarget::exact()); - let dag = optimize_physical_asap("L", &[QueryRoot::Operator(root)], &[], &data()) - .unwrap() - .remove(0) - .dag; + let dag = optimize_physical_asap( + "L", + &[QueryRoot::Operator(root)], + &[], + &data(), + &DeploymentCapabilities::default(), + ) + .unwrap() + .remove(0) + .dag; let horizon = PlanSelectionSettings::default().non_repeating_query_horizon_seconds; assert_eq!(horizon, 3_600.0); let per_evaluation = priced(&dag, &[repeating(1_000)]).cost_per_second.unwrap(); @@ -4985,6 +5172,18 @@ mod tests { /// `rows_per_s`: rebuilt at query time (B2), then maintained at /// ingestion time (B1). fn pane_candidates(rows_per_s: f64) -> Vec { + let candidates = pane_candidates_for(rows_per_s, &DeploymentCapabilities::default()); + assert_eq!(candidates.len(), 2); + candidates + } + + /// [`pane_candidates`] for a deployment with `deployment`'s + /// capabilities: B2, B1, then, when it keeps query-time results, B3 + /// (built at query time and kept across evaluations). + fn pane_candidates_for( + rows_per_s: f64, + deployment: &DeploymentCapabilities, + ) -> Vec { let root = lower_promql("quantile_over_time(0.99, m[5m])", p99_target()); let inventory = crate::per_computation_candidates::generate_per_computation_candidates( vec![(0, QueryRoot::Operator(root))], @@ -5016,15 +5215,149 @@ mod tests { .unwrap() .roots .unwrap(); - let mut candidates = - optimize_physical_asap("L1", &roots, &[every_minute()], &arriving(rows_per_s)).unwrap(); + let mut candidates = optimize_physical_asap( + "L1", + &roots, + &[every_minute()], + &arriving(rows_per_s), + deployment, + ) + .unwrap(); for (k, c) in candidates.iter_mut().enumerate() { c.id = format!("P1-m{k}"); } - assert_eq!(candidates.len(), 2); candidates } + /// The raw samples the scan `scan` reads per computation, from its + /// priced detail (`scan samples; ...`). + fn scanned_samples(cost: &CandidateCost, scan: PhysicalASAPNodeId) -> u64 { + let detail = &cost.per_node[&scan].detail; + detail + .strip_prefix("scan ") + .and_then(|rest| rest.split(' ').next()) + .and_then(|rows| rows.parse().ok()) + .unwrap_or_else(|| panic!("not a scan: {detail}")) + } + + /// A deployment that keeps query-time results across evaluations. + fn keeping_query_time_results() -> DeploymentCapabilities { + DeploymentCapabilities { + query_time_retention: true, + ..DeploymentCapabilities::default() + } + } + + /// `query_time_retention` gates keeping a query-time result across + /// evaluations, and only that. Without it, Stage 2 offers the 1-min + /// panes of a query repeating every minute rebuilt (B2) or maintained + /// (B1); with it, also built at query time and kept (B3, `MatAtQuery`). + /// B3 is computed once per evaluation, reads one pane (1 min) of raw + /// data, keeps that much raw data on disk, and holds its 4 older panes + /// in memory between evaluations. The same DAG priced for a deployment + /// that keeps no query-time result is reused only within one + /// evaluation: the scan reads the 5-min window and nothing is kept. + #[test] + fn query_time_retention_keeps_a_pane_chain_across_evaluations() { + let candidates = pane_candidates_for(10_000.0, &keeping_query_time_results()); + assert_eq!(candidates.len(), 3); + let b2 = &candidates[0].dag; + let b3 = &candidates[2].dag; + let chain = node_of(b3, is_chain); + assert_eq!( + b3.nodes + .iter() + .find(|n| n.id == chain) + .unwrap() + .materialization, + Materialization::MatAtQuery(Medium::Memory) + ); + let recurrence = [repeating(MINUTE)]; + let settings = PlanSelectionSettings::default(); + let kept = super::price( + b3, + &recurrence, + &arriving(10_000.0), + &settings, + &keeping_query_time_results(), + ) + .unwrap(); + let rebuilt = price(b2, &recurrence, &arriving(10_000.0), &settings).unwrap(); + let scan = node_of(b3, is_scan); + assert_eq!(scanned_samples(&kept, scan), 10_000 * 60); + assert_eq!( + scanned_samples(&rebuilt, node_of(b2, is_scan)), + 10_000 * 300 + ); + assert_eq!(kept.usage.disk_bytes * 5, rebuilt.usage.disk_bytes); + let Payload::ASAP(ASAPOp::PaneChain { family, .. }) = + &b3.nodes.iter().find(|n| n.id == chain).unwrap().payload + else { + unreachable!() + }; + let (_, state_bytes) = summary_shape(family); + let chain_cost = &kept.per_node[&chain]; + assert!(close(chain_cost.per_second, 1.0 / 60.0), "{chain_cost:?}"); + let groups = chain_cost.usage.peak_memory_bytes / (5 * state_bytes); + let between = 4 * groups * state_bytes; + assert_eq!(chain_cost.usage.average_memory_bytes, between as f64); + assert!(chain_cost + .detail + .contains(&format!("keeps {between} bytes across evaluations"))); + let within_one = price(b3, &recurrence, &arriving(10_000.0), &settings).unwrap(); + assert_eq!(scanned_samples(&within_one, scan), 10_000 * 300); + assert!(!within_one.per_node[&chain] + .detail + .contains("across evaluations")); + assert!(within_one.per_node[&chain].usage.average_memory_bytes < between as f64); + } + + /// Without raw data retained, a candidate that scans raw data at query + /// time is invalid, with the window it would read: B2 (5 min) and B3 + /// (one 1-min pane) are rejected, and B1, which reads raw rows only at + /// ingestion time, is selected. With raw data retained, all three are + /// valid. + #[test] + fn without_raw_data_only_ingestion_time_scans_are_valid() { + let candidates = pane_candidates_for(10_000.0, &keeping_query_time_results()); + let select = |deployment: &DeploymentCapabilities| { + select_plan( + &candidates, + &[every_minute()], + &arriving(10_000.0), + PlanningModels::builtin().with_deployment(deployment), + ) + .unwrap() + }; + let retained = select(&keeping_query_time_results()); + assert!(retained.rejected.iter().all(|r| r.valid), "{retained:?}"); + let dropped = select(&DeploymentCapabilities { + raw_data_retained: false, + ..keeping_query_time_results() + }); + assert_eq!(dropped.selected, candidates[1].id); + let reasons: BTreeMap<&str, (bool, &str)> = dropped + .rejected + .iter() + .map(|r| (r.id.as_str(), (r.valid, r.reason.as_str()))) + .collect(); + let unretained = "at query time, but the deployment does not retain raw data"; + assert_eq!( + reasons[candidates[0].id.as_str()], + ( + false, + format!("scans 300000 ms of raw m {unretained}").as_str() + ) + ); + assert_eq!( + reasons[candidates[2].id.as_str()], + ( + false, + format!("scans 60000 ms of raw m {unretained}").as_str() + ) + ); + } + fn is_chain(payload: &Payload) -> bool { matches!(payload, Payload::ASAP(ASAPOp::PaneChain { .. })) } @@ -5286,10 +5619,16 @@ mod tests { .iter() .map(|q| QueryRoot::Operator(lower_promql(q, AccuracyTarget::exact()))) .collect(); - let dag = optimize_physical_asap("L", &roots, &[], &data()) - .unwrap() - .remove(0) - .dag; + let dag = optimize_physical_asap( + "L", + &roots, + &[], + &data(), + &DeploymentCapabilities::default(), + ) + .unwrap() + .remove(0) + .dag; priced(&dag, &recurrences(queries.len())).usage.disk_bytes }; let five_minutes = disk(&["sum_over_time(m[5m])"]); diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index 30d8c86de..454a18ad8 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -68,6 +68,18 @@ // `pareto-front`). Cost models 1 and 2 use the illustrative unit costs. // Each shown candidate's `usage` lists what it uses; `cost_per_second` is null under // cost model 3, and `front` lists the Pareto front under `pareto-front`. +// `--deployment ''` declares what the deployment's executor can run +// and keep (`DeploymentCapabilities`): `{"supported_summaries": [{"family": +// "Kll", "readouts": ["Quantile"]}, ...], "raw_data_retained": true, +// "query_time_retention": false}`. Every field is optional: by default every +// sketch family and readout the planner can build is supported +// (`supported_summaries` null), raw data is retained, and no query-time +// result is kept across evaluations. Pass 1 offers no summary the +// deployment does not support (exact computations are always supported); +// without raw data, a candidate that scans raw data at query time is +// invalid; with `query_time_retention`, a tumbling-pane chain whose readers +// repeat at its pane width may be built at query time and kept across +// evaluations. The document's top-level `deployment` shows the inputs used. // `--sample-interval-ms` is the sample interval declared for every metric the // PromQL queries read (default 15000); the data declares no other facts. // `--table ''` (repeatable) declares a table the SQL queries read: `{"name": ..., @@ -82,6 +94,7 @@ use asap_aware_mapping::stage3_plan_selection::{ ClusterLimits, ClusterObjective, CostModel, PlanSelectionSettings, Rejection, ResourceUnitCost, ResourceUsage, Selection, SelectionError, }; +use asap_aware_mapping::DeploymentCapabilities; use asap_devtools::candidate_labels::candidate_labels; use asap_devtools::planner_layering_examples::{ example1, example2, pattern_a, pattern_b, PatternARecurrence, SHARED_ROWS_PER_SECOND, @@ -107,7 +120,7 @@ const USAGE: &str = [--name ]... [--table ...] [--epsilon --delta --metric ] [--sample-interval-ms ])) \ [--cost-model usage-over-time|provisioned-capacity|cluster-capacity [--cluster-memory-bytes \ --cluster-disk-bytes [--cluster-cpu-cores ] --cluster-objective lowest-latency|least-memory|pareto-front]] \ -[--max-combinations ] [--max-displayed ] --out "; +[--deployment ] [--max-combinations ] [--max-displayed ] --out "; /// The default `--max-combinations`: the most Stage 1 candidates (Pass 2's /// included) a run enumerates. @@ -193,6 +206,7 @@ fn run(args: Vec) -> Result<(), Failure> { let mut cost_model: Option = None; let (mut cluster_memory, mut cluster_disk, mut cluster_cores) = (None, None, None); let mut cluster_objective: Option = None; + let mut deployment = DeploymentCapabilities::default(); let mut out = None; let mut args = args.into_iter(); while let Some(flag) = args.next() { @@ -241,6 +255,10 @@ fn run(args: Vec) -> Result<(), Failure> { cluster_cores = Some(value()?.parse::().map_err(|e| format!("{e}"))?) } "--cluster-objective" => cluster_objective = Some(value()?), + "--deployment" => { + deployment = + serde_json::from_str(&value()?).map_err(|e| format!("--deployment: {e}"))? + } "--out" => out = Some(value()?), other => return Err(format!("unknown argument {other}").into()), } @@ -289,7 +307,7 @@ fn run(args: Vec) -> Result<(), Failure> { cost_model, ..PlanSelectionSettings::default() }; - let document = match (example.as_deref(), queries.is_empty()) { + let mut document = match (example.as_deref(), queries.is_empty()) { (Some(example), true) => { let (workload, catalog) = match example { "planner-layering-1" => (example1(), SqlCatalog::new()), @@ -318,6 +336,7 @@ fn run(args: Vec) -> Result<(), Failure> { &data, Some(max_combinations), &plan_selection, + &deployment, ); document_of(workload_queries(&workload), &roots, result, max_displayed)? } @@ -385,11 +404,14 @@ fn run(args: Vec) -> Result<(), Failure> { &data, Some(max_combinations), &plan_selection, + &deployment, ); document_of(described, &roots, result, max_displayed)? } _ => return Err("give exactly one of --example or --promql/--sql queries".into()), }; + // Read-only: the deployment inputs the run planned with. + document["deployment"] = serde_json::to_value(&deployment).map_err(|e| e.to_string())?; let text = serde_json::to_string_pretty(&document).map_err(|e| e.to_string())? + "\n"; std::fs::write(&out, text).map_err(|e| format!("{out}: {e}").into()) } diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index 15cb4fda6..bbf428ddb 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -10,7 +10,7 @@ use asap_aware_mapping::stage2_physical_asap_optimization::{ physical_candidate_count, PhysicalCandidate, }; use asap_aware_mapping::stage3_plan_selection::{PlanSelectionSettings, Selection, SelectionError}; -use asap_aware_mapping::PlanningModels; +use asap_aware_mapping::{DeploymentCapabilities, PlanningModels}; use asap_types::ir::schema_support::with_promql_series_identity; use asap_types::ir::QueryRoot; use asap_types::workload::{DataWorkload, PlanningWorkload, QueryWorkloadEntry}; @@ -147,6 +147,7 @@ pub fn run_planning_stages( &data, max_combinations, &PlanSelectionSettings::default(), + &DeploymentCapabilities::default(), ) } @@ -154,17 +155,21 @@ pub fn run_planning_stages( /// languages: query `i` is `roots[i]`, with workload entry `entries[i]` /// (its accuracy target, latency bound, recurrence and predictability). /// Plan selection compares candidates under `plan_selection`'s cost model. +/// `deployment` is what the executor can run and keep: Pass 1 offers only +/// the summaries it supports, and Stages 2 and 3 read its retention. pub fn run_planning_stages_over( roots: Vec, entries: &[QueryWorkloadEntry], data: &DataWorkload, max_combinations: Option, plan_selection: &PlanSelectionSettings, + deployment: &DeploymentCapabilities, ) -> Result { validate(&roots)?; let per_computation = generate_per_computation_candidates(roots.iter().cloned().enumerate().collect()) - .map_err(|e| PlanningStagesError::Pass1(e.to_string()))?; + .map_err(|e| PlanningStagesError::Pass1(e.to_string()))? + .without_unsupported_summaries(deployment); // Count Stage 1 candidates as they are enumerated and stop as soon as // the count passes the limit; never truncate. let recurrence: Vec<_> = entries.iter().map(|e| e.recurrence.clone()).collect(); @@ -180,7 +185,7 @@ pub fn run_planning_stages_over( if let Some(limit) = max_combinations { let mut physical = 0usize; for roots in stage1.iter().filter_map(|c| c.roots.as_ref().ok()) { - physical += physical_candidate_count(roots, entries, data).unwrap_or(0); + physical += physical_candidate_count(roots, entries, data, deployment).unwrap_or(0); if physical > limit { return Err(PlanningStagesError::TooManyPhysicalCandidates { limit }); } @@ -191,7 +196,9 @@ pub fn run_planning_stages_over( stage1, entries, data, - PlanningModels::builtin().with_plan_selection(plan_selection), + PlanningModels::builtin() + .with_plan_selection(plan_selection) + .with_deployment(deployment), ); for candidate in &run.stage1_logical_asap_optimization { if let Ok(roots) = &candidate.roots { diff --git a/crates/devtools/tests/stage_pipeline.rs b/crates/devtools/tests/stage_pipeline.rs index 8a9484a89..6686dffa7 100644 --- a/crates/devtools/tests/stage_pipeline.rs +++ b/crates/devtools/tests/stage_pipeline.rs @@ -710,6 +710,82 @@ fn cost_model_flag_chooses_how_candidates_are_compared() { assert!(cluster["costs"]["P3-m1"]["cost_per_second"].is_null()); } +/// `--deployment` declares the executor's capabilities, which the +/// document's `deployment` shows (defaults filled in; without the flag, +/// every summary supported, raw data retained, no query-time retention). +/// For Pattern B with KLL only and query-time retention: no DDSketch +/// candidate is offered, and the KLL panes gain B3, built at query time and +/// kept (`P3-m2`, 8,884.53 cost/s); the whole-window KLL (`P2`) is still +/// selected. Without raw data, every query-time scan is invalid and B1 +/// (`P3-m1`) is selected. A deployment with an unknown field is refused. +#[test] +fn deployment_flag_sets_the_capabilities_and_shows_them() { + let default = generate_example("planner-layering-3b", &[]); + assert_eq!( + default["deployment"], + serde_json::json!({ + "supported_summaries": null, + "raw_data_retained": true, + "query_time_retention": false, + }) + ); + let kll_kept = generate_example( + "planner-layering-3b", + &[ + "--deployment", + r#"{"supported_summaries": [{"family": "Kll", "readouts": ["Quantile"]}], + "query_time_retention": true}"#, + ], + ); + assert_eq!( + kll_kept["deployment"], + serde_json::json!({ + "supported_summaries": [{ "family": "Kll", "readouts": ["Quantile"] }], + "raw_data_retained": true, + "query_time_retention": true, + }) + ); + let physical: Vec<&str> = kll_kept["stage2_physical_asap_optimization"]["candidates"] + .as_array() + .unwrap() + .iter() + .map(|c| c["id"].as_str().unwrap()) + .collect(); + assert_eq!(physical, ["P1", "P2", "P3", "P3-m1", "P3-m2"]); + let selection = &kll_kept["stage3_plan_selection"]; + assert_eq!(selection["selected"], "P2"); + assert_eq!(selection["costs"]["P3-m2"]["cost_per_second"], 8884.53); + + let no_raw = generate_example( + "planner-layering-3b", + &["--deployment", r#"{"raw_data_retained": false}"#], + ); + let selection = &no_raw["stage3_plan_selection"]; + assert_eq!(selection["selected"], "P3-m1"); + let p1 = selection["rejected"] + .as_array() + .unwrap() + .iter() + .find(|r| r["id"] == "P1") + .unwrap(); + assert_eq!( + p1["reason"], + "scans 300000 ms of raw latency_ms at query time, but the deployment does not retain raw data" + ); + + let (status, stderr) = failure(&[ + "--example", + "planner-layering-3b", + "--deployment", + r#"{"raw_data": false}"#, + ]); + assert_eq!(status, Some(2)); + assert!( + stderr.contains("--deployment: unknown field `raw_data`"), + "{stderr}" + ); +} + /// Cluster limits need cost model 3, and cost model 3 needs them. #[test] fn cluster_flags_and_cost_model_3_go_together() { diff --git a/crates/integration-tests/tests/deployment_capabilities.rs b/crates/integration-tests/tests/deployment_capabilities.rs new file mode 100644 index 000000000..6f02b5f4b --- /dev/null +++ b/crates/integration-tests/tests/deployment_capabilities.rs @@ -0,0 +1,131 @@ +//! The deployment's capabilities (#509's execution-capability input) over +//! the planner-layering examples: the summaries the executor supports +//! restrict Stage 1 (Pass 1, and so Pass 2). + +mod planner_layering_common; + +use asap_aware_mapping::stage3_plan_selection::PlanSelectionSettings; +use asap_aware_mapping::{DeploymentCapabilities, SupportedSummary}; +use asap_devtools::planning_stages::lower_sql_roots; +use asap_types::ir::flat::FlatDag; +use asap_types::post_asap::{SketchAlgorithm, StatisticKind}; +use planner_layering_common::*; + +/// A deployment whose executor supports only `supported`: each family with +/// its readouts. +fn supporting(supported: &[(SketchAlgorithm, &[StatisticKind])]) -> DeploymentCapabilities { + DeploymentCapabilities { + supported_summaries: Some( + supported + .iter() + .map(|(family, readouts)| SupportedSummary { + family: family.clone(), + readouts: readouts.to_vec(), + }) + .collect(), + ), + ..DeploymentCapabilities::default() + } +} + +/// Every (family, readout) a logical candidate's DAG uses: each sketch +/// build with each statistic estimated from it. +fn summaries_used(dag: &FlatDag) -> Vec<(SketchAlgorithm, StatisticKind)> { + sketch_builds(dag) + .into_iter() + .flat_map(|(build, family, _)| { + estimates_of(dag, build) + .into_iter() + .map(move |(_, statistic)| (family.clone(), statistic.kind())) + }) + .collect() +} + +/// Example 2 (three SQL flow statistics) planned for `deployment`. +fn example2_for(deployment: &DeploymentCapabilities) -> Run { + let (workload, catalog) = example2(); + let roots = lower_sql_roots(&workload, &catalog).expect("example 2 lowers"); + run_stages_with( + &workload, + roots, + &PlanSelectionSettings::default(), + deployment, + ) +} + +/// Restricting the supported summaries removes exactly the Stage 1 +/// candidates (Pass 1's and Pass 2's) that build an unsupported family or +/// read an unsupported readout from one, and keeps the others in order: +/// +/// * Example 1 without Count-Min heaps: the top-k keeps its CountSketch +/// heap; +/// * Example 2 with UnivMon for distinct counts and L2 only (no entropy), +/// and HLL: KMV and UnivMon's entropy go, so the entropy is exact, and +/// the shared UnivMon builds for the distinct count and L2 stay (Pass +/// 2's summary-capability rule); +/// * Example 3's Pattern B with KLL only: DDSketch goes, whole-window and +/// tumbling panes alike (Pass 2's window-composition rule). +#[test] +fn restricting_summaries_removes_exactly_the_candidates_that_use_them() { + let all = DeploymentCapabilities::default(); + let heap = supporting(&[(SketchAlgorithm::CountSketchWithHeap, &[StatisticKind::TopK])]); + assert_removes_exactly( + "example 1", + &run_promql_for(&example1(), &all), + &run_promql_for(&example1(), &heap), + &heap, + ); + let distinct_and_l2 = supporting(&[ + ( + SketchAlgorithm::UnivMon, + &[StatisticKind::Distinct, StatisticKind::FrequencyL2], + ), + (SketchAlgorithm::Hll, &[StatisticKind::Distinct]), + ]); + assert_removes_exactly( + "example 2", + &example2_for(&all), + &example2_for(&distinct_and_l2), + &distinct_and_l2, + ); + let kll = supporting(&[(SketchAlgorithm::Kll, &[StatisticKind::Quantile])]); + assert_removes_exactly( + "pattern B", + &run_promql_for(&pattern_b(), &all), + &run_promql_for(&pattern_b(), &kll), + &kll, + ); +} + +/// `restricted` (planned for `deployment`) has exactly `unrestricted`'s +/// Stage 1 candidates whose every summary `deployment` supports, in order; +/// some are removed, a supported summary stays, and the cheapest valid +/// candidate is selected. +fn assert_removes_exactly( + name: &str, + unrestricted: &Run, + restricted: &Run, + deployment: &DeploymentCapabilities, +) { + let expected: Vec<&FlatDag> = unrestricted + .logical + .iter() + .map(|c| &c.dag) + .filter(|dag| { + summaries_used(dag) + .iter() + .all(|(family, readout)| deployment.supports(family, *readout)) + }) + .collect(); + let found: Vec<&FlatDag> = restricted.logical.iter().map(|c| &c.dag).collect(); + assert!( + expected.len() < unrestricted.logical.len(), + "{name}: the restriction removes some candidate" + ); + assert!( + found.iter().any(|dag| !summaries_used(dag).is_empty()), + "{name}: a supported summary stays" + ); + assert_eq!(found, expected, "{name}"); + assert_selects_cheapest_valid(restricted); +} diff --git a/crates/integration-tests/tests/planner_layering_common/mod.rs b/crates/integration-tests/tests/planner_layering_common/mod.rs index e176d4a48..856664d1f 100644 --- a/crates/integration-tests/tests/planner_layering_common/mod.rs +++ b/crates/integration-tests/tests/planner_layering_common/mod.rs @@ -18,6 +18,7 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use asap_aware_mapping::stage2_physical_asap_optimization::PhysicalCandidate; use asap_aware_mapping::stage3_plan_selection::PlanSelectionSettings; use asap_aware_mapping::stage3_plan_selection::Selection; +use asap_aware_mapping::DeploymentCapabilities; use asap_devtools::planning_stages::{lower_promql_roots, run_planning_stages_over}; use asap_types::ir::flat::{flatten, FlatDag, NodeId}; use asap_types::ir::physical_export::PhysicalASAPDAG; @@ -42,6 +43,17 @@ pub fn run_promql_under(workload: &PlanningWorkload, settings: &PlanSelectionSet run_stages_under(workload, lower_promql(workload), settings) } +/// Lower and plan a PromQL workload for a deployment with `deployment`'s +/// capabilities, under the default cost model. +pub fn run_promql_for(workload: &PlanningWorkload, deployment: &DeploymentCapabilities) -> Run { + run_stages_with( + workload, + lower_promql(workload), + &PlanSelectionSettings::default(), + deployment, + ) +} + /// Whether `form` answers a query window by merging several summaries. pub fn needs_merge(form: WindowForm, query_window_ms: u64) -> bool { match form { @@ -182,10 +194,26 @@ pub fn run_stages_under( workload: &PlanningWorkload, roots: Vec, settings: &PlanSelectionSettings, +) -> Run { + run_stages_with( + workload, + roots, + settings, + &DeploymentCapabilities::default(), + ) +} + +/// [`run_stages`], selecting under `settings` for a deployment with +/// `deployment`'s capabilities. +pub fn run_stages_with( + workload: &PlanningWorkload, + roots: Vec, + settings: &PlanSelectionSettings, + deployment: &DeploymentCapabilities, ) -> Run { let entries: Vec<_> = workload.query_workload.entries().collect(); let data = workload.data_workload.clone().unwrap_or_default(); - let run = run_planning_stages_over(roots, &entries, &data, None, settings) + let run = run_planning_stages_over(roots, &entries, &data, None, settings, deployment) .expect("planning stages run"); let logical_candidates = run .stage1_logical_asap_optimization diff --git a/crates/integration-tests/tests/planner_layering_example4.rs b/crates/integration-tests/tests/planner_layering_example4.rs index 2bb9cde7d..c9851b5fd 100644 --- a/crates/integration-tests/tests/planner_layering_example4.rs +++ b/crates/integration-tests/tests/planner_layering_example4.rs @@ -18,6 +18,7 @@ use std::collections::{BTreeMap, BTreeSet}; use asap_aware_mapping::stage3_plan_selection::{ ClusterLimits, ClusterObjective, CostModel, PlanSelectionSettings, ResourceUnitCost, }; +use asap_aware_mapping::DeploymentCapabilities; use asap_types::ir::flat::NodeId; use asap_types::post_asap::SketchAlgorithm; use asap_types::workload::{DataArrival, PlanningWorkload}; @@ -268,13 +269,20 @@ fn stage3_plan_selection_a_monthly_amortizes_ingestion_time_maintenance() { // ── Pattern B ──────────────────────────────────────────────────────────── +/// A deployment that keeps query-time results across evaluations, which B3 +/// needs; the default deployment keeps none. +fn keeping_query_time_results() -> DeploymentCapabilities { + DeploymentCapabilities { + query_time_retention: true, + ..DeploymentCapabilities::default() + } +} + /// The 1-min tumbling KLL candidate yields B1 (ingestion time), B2 (not materialized) and B3 (query time, kept). #[test] -#[ignore = "needs B3: panes built at query time and kept across evaluations (G9, #625)"] fn stage2_physical_asap_optimization_b_tumbling_kll_has_three_materialization_options() { - let found: BTreeSet<_> = options_of(&run_promql(&pattern_b()), tumbling(), 1) - .into_keys() - .collect(); + let run = run_promql_for(&pattern_b(), &keeping_query_time_results()); + let found: BTreeSet<_> = options_of(&run, tumbling(), 1).into_keys().collect(); assert_eq!( found, BTreeSet::from([MatAtIngestion, MatAtQuery, NotMaterialized]) @@ -305,9 +313,10 @@ fn stage2_physical_asap_optimization_b_b1_builds_at_ingestion_and_merges_at_quer } } -/// Implementer test (until B3, G9): the tumbling KLL candidate yields B1 -/// (maintained at ingestion time) and B2 (rebuilt at every evaluation); -/// both are priced, and B2's evaluation stays within the 200 ms bound. +/// Implementer test: for the default deployment, which keeps no query-time +/// result across evaluations, the tumbling KLL candidate yields B1 +/// (maintained at ingestion time) and B2 (rebuilt at every evaluation), not +/// B3; both are priced, and B2's evaluation stays within the 200 ms bound. #[test] fn stage2_physical_asap_optimization_b_tumbling_kll_has_b1_and_b2() { let run = run_promql(&pattern_b()); @@ -324,16 +333,22 @@ fn stage2_physical_asap_optimization_b_tumbling_kll_has_b1_and_b2() { } /// B3 runs nothing at ingestion time; it keeps the tumbling KLLs it builds at query time. +/// Implementer addition: kept for the readers' 5-min look-back, it is +/// priced as building one 1-min pane per evaluation, so it reads a fifth of +/// B2's raw data and keeps a fifth of it on disk. #[test] -#[ignore = "needs B3: panes built at query time and kept across evaluations (G9, #625)"] fn stage2_physical_asap_optimization_b_b3_keeps_query_time_windows() { - let run = run_promql(&pattern_b()); - let (id, _) = options_of(&run, tumbling(), 1)[&MatAtQuery].clone(); + let run = run_promql_for(&pattern_b(), &keeping_query_time_results()); + let options = options_of(&run, tumbling(), 1); + let (id, build) = options[&MatAtQuery].clone(); let p = run.physical(&id); assert!( p.dag().nodes.iter().all(|n| !runs_at_ingestion(p, n.id)), "{id}" ); + assert_eq!(retention_ms(p, build), Some(PATTERN_B_WINDOW_MS)); + let disk = |id: &str| run.selection.costs[id].usage.disk_bytes; + assert_eq!(disk(&id) * 5, disk(&options[&NotMaterialized].0)); } /// The sliding-window KLL is kept from ingestion time or from query time; not materializing it is the no-window plan. @@ -348,9 +363,9 @@ fn stage2_physical_asap_optimization_b_sliding_kll_has_two_materialization_optio /// B2 rebuilds all five tumbling KLLs at every evaluation, so it costs at least B1 and B3. #[test] -#[ignore = "needs B3: panes built at query time and kept across evaluations (G9, #625)"] +#[ignore = "the built-in model contradicts it: with query-time retention, B1 keeps 6 panes of 1M per-series KLLs for 13,326.5 cost/s and B3 keeps 4 for 8,884.5, while B2 rebuilds them for 2.183; needs the model-relative rewrite (Q49)"] fn stage3_plan_selection_b_rebuilding_every_window_costs_most() { - let run = run_promql(&pattern_b()); + let run = run_promql_for(&pattern_b(), &keeping_query_time_results()); let options = options_of(&run, tumbling(), 1); let b2 = cost(&run, &options[&NotMaterialized].0); for m in [MatAtIngestion, MatAtQuery] { @@ -360,7 +375,7 @@ fn stage3_plan_selection_b_rebuilding_every_window_costs_most() { /// Repeating over arriving data, the built-in models pick B1 among the tumbling options. #[test] -#[ignore = "the built-in model contradicts it: B1 keeps 6 panes of 1M per-series KLLs (107 GB) for 13,326.5 cost/s, B2 rebuilds them for 2.183; needs B3 (G9) and the model-relative rewrite (Q49, G9)"] +#[ignore = "the built-in model contradicts it: B1 keeps 6 panes of 1M per-series KLLs (107 GB) for 13,326.5 cost/s, B2 rebuilds them for 2.183; needs the model-relative rewrite (Q49)"] fn stage3_plan_selection_b_prefers_ingestion_time_tumbling_windows() { let run = run_promql(&pattern_b()); let options = options_of(&run, tumbling(), 1); diff --git a/crates/integration-tests/tests/stage2_physical_asap_optimization_sql.rs b/crates/integration-tests/tests/stage2_physical_asap_optimization_sql.rs index 0baa48678..bc07e3c75 100644 --- a/crates/integration-tests/tests/stage2_physical_asap_optimization_sql.rs +++ b/crates/integration-tests/tests/stage2_physical_asap_optimization_sql.rs @@ -117,6 +117,7 @@ async fn check_every_choice( &roots, &[], &asap_types::workload::DataWorkload::default(), + &asap_aware_mapping::DeploymentCapabilities::default(), ) .unwrap_or_else(|e| panic!("{sql}, choice {choice:?} pruned: {e}")) .remove(0); diff --git a/crates/types/src/post_asap/family_capability.rs b/crates/types/src/post_asap/family_capability.rs index 7c352d1f4..6ef7ea170 100644 --- a/crates/types/src/post_asap/family_capability.rs +++ b/crates/types/src/post_asap/family_capability.rs @@ -26,8 +26,10 @@ pub struct FamilyCapability { pub readouts: &'static [(StatisticKind, MergedError)], } -/// The kind of a `SummaryEstimate` statistic, without its arguments. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// The kind of a `SummaryEstimate` statistic, without its arguments: the +/// readout vocabulary of this table and of a deployment's supported +/// summaries. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub enum StatisticKind { /// The total of the updates: a bare `PointCount` (no item value). Total, diff --git a/tools/dag-viewer/examples/planner-layering-example1.json b/tools/dag-viewer/examples/planner-layering-example1.json index 329bd65e6..88811b081 100644 --- a/tools/dag-viewer/examples/planner-layering-example1.json +++ b/tools/dag-viewer/examples/planner-layering-example1.json @@ -1,4 +1,9 @@ { + "deployment": { + "query_time_retention": false, + "raw_data_retained": true, + "supported_summaries": null + }, "format": "asap-stage-pipeline/v1", "stage0_query_language_frontends": { "dag": { From 8fc1f06e8f37c270ab5a73c491150f3e9bd73434 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 16 Oct 2026 17:20:50 +0000 Subject: [PATCH 2/2] fix(asap-aware-mapping)!: offer both query-time options; check phases; surface Pass 1 pruning - Materialization gains MatAtQueryKeptAcrossEvaluations. With query_time_results_retained, Stage 2 offers it beside MatAtQuery (shared within one evaluation) and Stage 3 prices each, so a chain two panels read is no longer always priced as kept. - builds_one_pane_per_evaluation requires equal known evaluation phases. - Stored data with raw_data_retained false is bad input (DeploymentCapabilities::validate); the raw-data rejection lists every query-time scan. - Rename query_time_retention to query_time_results_retained and PlanningModels::deployment to deployment_capabilities (with with_deployment_capabilities, --deployment-capabilities and the document key). - stage_pipeline shows Pass 1 pruned candidates per target (pass1_pruned); FusedComputation records its pruned candidates. - Remove NodeCost.usage, NodeCost.per_second and NodeUsage; price() accumulates the candidate's ResourceUsage directly. The node map and kept chains are built once per candidate. - The kept arm charges N - 1 panes on disk; drop StatisticKind's Hash. --- .../src/deployment_capabilities.rs | 68 +- crates/asap-aware-mapping/src/lib.rs | 4 +- crates/asap-aware-mapping/src/pass/mod.rs | 25 +- .../src/pass/stage1_to_stage3.rs | 15 +- .../src/per_computation_candidates.rs | 56 +- .../src/stage2_physical_asap_optimization.rs | 105 ++- .../src/stage3_plan_selection.rs | 607 +++++++++++------- crates/devtools/src/bin/stage_pipeline.rs | 91 ++- crates/devtools/src/candidate_labels.rs | 29 +- crates/devtools/src/planning_stages.rs | 2 +- crates/devtools/tests/stage_pipeline.rs | 48 +- .../tests/deployment_capabilities.rs | 91 +++ .../tests/planner_layering_common/mod.rs | 11 +- .../tests/planner_layering_example4.rs | 2 +- crates/types/src/ir/timing.rs | 8 +- .../src/post_asap/execution_data_state.rs | 14 +- .../types/src/post_asap/family_capability.rs | 2 +- .../examples/planner-layering-example1.json | 7 +- 18 files changed, 835 insertions(+), 350 deletions(-) diff --git a/crates/asap-aware-mapping/src/deployment_capabilities.rs b/crates/asap-aware-mapping/src/deployment_capabilities.rs index 1cce407b1..838951f13 100644 --- a/crates/asap-aware-mapping/src/deployment_capabilities.rs +++ b/crates/asap-aware-mapping/src/deployment_capabilities.rs @@ -14,26 +14,32 @@ //! * whether raw samples are retained. If they are, Stage 3 counts them on //! disk once per source, at the longest window a query-time scan of it //! reads. If they are not, raw samples are not kept after they arrive, so -//! a candidate that scans raw data at query time is invalid: only -//! ingestion-time work reads raw rows; -//! * whether a query-time result (`MatAtQuery`) may be kept across -//! evaluations. Its retention is derived from its consumers: a -//! tumbling-pane chain whose every reader repeats at the pane width -//! builds one new pane per evaluation and keeps the others for the next -//! ones, covering the readers' longest look-back. Without it, a -//! `MatAtQuery` result is reused only within one evaluation. +//! a candidate that scans raw data at query time is invalid (the +//! rejection lists every such scan): only ingestion-time work reads raw +//! rows. Stored data (at rest or mixed) with no raw data retained is a +//! contradiction, refused as bad input ([`DeploymentCapabilities::validate`]); +//! * whether query-time results are retained across evaluations. If they +//! are, Stage 2 also offers a tumbling-pane chain whose readers evaluate +//! together at its pane width as `MatAtQueryKeptAcrossEvaluations`: each +//! evaluation builds one new pane and keeps the others for the next ones, +//! covering the readers' longest look-back. A chain two or more queries +//! read keeps its `MatAtQuery` option too (shared within one evaluation), +//! and Stage 3 picks the cheaper. If they are not, a query-time result is +//! reused only within one evaluation. //! //! The default supports everything the planner can build, retains raw data //! and keeps no query-time result across evaluations, which is the //! planner's behavior without deployment inputs. use asap_types::post_asap::{SketchAlgorithm, StatisticKind}; +use asap_types::workload::{DataArrival, DataWorkload}; use serde::{Deserialize, Serialize}; +use thiserror::Error; /// What the deployment's executor can run and keep. Planning reads it in -/// Stage 1 (`supported_summaries`), Stage 2 and Stage 3 (the two -/// retention switches). It is data only; `stage_pipeline` reads it from -/// `--deployment ` and shows it in the stage document. +/// Stage 1 (`supported_summaries`), Stage 2 (`query_time_results_retained`) +/// and Stage 3 (`raw_data_retained`). It is data only; `stage_pipeline` reads it from +/// `--deployment-capabilities ` and shows it in the stage document. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct DeploymentCapabilities { @@ -43,9 +49,9 @@ pub struct DeploymentCapabilities { pub supported_summaries: Option>, /// Whether raw samples are kept for query-time scans; default `true`. pub raw_data_retained: bool, - /// Whether a query-time (`MatAtQuery`) result may be kept across - /// evaluations; default `false`. - pub query_time_retention: bool, + /// Whether a query-time result may be kept across evaluations + /// (`MatAtQueryKeptAcrossEvaluations`); default `false`. + pub query_time_results_retained: bool, } /// One sketch family the executor can build, with the readouts it can @@ -64,7 +70,7 @@ impl Default for DeploymentCapabilities { Self { supported_summaries: None, raw_data_retained: true, - query_time_retention: false, + query_time_results_retained: false, } } } @@ -78,6 +84,34 @@ impl DeploymentCapabilities { .any(|s| s.family == *family && s.readouts.contains(&readout)) }) } + + /// Checks these capabilities against the data workload: stored data + /// (at rest, or mixed) is raw data the deployment keeps, so it + /// contradicts `raw_data_retained: false`. Plan selection refuses such + /// inputs before measuring any candidate. + pub fn validate(&self, data: &DataWorkload) -> Result<(), DeploymentCapabilitiesError> { + match &data.arrival { + DataArrival::AtRest | DataArrival::Mixed { .. } if !self.raw_data_retained => { + Err(DeploymentCapabilitiesError::StoredDataNotRetained { + arrival: data.arrival, + }) + } + _ => Ok(()), + } + } +} + +/// Deployment capabilities that contradict the other planning inputs +/// ([`DeploymentCapabilities::validate`]); always bad input. +#[derive(Debug, Clone, PartialEq, Error)] +pub enum DeploymentCapabilitiesError { + /// Bad input: the data workload declares stored raw data (`AtRest` or + /// `Mixed`), but the deployment says it retains none. + #[error("the data is stored ({arrival:?}), but the deployment does not retain raw data")] + StoredDataNotRetained { + /// The declared arrival that stores raw data. + arrival: DataArrival, + }, } #[cfg(test)] @@ -91,7 +125,7 @@ mod tests { fn default_supports_everything_and_json_fills_the_rest() { let default = DeploymentCapabilities::default(); assert!(default.supports(&SketchAlgorithm::Theta, StatisticKind::TopK)); - assert!(default.raw_data_retained && !default.query_time_retention); + assert!(default.raw_data_retained && !default.query_time_results_retained); let parsed: DeploymentCapabilities = serde_json::from_str( r#"{"supported_summaries": [{"family": "Kll", "readouts": ["Quantile"]}]}"#, ) @@ -99,7 +133,7 @@ mod tests { assert!(parsed.supports(&SketchAlgorithm::Kll, StatisticKind::Quantile)); assert!(!parsed.supports(&SketchAlgorithm::Kll, StatisticKind::Total)); assert!(!parsed.supports(&SketchAlgorithm::DDSketch, StatisticKind::Quantile)); - assert!(parsed.raw_data_retained && !parsed.query_time_retention); + assert!(parsed.raw_data_retained && !parsed.query_time_results_retained); assert!(serde_json::from_str::(r#"{"raw_data": false}"#).is_err()); } } diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 6efbf7269..a8e2c3954 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -181,7 +181,9 @@ pub use cost_model::{ CostModel, CostProvenance, CostUnit, DefaultCostModel, ExactCompositionCostInputs, ExactCompositionCostRequest, ValueOperationCapabilities, }; -pub use deployment_capabilities::{DeploymentCapabilities, SupportedSummary}; +pub use deployment_capabilities::{ + DeploymentCapabilities, DeploymentCapabilitiesError, SupportedSummary, +}; pub use exact_composition::{ExactComposition, ExactCompositionStrategy, OperationPlacement}; pub use explanation::{ explain_replacements, explain_replacements_with, ExplanationKind, ReplacementExplanation, diff --git a/crates/asap-aware-mapping/src/pass/mod.rs b/crates/asap-aware-mapping/src/pass/mod.rs index 464a27fc1..432ed33cd 100644 --- a/crates/asap-aware-mapping/src/pass/mod.rs +++ b/crates/asap-aware-mapping/src/pass/mod.rs @@ -55,10 +55,10 @@ static DEFAULT_DEPLOYMENT: std::sync::LazyLock = /// a built-in default, evidence about a particular deployment cannot. /// /// Plan selection reads `accuracy`, `evidence`, `plan_selection` and -/// `deployment`. It measures candidates with its own analytical model and -/// compares them under `plan_selection`'s cost model, so -/// [`Stage1ToStage3Pass`] does not read `cost`. Stage 1 and Stage 2 read -/// `deployment` too. +/// `deployment_capabilities`. It measures candidates with its own +/// analytical model and compares them under `plan_selection`'s cost model, +/// so [`Stage1ToStage3Pass`] does not read `cost`. Stage 1 and Stage 2 read +/// `deployment_capabilities` too. #[derive(Clone, Copy)] #[non_exhaustive] pub struct PlanningModels<'a> { @@ -71,9 +71,9 @@ pub struct PlanningModels<'a> { pub plan_selection: &'a PlanSelectionSettings, /// What the deployment's executor can run and keep: the summaries it /// supports (Stage 1), whether it retains raw data (Stage 3) and - /// whether it keeps query-time results across evaluations (Stages 2 - /// and 3); default [`DeploymentCapabilities::default`]. - pub deployment: &'a DeploymentCapabilities, + /// whether it keeps query-time results across evaluations (Stage 2); + /// default [`DeploymentCapabilities::default`]. + pub deployment_capabilities: &'a DeploymentCapabilities, } impl<'a> PlanningModels<'a> { @@ -87,7 +87,7 @@ impl<'a> PlanningModels<'a> { accuracy, evidence, plan_selection: &DEFAULT_PLAN_SELECTION, - deployment: &DEFAULT_DEPLOYMENT, + deployment_capabilities: &DEFAULT_DEPLOYMENT, } } @@ -98,7 +98,7 @@ impl<'a> PlanningModels<'a> { accuracy: &DEFAULT_ACCURACY_MODEL, evidence: &NO_ACCURACY_EVIDENCE, plan_selection: &DEFAULT_PLAN_SELECTION, - deployment: &DEFAULT_DEPLOYMENT, + deployment_capabilities: &DEFAULT_DEPLOYMENT, } } @@ -126,8 +126,11 @@ impl<'a> PlanningModels<'a> { /// These models with the deployment's capabilities (supported /// summaries, raw-data and query-time retention). - pub fn with_deployment(mut self, deployment: &'a DeploymentCapabilities) -> Self { - self.deployment = deployment; + pub fn with_deployment_capabilities( + mut self, + deployment_capabilities: &'a DeploymentCapabilities, + ) -> Self { + self.deployment_capabilities = deployment_capabilities; self } } diff --git a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs index 184ccae15..20971d318 100644 --- a/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs +++ b/crates/asap-aware-mapping/src/pass/stage1_to_stage3.rs @@ -489,7 +489,8 @@ pub fn run_stage1_to_stage3_from_pass1( data: &DataWorkload, models: PlanningModels<'_>, ) -> Stage1ToStage3Run { - let per_computation = per_computation.without_unsupported_summaries(models.deployment); + let per_computation = + per_computation.without_unsupported_summaries(models.deployment_capabilities); let recurrence: Vec<_> = entries.iter().map(|e| e.recurrence.clone()).collect(); let stage1 = stage1_logical_asap_optimization(&per_computation, &recurrence).collect(); run_stage2_and_stage3(per_computation, stage1, entries, data, models) @@ -500,6 +501,15 @@ pub fn run_stage1_to_stage3_from_pass1( /// enumerated): candidate `n` (from 1) is `L`, implemented as `P` /// with nothing materialized and as `P-m` (`k` from 1) for each other /// materialization assignment. +/// +/// It does not prune: `per_computation` must already be pruned for +/// `models.deployment_capabilities` +/// ([`PerComputationCandidates::without_unsupported_summaries`]), with +/// `stage1` enumerated from that pruned inventory, as +/// [`run_stage1_to_stage3_from_pass1`] does. Pruning here would come too +/// late: each of `stage1`'s choices indexes the inventory it was enumerated +/// from, so a candidate with an unsupported summary would already be in +/// `stage1`. pub fn run_stage2_and_stage3( per_computation: PerComputationCandidates, stage1: Vec, @@ -515,7 +525,8 @@ pub fn run_stage2_and_stage3( let number = index + 1; let id = format!("L{number}"); if let Ok(roots) = &candidate.roots { - match optimize_physical_asap(&id, roots, entries, data, models.deployment) { + match optimize_physical_asap(&id, roots, entries, data, models.deployment_capabilities) + { Ok(implemented) => { for (k, mut p) in implemented.into_iter().enumerate() { p.id = match k { diff --git a/crates/asap-aware-mapping/src/per_computation_candidates.rs b/crates/asap-aware-mapping/src/per_computation_candidates.rs index 19934a288..894cdb522 100644 --- a/crates/asap-aware-mapping/src/per_computation_candidates.rs +++ b/crates/asap-aware-mapping/src/per_computation_candidates.rs @@ -272,8 +272,14 @@ pub struct FusedComputation { /// The index in [`PerComputationCandidates::targets`] of the inner /// target it absorbs. pub absorbs: usize, - /// How the fused computation is computed: exact or a summary. + /// How the fused computation is computed: exact or a summary. Empty + /// when Pass 1 pruned every one ([`Self::pruned`]); the fusion is then + /// not offered. pub candidates: Vec, + /// The candidates Pass 1 pruned from the fused computation, with their + /// reasons (today only [`PruneReason::UnsupportedByDeployment`]); empty + /// by default. + pub pruned: Vec<(ComputationCandidate, PruneReason)>, } impl ComputationTarget { @@ -355,8 +361,9 @@ impl PerComputationCandidates { /// This inventory without the summary candidates `deployment` does not /// support: a target's own are moved to its pruned candidates with - /// [`PruneReason::UnsupportedByDeployment`], and a fused computation's - /// are dropped (with all of them, the fusion is). The readout is the + /// [`PruneReason::UnsupportedByDeployment`], and so are a fused + /// computation's (with all of them pruned, the fusion is not offered). + /// The readout is the /// statistic the measure's estimate reads (the outer measure's for a /// fused computation). With every summary supported it is unchanged. pub fn without_unsupported_summaries(mut self, deployment: &DeploymentCapabilities) -> Self { @@ -375,10 +382,15 @@ impl PerComputationCandidates { .choices .prune(PruneReason::UnsupportedByDeployment, unsupported); if let Some(fused) = &mut target.fused { - fused.candidates.retain(|candidate| !unsupported(candidate)); - if fused.candidates.is_empty() { - target.fused = None; - } + let (pruned, kept): (Vec<_>, _) = std::mem::take(&mut fused.candidates) + .into_iter() + .partition(unsupported); + fused.candidates = kept; + fused.pruned.extend( + pruned + .into_iter() + .map(|candidate| (candidate, PruneReason::UnsupportedByDeployment)), + ); } } self @@ -683,6 +695,7 @@ fn add_nested_sum_targets( targets[t].fused = Some(FusedComputation { absorbs: inner, candidates: vec![merge], + pruned: Vec::new(), }); } } @@ -813,6 +826,7 @@ fn add_whole_expression_targets( target.fused = Some(FusedComputation { absorbs: inner, candidates: heaps, + pruned: Vec::new(), }); } } @@ -4657,8 +4671,9 @@ mod tests { } /// A fused whole-expression top-k loses the heap families the - /// deployment cannot answer top-k from, and the fusion goes with the - /// last of them: the target and the inner sum keep their exact options. + /// deployment cannot answer top-k from, each recorded with its reason, + /// and is not offered once it has none left: the target and the inner + /// sum keep their exact options. #[test] fn a_fused_computation_loses_its_unsupported_heaps() { let query = "topk by (job) (10, sum_over_time(m[1m]))"; @@ -4673,6 +4688,13 @@ mod tests { SketchAlgorithm::CountSketchWithHeap )] ); + assert_eq!( + one_heap.targets[topk].fused.as_ref().unwrap().pruned, + [( + ComputationCandidate::Summary(SketchAlgorithm::CmsWithHeap), + PruneReason::UnsupportedByDeployment + )] + ); assert_eq!(one_heap.combinations(), 2); // Count-Min with a heap answers item frequencies, not top-k. let none = inventory(query).without_unsupported_summaries(&supporting(&[( @@ -4680,7 +4702,21 @@ mod tests { &[StatisticKind::ItemFrequency], )])); let topk = topk_target(&none); - assert!(none.targets[topk].fused.is_none()); + let fused = none.targets[topk].fused.as_ref().unwrap(); + assert!(fused.candidates.is_empty()); + assert_eq!( + fused.pruned, + [ + ( + ComputationCandidate::Summary(SketchAlgorithm::CmsWithHeap), + PruneReason::UnsupportedByDeployment + ), + ( + ComputationCandidate::Summary(SketchAlgorithm::CountSketchWithHeap), + PruneReason::UnsupportedByDeployment + ), + ] + ); assert_eq!( none.targets[topk].choices.candidates, [ComputationCandidate::Exact] diff --git a/crates/asap-aware-mapping/src/stage2_physical_asap_optimization.rs b/crates/asap-aware-mapping/src/stage2_physical_asap_optimization.rs index 35acaae63..edd463b55 100644 --- a/crates/asap-aware-mapping/src/stage2_physical_asap_optimization.rs +++ b/crates/asap-aware-mapping/src/stage2_physical_asap_optimization.rs @@ -7,21 +7,22 @@ //! Each summary build (a `SummaryAgg` or a `PaneChain`) is one //! materialization candidate ([`MaterializationCandidate`]) whose //! output materialization may store (#509 §Materialization, Algorithm 3). -//! Each has up to three options: +//! Each has up to four options: //! //! * `NotMaterialized`: rebuilt at query time by every query that reads it; //! * `MatAtIngestion(Memory)`: maintained as data arrives and kept for the //! queries, when the build is eligible ([`IngestionPruneReason`]); -//! * `MatAtQuery(Memory)`: built at query time for every query that reads -//! it, offered for a build two or more queries read, and, when the -//! deployment keeps query-time results across evaluations -//! ([`DeploymentCapabilities::query_time_retention`]), for a +//! * `MatAtQuery(Memory)`: built once at query time for every query of one +//! evaluation that reads it and reused only within that evaluation, +//! offered for a build two or more queries read; +//! * `MatAtQueryKeptAcrossEvaluations(Memory)`: built at query time and +//! kept across evaluations (#509 Example 4's B3), offered when the +//! deployment keeps query-time results +//! ([`DeploymentCapabilities::query_time_results_retained`]) for a //! tumbling-pane chain that builds one new pane per evaluation -//! ([`builds_one_pane_per_evaluation`]): every reader repeats at the pane -//! width. Such a chain is kept across evaluations (#509 Example 4's B3): -//! each evaluation builds the newest pane and reuses the others. Any -//! other `MatAtQuery` state is reused only within one evaluation. The IR -//! records no retention: Stage 3 derives it from the readers the same way. +//! ([`builds_one_pane_per_evaluation`]): each evaluation builds the +//! newest pane and reuses the others. Both query-time options are offered +//! when both apply, and Stage 3 prices each. //! //! A build at ingestion time needs every build below it at ingestion time too //! (#509 Algorithm 3, line 13); such assignments are not generated. Every @@ -189,24 +190,30 @@ pub(crate) struct MaterializationCandidate { pub(crate) readers: Vec, /// `Ok` when `MatAtIngestion` is an option. pub(crate) at_ingestion: Result<(), IngestionPruneReason>, - /// Whether it may be kept from query time across evaluations: the + /// Whether `MatAtQueryKeptAcrossEvaluations` is an option: the /// deployment keeps query-time results and it is a tumbling-pane chain /// that builds one new pane per evaluation. - pub(crate) kept_across_evaluations: bool, + pub(crate) may_keep_across_evaluations: bool, } impl MaterializationCandidate { /// Its options, `NotMaterialized` first: `MatAtIngestion` when eligible, - /// `MatAtQuery` when two or more queries read it or it may be kept - /// across evaluations. + /// `MatAtQuery` when two or more queries read it, and + /// `MatAtQueryKeptAcrossEvaluations` when it may be kept across + /// evaluations. pub(crate) fn options(&self) -> Vec { let mut options = vec![Materialization::NotMaterialized]; if self.at_ingestion.is_ok() { options.push(Materialization::MatAtIngestion(Medium::Memory)); } - if self.readers.len() > 1 || self.kept_across_evaluations { + if self.readers.len() > 1 { options.push(Materialization::MatAtQuery(Medium::Memory)); } + if self.may_keep_across_evaluations { + options.push(Materialization::MatAtQueryKeptAcrossEvaluations( + Medium::Memory, + )); + } options } } @@ -224,17 +231,39 @@ pub(crate) fn fixed_interval_ms(recurrence: &QueryRecurrence) -> Option { /// Whether a tumbling-pane chain of `pane_width_ms`-wide panes read by /// queries repeating as `readers` builds exactly one new pane per -/// evaluation: there is a reader, and every reader repeats at a fixed -/// interval equal to the pane width. Its other panes are the previous +/// evaluation: there is a reader, every reader repeats at a fixed interval +/// equal to the pane width, and the readers whose evaluation phase is known +/// share one phase (modulo the interval). Its other panes are the previous /// evaluations' newest, so a deployment that keeps query-time results /// reuses them (#509 Example 4's B3). +/// +/// Co-scheduling assumption: readers with the same interval and no known +/// phase run at the same instants as each other and as the readers with a +/// known phase, as [`crate::stage3_plan_selection`] assumes when it counts +/// one evaluation per interval. Readers at different known phases would +/// each need their own newest pane, so they do not share one kept chain. pub(crate) fn builds_one_pane_per_evaluation<'a>( pane_width_ms: u64, readers: impl IntoIterator, ) -> bool { - let mut readers = readers.into_iter().peekable(); - readers.peek().is_some() - && readers.all(|recurrence| fixed_interval_ms(recurrence) == Some(pane_width_ms)) + let mut phase_ms = None; + let mut any_reader = false; + for recurrence in readers { + any_reader = true; + if fixed_interval_ms(recurrence) != Some(pane_width_ms) { + return false; + } + if let QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + evaluation_phase, .. + }) = recurrence + { + let phase = evaluation_phase.0 % pane_width_ms; + if *phase_ms.get_or_insert(phase) != phase { + return false; + } + } + } + any_reader } /// Stage 2 for one CandidateLogicalASAPDAG: implement every operator @@ -384,7 +413,7 @@ pub(crate) fn materialization_candidates( .iter() .map(|&q| entries.get(q).map(|e| &e.recurrence)) .collect(); - let kept_across_evaluations = deployment.query_time_retention + let may_keep_across_evaluations = deployment.query_time_results_retained && match (node.asap(), recurrences) { (Some(ASAPOp::PaneChain { pane_width_ms, .. }), Some(recurrences)) => { builds_one_pane_per_evaluation(*pane_width_ms, recurrences) @@ -395,7 +424,7 @@ pub(crate) fn materialization_candidates( node: Rc::clone(node), readers, at_ingestion, - kept_across_evaluations, + may_keep_across_evaluations, } }) .collect() @@ -1421,4 +1450,36 @@ mod tests { [ExecutionTiming::IngestionTime, ExecutionTiming::QueryTime] ); } + + /// Readers share one kept pane chain only when they evaluate at the + /// same instants: two readers every minute at phases 0 and 2 min do, + /// at phases 0 and 30 s they do not (each would need its own newest + /// pane). A reader with no known phase is assumed to run with the + /// others. + #[test] + fn readers_at_different_phases_do_not_build_one_pane_per_evaluation() { + let at = |phase_ms: u64| { + QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(MINUTE as u32), + evaluation_phase: TimestampMs(phase_ms), + }) + }; + let unknown_phase = every_minute().recurrence; + assert!(builds_one_pane_per_evaluation( + MINUTE, + [&at(0), &at(2 * MINUTE)] + )); + assert!(!builds_one_pane_per_evaluation( + MINUTE, + [&at(0), &at(30_000)] + )); + assert!(builds_one_pane_per_evaluation( + MINUTE, + [&at(30_000), &unknown_phase] + )); + assert!(!builds_one_pane_per_evaluation( + MINUTE, + [&at(0), &unknown_phase, &at(30_000)] + )); + } } diff --git a/crates/asap-aware-mapping/src/stage3_plan_selection.rs b/crates/asap-aware-mapping/src/stage3_plan_selection.rs index ffea9ebf5..479a7c696 100644 --- a/crates/asap-aware-mapping/src/stage3_plan_selection.rs +++ b/crates/asap-aware-mapping/src/stage3_plan_selection.rs @@ -58,13 +58,14 @@ //! `MatAtQuery` node's state is reused only within one evaluation, so it is //! held only while that evaluation runs: its bytes times the latency of the //! longest query reading it, once per evaluation, which is close to nothing. -//! When the deployment keeps query-time results -//! ([`DeploymentCapabilities::query_time_retention`]), a `MatAtQuery` -//! tumbling-pane chain whose every reader repeats at its pane width is kept -//! across evaluations (#509 Example 4's B3): each evaluation scans one pane -//! of raw data and builds the newest pane, and the `N − 1` older panes, -//! which complete the readers' look-back, are kept in memory every second -//! (all `N` at the peak). +//! A `MatAtQueryKeptAcrossEvaluations` tumbling-pane chain (#509 Example +//! 4's B3, offered by Stage 2 when the deployment keeps query-time results, +//! [`crate::DeploymentCapabilities::query_time_results_retained`]) is kept across +//! evaluations: each evaluation scans one pane of raw data and builds the +//! newest pane, and the `N − 1` older panes, which complete the readers' +//! look-back, are kept every second (in memory, all `N` at the peak). When +//! Stage 2 offers both query-time options for a chain, plan selection +//! prices each and the cheaper wins. //! Memory a computation //! holds only while it runs is not charged under cost model 1: it counts in //! [`ResourceUsage::peak_memory_bytes`], which cost models 2 and 3 read. A @@ -114,7 +115,7 @@ //! Disk holds state materialized on `Medium::Disk` (Stage 2 places none //! there yet) and the raw data query-time scans read: per source, the //! longest window a scan reads. A deployment that does not retain raw data -//! ([`DeploymentCapabilities::raw_data_retained`]) keeps no raw samples +//! ([`crate::DeploymentCapabilities::raw_data_retained`]) keeps no raw samples //! after they arrive, so a candidate with a query-time scan is invalid, //! with the window it would read as the reason. //! @@ -170,7 +171,7 @@ use crate::accuracy::{OperandDomain, PropagationStats}; use crate::analytical_cost::{ estimate_operator, AnalyticalCostError, PhysicalOperator, ResourceEstimate, }; -use crate::deployment_capabilities::DeploymentCapabilities; +use crate::deployment_capabilities::DeploymentCapabilitiesError; use crate::per_computation_candidates::{ approximate_inputs, distinct_bound_source, operand_domain_statistic, operand_guarantee, operand_of, project_column_guarantee, root_guarantee, with_resized_summaries, @@ -178,7 +179,7 @@ use crate::per_computation_candidates::{ use crate::physical_operator_statistics::{ EdgeStatistics, OperatorStatistics, PartitionStatistics, UnaryEdgeStatistics, }; -use crate::stage2_physical_asap_optimization::{builds_one_pane_per_evaluation, PhysicalCandidate}; +use crate::stage2_physical_asap_optimization::PhysicalCandidate; use crate::PlanningModels; /// Groups assumed for every `by (...)` reduction, absent group-count evidence. @@ -198,8 +199,8 @@ pub const COST_SOURCE: &str = "analytical-cost-v2 (illustrative statistics)"; pub const DEFAULT_NON_REPEATING_QUERY_HORIZON_SECONDS: f64 = 3_600.0; /// What one candidate uses, per second of wall time unless the name says -/// otherwise, without prices: its nodes' [`NodeUsage`] summed, with the -/// candidate's peak cores and per-query latency. Every [`CostModel`] reads +/// otherwise, without prices: its nodes' CPU, scans, kept memory and disk +/// added up, with the candidate's peak cores and per-query latency. Every [`CostModel`] reads /// it. #[derive(Debug, Clone, Default, PartialEq)] pub struct ResourceUsage { @@ -225,22 +226,6 @@ pub struct ResourceUsage { pub latency_per_query: Vec, } -/// What one node of a candidate uses: the [`ResourceUsage`] fields that add -/// up over nodes. Cores and latency are the whole candidate's. -#[derive(Debug, Clone, Default, PartialEq)] -pub struct NodeUsage { - /// CPU operations per second. - pub cpu_ops_per_second: f64, - /// Bytes scanned per second. - pub scanned_bytes_per_second: f64, - /// Kept memory averaged over time, in byte-seconds per second. - pub average_memory_bytes: f64, - /// The most memory the node holds at once (every evaluation overlapping). - pub peak_memory_bytes: u64, - /// Bytes the node keeps on disk. - pub disk_bytes: u64, -} - /// The price of one unit of each resource, per second of wall time. Cost /// model 1 reads the usage prices (CPU, scan, memory byte-seconds, disk); /// cost model 2 the provisioned prices (memory, CPU cores) and disk. @@ -421,23 +406,23 @@ impl PlanSelectionSettings { } } -/// One node's usage and price, per second of wall time. `multiplier` is the -/// number of consumers it is computed for (1 when materialized) and -/// `per_second` the times it is computed per second ([`computations`]). +/// One node's price per second of wall time, with how it was measured. #[derive(Debug, Clone, PartialEq)] pub struct NodeCost { - pub usage: NodeUsage, /// Its price per second under cost model 1; `None` under cost models 2 /// and 3, which price (or limit) the whole candidate's peaks. pub cost_per_second: Option, + /// The number of consumers it is computed for (1 when materialized, + /// [`computations`]). pub multiplier: u64, - pub per_second: f64, + /// How it was measured, e.g. "scan 1000 samples; x 1.0000/s; keeps + /// ... bytes": one computation's work, the times it is computed per + /// second and what it keeps. pub detail: String, } -/// One candidate's usage and price over the whole workload: one entry per -/// DAG node; `usage` sums them, with the candidate's peak cores and -/// per-query latency. +/// One candidate's usage and price over the whole workload: `usage` is +/// the whole candidate's, `per_node` one price per DAG node. #[derive(Debug, Clone, PartialEq)] pub struct CandidateCost { pub usage: ResourceUsage, @@ -588,6 +573,10 @@ pub enum SelectionError { roots: usize, entries: usize, }, + /// Bad input: the deployment's capabilities contradict the data + /// workload ([`crate::DeploymentCapabilities::validate`]). + #[error("deployment capabilities: {0}")] + InvalidDeploymentCapabilities(DeploymentCapabilitiesError), #[error("candidate {candidate}, node {node:?}: {error}")] Cost { candidate: String, @@ -616,6 +605,10 @@ pub fn select_plan( settings .validate() .map_err(SelectionError::InvalidSettings)?; + models + .deployment_capabilities + .validate(data) + .map_err(SelectionError::InvalidDeploymentCapabilities)?; let targets = &accuracy_targets(entries); let mut costs = BTreeMap::new(); let mut accepted = BTreeMap::new(); @@ -654,20 +647,26 @@ pub fn select_plan( continue; } }; - if let Some(reason) = raw_data_violation(&candidate.dag, &recurrence, models.deployment) { - rejected.push(Rejection { - id: candidate.id.clone(), - valid: false, - reason, - }); - continue; + // Built once per candidate for the raw-data check and pricing. + let nodes = node_map(&candidate.dag); + let kept_chains = kept_pane_chains(&candidate.dag); + if !models.deployment_capabilities.raw_data_retained { + if let Some(reason) = raw_data_violation(&candidate.dag, &nodes, &kept_chains) { + rejected.push(Rejection { + id: candidate.id.clone(), + valid: false, + reason, + }); + continue; + } } let cost = price( &candidate.dag, + &nodes, + &kept_chains, &recurrence, data, settings, - models.deployment, ) .map_err(|(node, error)| SelectionError::Cost { candidate: candidate.id.clone(), @@ -1892,14 +1891,15 @@ fn computations( /// time ranges keep, times how often it is computed per second /// ([`computations`]); an ingestion-time node over one second of arriving /// rows. A materialized node's kept state is counted in byte-seconds per -/// second. `deployment` says whether a query-time pane chain is kept across -/// evaluations ([`kept_pane_chains`]). +/// second. `nodes` and `kept_chains` are `dag`'s ([`node_map`], +/// [`kept_pane_chains`]). fn price( dag: &PhysicalASAPDAG, + nodes: &NodeMap<'_>, + kept_chains: &HashSet, recurrence: &[QueryRecurrence], data: &DataWorkload, settings: &PlanSelectionSettings, - deployment: &DeploymentCapabilities, ) -> Result { let first = dag.nodes.first().map(|n| n.id).unwrap_or_default(); let speed = settings.resource_speed; @@ -1915,8 +1915,6 @@ fn price( / 1_000.0; // The sources each node reads, filled children first. let mut sources_read_by: HashMap> = HashMap::new(); - let nodes: HashMap<_, _> = dag.nodes.iter().map(|n| (n.id, n)).collect(); - let kept_chains = kept_pane_chains(dag, &nodes, recurrence, deployment); let multipliers = computations( dag, recurrence, @@ -1925,6 +1923,9 @@ fn price( .map_err(|error| (first, error))?; let mut output: HashMap = HashMap::new(); let mut per_node = BTreeMap::new(); + // The candidate's usage, accumulated node by node; peak cores and + // latency are filled in once every node is measured. + let mut usage = ResourceUsage::default(); // Each node's work in ms on one core: one computation at query time, // one second of arriving rows at ingestion time. let mut work_ms: HashMap = HashMap::new(); @@ -1989,7 +1990,7 @@ fn price( // query time, the longest window read above it. let lookback = match ingestion { true => 1_000, - false => query_time_lookback_ms(dag, &nodes, node.id, &kept_chains), + false => query_time_lookback_ms(dag, nodes, node.id, kept_chains), }; let out = edge(((rows_per_ms * lookback as f64).round() as u64).max(1)); let estimate = estimate_operator( @@ -2242,11 +2243,14 @@ fn price( + estimate.scan_bytes() as f64 * speed.ms_per_scanned_byte, ); let (multiplier, per_second) = multipliers[&node.id]; - let mut usage = NodeUsage { - cpu_ops_per_second: estimate.cpu_ops() * per_second, - scanned_bytes_per_second: estimate.scan_bytes() as f64 * per_second, - ..NodeUsage::default() - }; + usage.cpu_ops_per_second += estimate.cpu_ops() * per_second; + usage.scanned_bytes_per_second += estimate.scan_bytes() as f64 * per_second; + // What this node keeps: memory averaged over time (byte-seconds per + // second), its peak memory and its disk bytes, added to the + // candidate's usage below. + let mut average_memory_bytes = 0.0; + let mut peak_memory_bytes = 0u64; + let mut disk_bytes = 0u64; let mut detail = format!("{detail}; x {per_second:.4}/s"); // State kept between computations: at ingestion time, the window // being built and the completed ones queries read (a chain of N @@ -2260,21 +2264,21 @@ fn price( let state = out.bytes.max(estimate.peak_memory_bytes()); let (kept, medium) = match (node.materialization, &node.payload) { (Materialization::NotMaterialized, _) => { - usage.peak_memory_bytes = estimate.peak_memory_bytes().saturating_mul(multiplier); + peak_memory_bytes = estimate.peak_memory_bytes().saturating_mul(multiplier); (0, Medium::Memory) } // A pane chain kept across evaluations holds its N − 1 older // panes between evaluations, for the next ones, and all N // while one runs. - (Materialization::MatAtQuery(medium), _) if kept_chains.contains(&node.id) => { + (Materialization::MatAtQueryKeptAcrossEvaluations(medium), _) => { let panes = panes_of(node); let between = state / panes * (panes - 1); match medium { Medium::Memory => { - usage.average_memory_bytes = between as f64; - usage.peak_memory_bytes = state; + average_memory_bytes = between as f64; + peak_memory_bytes = state; } - Medium::Disk => usage.disk_bytes = state, + Medium::Disk => disk_bytes = between, } detail += &format!("; keeps {between} bytes across evaluations"); (0, medium) @@ -2283,9 +2287,9 @@ fn price( match medium { Medium::Memory => { held.insert(node.id, (state, per_second)); - usage.peak_memory_bytes = state; + peak_memory_bytes = state; } - Medium::Disk => usage.disk_bytes = state, + Medium::Disk => disk_bytes = state, } (0, medium) } @@ -2304,10 +2308,10 @@ fn price( if kept > 0 { match medium { Medium::Memory => { - usage.average_memory_bytes = kept as f64; - usage.peak_memory_bytes = kept; + average_memory_bytes = kept as f64; + peak_memory_bytes = kept; } - Medium::Disk => usage.disk_bytes = kept, + Medium::Disk => disk_bytes = kept, } detail += &format!("; keeps {kept} bytes"); } @@ -2327,9 +2331,12 @@ fn price( (estimate.cpu_ops() * prices.cost_per_cpu_op + estimate.scan_bytes() as f64 * prices.cost_per_scanned_byte) * per_second - + prices.cost_per_memory_byte_second * usage.average_memory_bytes - + prices.cost_per_disk_byte_second * usage.disk_bytes as f64 + + prices.cost_per_memory_byte_second * average_memory_bytes + + prices.cost_per_disk_byte_second * disk_bytes as f64 }); + usage.average_memory_bytes += average_memory_bytes; + usage.peak_memory_bytes = usage.peak_memory_bytes.saturating_add(peak_memory_bytes); + usage.disk_bytes = usage.disk_bytes.saturating_add(disk_bytes); if cost.is_some_and(|cost| !cost.is_finite()) { return Err((node.id, AnalyticalCostError::Overflow)); } @@ -2337,17 +2344,15 @@ fn price( per_node.insert( node.id, NodeCost { - usage, cost_per_second: cost, multiplier, - per_second, detail, }, ); } for (bytes, id) in raw_data.into_values() { let node = per_node.get_mut(&id).expect("every node is measured"); - node.usage.disk_bytes += bytes; + usage.disk_bytes = usage.disk_bytes.saturating_add(bytes); node.detail += &format!("; keeps {bytes} bytes of raw data on disk"); if let (Some(cost), Some(prices)) = (&mut node.cost_per_second, usage_prices) { *cost += prices.cost_per_disk_byte_second * bytes as f64; @@ -2363,14 +2368,14 @@ fn price( } let bytes = copies.saturating_mul(window); let measured = per_node.get_mut(&id).expect("every node is measured"); - measured.usage.peak_memory_bytes = measured.usage.peak_memory_bytes.saturating_add(bytes); + usage.peak_memory_bytes = usage.peak_memory_bytes.saturating_add(bytes); measured.detail += &format!("; {copies} partial copies of its newest window, {bytes} bytes"); } - let latency = query_latencies(dag, &nodes, &output, &work_ms, &speed); + let latency = query_latencies(dag, nodes, &output, &work_ms, &speed); // A MatAtQuery state is held for one evaluation: from its build until // the longest query reading it finishes, about that query's latency. - let reach = query_reach(dag, &nodes); + let reach = query_reach(dag, nodes); for (id, (state, per_second)) in held { let evaluation_ms = reach .iter() @@ -2380,7 +2385,7 @@ fn price( .fold(0.0, f64::max); let byte_seconds = state as f64 * evaluation_ms / 1_000.0 * per_second; let node = per_node.get_mut(&id).expect("every node is measured"); - node.usage.average_memory_bytes += byte_seconds; + usage.average_memory_bytes += byte_seconds; node.detail += &format!("; holds {state} bytes for {evaluation_ms:.1} ms"); if let (Some(cost), Some(prices)) = (&mut node.cost_per_second, usage_prices) { *cost += prices.cost_per_memory_byte_second * byte_seconds; @@ -2389,17 +2394,7 @@ fn price( } } } - let mut usage = ResourceUsage::default(); - for node in per_node.values() { - usage.cpu_ops_per_second += node.usage.cpu_ops_per_second; - usage.scanned_bytes_per_second += node.usage.scanned_bytes_per_second; - usage.average_memory_bytes += node.usage.average_memory_bytes; - usage.peak_memory_bytes = usage - .peak_memory_bytes - .saturating_add(node.usage.peak_memory_bytes); - usage.disk_bytes = usage.disk_bytes.saturating_add(node.usage.disk_bytes); - } - usage.peak_cpu_cores = peak_cpu_cores(dag, &nodes, &work_ms, &multipliers, &latency); + usage.peak_cpu_cores = peak_cpu_cores(dag, nodes, &work_ms, &multipliers, &latency); usage.latency_per_query = latency; let cost = match &settings.cost_model { CostModel::UsageOverTime(_) => per_node.values().map(|node| node.cost_per_second).sum(), @@ -2566,7 +2561,7 @@ fn query_reach( /// [`DEFAULT_LOOKBACK_MS`] without one. fn query_time_lookback_ms( dag: &PhysicalASAPDAG, - nodes: &HashMap, + nodes: &NodeMap<'_>, scan: PhysicalASAPNodeId, kept_chains: &HashSet, ) -> u64 { @@ -2576,67 +2571,69 @@ fn query_time_lookback_ms( .unwrap_or(DEFAULT_LOOKBACK_MS) } -/// The `MatAtQuery` tumbling-pane chains of `dag` kept across evaluations: -/// none unless the deployment keeps query-time results; otherwise each one -/// that builds one new pane per evaluation, every query reaching it -/// repeating at its pane width (`builds_one_pane_per_evaluation`). Its -/// retention is derived from those readers: the newest pane is built at -/// each evaluation, and the older ones, which complete the readers' -/// longest look-back, are kept from earlier evaluations. -fn kept_pane_chains( - dag: &PhysicalASAPDAG, - nodes: &HashMap, - recurrence: &[QueryRecurrence], - deployment: &DeploymentCapabilities, -) -> HashSet { - if !deployment.query_time_retention { - return HashSet::new(); - } - let reach = query_reach(dag, nodes); +/// The nodes of `dag` by id. +fn node_map(dag: &PhysicalASAPDAG) -> NodeMap<'_> { + dag.nodes.iter().map(|n| (n.id, n)).collect() +} + +/// The nodes of a physical DAG by id. +type NodeMap<'a> = + HashMap; + +/// The tumbling-pane chains of `dag` kept across evaluations +/// (`MatAtQueryKeptAcrossEvaluations`, which Stage 2 offers only when the +/// deployment keeps query-time results and every reader builds one new +/// pane per evaluation). Each evaluation builds the newest pane, and the +/// older ones, which complete the readers' look-back, are kept from +/// earlier evaluations. +fn kept_pane_chains(dag: &PhysicalASAPDAG) -> HashSet { dag.nodes .iter() - .filter_map(|node| match (&node.payload, node.materialization) { - ( - Payload::ASAP(ASAPOp::PaneChain { pane_width_ms, .. }), - Materialization::MatAtQuery(_), - ) => { - let readers = reach - .iter() - .zip(recurrence) - .filter(|(reached, _)| reached.contains(&node.id)) - .map(|(_, recurrence)| recurrence); - builds_one_pane_per_evaluation(*pane_width_ms, readers).then_some(node.id) - } - _ => None, + .filter(|node| { + matches!( + (&node.payload, node.materialization), + ( + Payload::ASAP(ASAPOp::PaneChain { .. }), + Materialization::MatAtQueryKeptAcrossEvaluations(_), + ) + ) }) + .map(|node| node.id) .collect() } -/// When the deployment does not retain raw data, the first query-time scan -/// of `dag` as a rejection reason. Raw samples are then not kept after they -/// arrive, so no window of them is there to scan at query time: only -/// ingestion-time work reads raw rows. +/// For a deployment that does not retain raw data, every query-time scan +/// of `dag` with the window it would read, as a rejection reason (`None` +/// without one). Raw samples are then not kept after they arrive, so no +/// window of them is there to scan at query time: only ingestion-time work +/// reads raw rows. `nodes` and `kept_chains` are `dag`'s ([`node_map`], +/// [`kept_pane_chains`]). fn raw_data_violation( dag: &PhysicalASAPDAG, - recurrence: &[QueryRecurrence], - deployment: &DeploymentCapabilities, + nodes: &NodeMap<'_>, + kept_chains: &HashSet, ) -> Option { - if deployment.raw_data_retained { - return None; - } - let nodes: HashMap<_, _> = dag.nodes.iter().map(|n| (n.id, n)).collect(); - let kept_chains = kept_pane_chains(dag, &nodes, recurrence, deployment); - dag.nodes.iter().find_map(|node| match &node.payload { - Payload::NonASAP(NonASAPOp::Scan { source, .. }) - if node.output_state.timing != ExecutionTiming::IngestionTime => - { - Some(format!( - "scans {} ms of raw {} at query time, but the deployment does not retain raw data", - query_time_lookback_ms(dag, &nodes, node.id, &kept_chains), - source_name(source) - )) - } - _ => None, + let scans: Vec = dag + .nodes + .iter() + .filter_map(|node| match &node.payload { + Payload::NonASAP(NonASAPOp::Scan { source, .. }) + if node.output_state.timing != ExecutionTiming::IngestionTime => + { + Some(format!( + "{} ms of raw {}", + query_time_lookback_ms(dag, nodes, node.id, kept_chains), + source_name(source) + )) + } + _ => None, + }) + .collect(); + (!scans.is_empty()).then(|| { + format!( + "scans {} at query time, but the deployment does not retain raw data", + scans.join(", ") + ) }) } @@ -2830,7 +2827,7 @@ fn ddsketch_bins(alpha: f64) -> u64 { mod tests { use super::*; - /// [`super::price`] under the default deployment capabilities. + /// [`super::price`] with `dag`'s node map and kept pane chains. fn price( dag: &PhysicalASAPDAG, recurrence: &[QueryRecurrence], @@ -2839,13 +2836,15 @@ mod tests { ) -> Result { super::price( dag, + &node_map(dag), + &kept_pane_chains(dag), recurrence, data, settings, - &DeploymentCapabilities::default(), ) } use crate::accuracy::AccuracyEvidenceProvider; + use crate::deployment_capabilities::DeploymentCapabilities; use crate::stage2_physical_asap_optimization::optimize_physical_asap; use crate::test_support::{entry, entry_with, lower_promql}; use asap_types::ir::{ExprSemantics, ScalarExpr}; @@ -4858,7 +4857,7 @@ mod tests { fast.per_node[&id].cost_per_second, slow.per_node[&id].cost_per_second ); - assert_eq!(fast.per_node[&id].per_second, 1.0); + assert!(fast.per_node[&id].detail.contains("; x 1.0000/s")); } let estimate = dag.roots[0]; assert!(close( @@ -4895,7 +4894,9 @@ mod tests { } else { let evaluation_s = with.usage.latency_per_query[0].total_ms() / 1_000.0; assert!(evaluation_s > 0.0 && evaluation_s < 1.0, "{evaluation_s}"); - state * evaluation_s * with.per_node[&build].per_second + // Once per evaluation of the query repeating every 10 s. + assert!(with.per_node[&build].detail.contains("; x 0.1000/s")); + state * evaluation_s * 0.1 }; let term = with.per_node[&build].cost_per_second.unwrap() - without.per_node[&build].cost_per_second.unwrap(); @@ -5243,45 +5244,45 @@ mod tests { /// A deployment that keeps query-time results across evaluations. fn keeping_query_time_results() -> DeploymentCapabilities { DeploymentCapabilities { - query_time_retention: true, + query_time_results_retained: true, ..DeploymentCapabilities::default() } } - /// `query_time_retention` gates keeping a query-time result across + /// `query_time_results_retained` gates keeping a query-time result across /// evaluations, and only that. Without it, Stage 2 offers the 1-min /// panes of a query repeating every minute rebuilt (B2) or maintained - /// (B1); with it, also built at query time and kept (B3, `MatAtQuery`). - /// B3 is computed once per evaluation, reads one pane (1 min) of raw - /// data, keeps that much raw data on disk, and holds its 4 older panes - /// in memory between evaluations. The same DAG priced for a deployment - /// that keeps no query-time result is reused only within one - /// evaluation: the scan reads the 5-min window and nothing is kept. + /// (B1); with it, also built at query time and kept (B3, + /// `MatAtQueryKeptAcrossEvaluations`). B3 is computed once per + /// evaluation, reads one pane (1 min) of raw data, keeps that much raw + /// data on disk, and holds its 4 older panes in memory between + /// evaluations (on disk, if it were placed there). The same chain + /// `MatAtQuery` is reused only within one evaluation: the scan reads the + /// 5-min window and nothing is kept between evaluations. #[test] - fn query_time_retention_keeps_a_pane_chain_across_evaluations() { + fn query_time_results_retained_keeps_a_pane_chain_across_evaluations() { + assert_eq!(pane_candidates(10_000.0).len(), 2); let candidates = pane_candidates_for(10_000.0, &keeping_query_time_results()); assert_eq!(candidates.len(), 3); let b2 = &candidates[0].dag; let b3 = &candidates[2].dag; let chain = node_of(b3, is_chain); + let with_chain = |materialization: Materialization| { + let mut dag = b3.clone(); + for node in dag.nodes.iter_mut().filter(|n| n.id == chain) { + node.materialization = materialization; + } + dag + }; assert_eq!( - b3.nodes - .iter() - .find(|n| n.id == chain) - .unwrap() - .materialization, - Materialization::MatAtQuery(Medium::Memory) + *b3, + with_chain(Materialization::MatAtQueryKeptAcrossEvaluations( + Medium::Memory + )) ); let recurrence = [repeating(MINUTE)]; let settings = PlanSelectionSettings::default(); - let kept = super::price( - b3, - &recurrence, - &arriving(10_000.0), - &settings, - &keeping_query_time_results(), - ) - .unwrap(); + let kept = price(b3, &recurrence, &arriving(10_000.0), &settings).unwrap(); let rebuilt = price(b2, &recurrence, &arriving(10_000.0), &settings).unwrap(); let scan = node_of(b3, is_scan); assert_eq!(scanned_samples(&kept, scan), 10_000 * 60); @@ -5297,19 +5298,30 @@ mod tests { }; let (_, state_bytes) = summary_shape(family); let chain_cost = &kept.per_node[&chain]; - assert!(close(chain_cost.per_second, 1.0 / 60.0), "{chain_cost:?}"); - let groups = chain_cost.usage.peak_memory_bytes / (5 * state_bytes); - let between = 4 * groups * state_bytes; - assert_eq!(chain_cost.usage.average_memory_bytes, between as f64); + assert!(chain_cost.detail.contains("; x 0.0167/s"), "{chain_cost:?}"); + // One state per series: arriving() declares 10 000. + let between = 4 * 10_000 * state_bytes; assert!(chain_cost .detail .contains(&format!("keeps {between} bytes across evaluations"))); - let within_one = price(b3, &recurrence, &arriving(10_000.0), &settings).unwrap(); + // On disk, the same N − 1 older panes are kept. + let on_disk = with_chain(Materialization::MatAtQueryKeptAcrossEvaluations( + Medium::Disk, + )); + let on_disk = price(&on_disk, &recurrence, &arriving(10_000.0), &settings).unwrap(); + assert_eq!( + on_disk.usage.disk_bytes, + kept.usage.disk_bytes + between, + "{on_disk:?}" + ); + // Shared within one evaluation, nothing is kept between them. + let shared = with_chain(Materialization::MatAtQuery(Medium::Memory)); + let within_one = price(&shared, &recurrence, &arriving(10_000.0), &settings).unwrap(); assert_eq!(scanned_samples(&within_one, scan), 10_000 * 300); assert!(!within_one.per_node[&chain] .detail .contains("across evaluations")); - assert!(within_one.per_node[&chain].usage.average_memory_bytes < between as f64); + assert!(within_one.usage.average_memory_bytes < kept.usage.average_memory_bytes); } /// Without raw data retained, a candidate that scans raw data at query @@ -5325,7 +5337,7 @@ mod tests { &candidates, &[every_minute()], &arriving(10_000.0), - PlanningModels::builtin().with_deployment(deployment), + PlanningModels::builtin().with_deployment_capabilities(deployment), ) .unwrap() }; @@ -5358,6 +5370,100 @@ mod tests { ); } + /// Without raw data retained, the rejection lists every query-time + /// scan with the window it would read, not just the first. + #[test] + fn without_raw_data_every_query_time_scan_is_listed() { + let roots: Vec<(usize, QueryRoot)> = ["sum_over_time(a[5m])", "sum_over_time(b[1m])"] + .into_iter() + .map(|q| QueryRoot::Operator(lower_promql(q, AccuracyTarget::exact()))) + .enumerate() + .collect(); + // The exact candidate of each query. + let inventory = + crate::per_computation_candidates::generate_per_computation_candidates(roots).unwrap(); + let exact = vec![0; inventory.targets.len()]; + let roots: Vec = + crate::per_computation_candidates::compose_logical_candidate(&inventory, &exact) + .unwrap() + .into_iter() + .map(|(_, root)| root) + .collect(); + let mut candidate = optimize_physical_asap( + "L1", + &roots, + &[], + &data(), + &DeploymentCapabilities::default(), + ) + .unwrap() + .remove(0); + candidate.id = "P1".into(); + let entries = [entry(repeating(MINUTE)), entry(repeating(MINUTE))]; + let no_raw_data = DeploymentCapabilities { + raw_data_retained: false, + ..DeploymentCapabilities::default() + }; + let error = select_plan( + &[candidate], + &entries, + &data(), + PlanningModels::builtin().with_deployment_capabilities(&no_raw_data), + ) + .unwrap_err(); + let SelectionError::NoValidCandidate(rejected) = error else { + panic!("{error}") + }; + assert_eq!( + rejected[0].reason, + "scans 300000 ms of raw a, 60000 ms of raw b at query time, \ + but the deployment does not retain raw data" + ); + } + + /// Stored data (at rest, or mixed) with no raw data retained is a + /// contradiction in the inputs, refused before any candidate is + /// measured; arriving or undeclared data is not. + #[test] + fn stored_data_without_raw_data_retained_is_bad_input() { + let candidates = pane_candidates(10_000.0); + let no_raw_data = DeploymentCapabilities { + raw_data_retained: false, + ..DeploymentCapabilities::default() + }; + let select = |arrival: DataArrival| { + select_plan( + &candidates, + &[every_minute()], + &DataWorkload { arrival, ..data() }, + PlanningModels::builtin().with_deployment_capabilities(&no_raw_data), + ) + }; + for arrival in [ + DataArrival::AtRest, + DataArrival::Mixed { + rows_per_second: 10_000.0, + }, + ] { + let error = select(arrival).unwrap_err(); + assert_eq!( + error.to_string(), + format!( + "deployment capabilities: the data is stored ({arrival:?}), \ + but the deployment does not retain raw data" + ), + ); + } + for arrival in [ + DataArrival::Ingesting { + rows_per_second: 10_000.0, + }, + DataArrival::Unknown, + ] { + assert!(select(arrival).is_ok()); + } + } + fn is_chain(payload: &Payload) -> bool { matches!(payload, Payload::ASAP(ASAPOp::PaneChain { .. })) } @@ -5384,7 +5490,7 @@ mod tests { let work = &price(dag, &recurrence, &arriving(10_000.0), &without_memory) .unwrap() .per_node[&chain]; - assert_eq!(work.per_second, 1.0); + assert!(work.detail.contains("; x 1.0000/s"), "{work:?}"); assert!( close( work.cost_per_second.unwrap(), @@ -5497,42 +5603,66 @@ mod tests { } } - /// A node's usage adds up to its candidate's: CPU, scanned bytes, - /// average and peak memory and disk; peak cores and latency are the - /// candidate's own. + /// The candidate's usage is what its nodes are charged for: priced at + /// one unit for a single resource under cost model 1 (CPU, scanned + /// bytes, memory byte-seconds or disk), the nodes' prices add up to the + /// candidate's usage of it. #[test] - fn node_usage_adds_up_to_the_candidates() { + fn node_prices_add_up_to_the_candidates_usage() { + let only = |set: fn(&mut ResourceUnitCost)| { + let mut prices = ResourceUnitCost { + cost_per_cpu_op: 0.0, + cost_per_scanned_byte: 0.0, + cost_per_memory_byte_second: 0.0, + cost_per_provisioned_memory_byte_second: 0.0, + cost_per_provisioned_cpu_core_second: 0.0, + cost_per_disk_byte_second: 0.0, + }; + set(&mut prices); + PlanSelectionSettings { + cost_model: CostModel::UsageOverTime(prices), + ..PlanSelectionSettings::default() + } + }; + // Each resource's unit price and the candidate's usage of it. + type Resource = (fn(&mut ResourceUnitCost), fn(&ResourceUsage) -> f64); + let resources: [Resource; 4] = [ + (|p| p.cost_per_cpu_op = 1.0, |u| u.cpu_ops_per_second), + ( + |p| p.cost_per_scanned_byte = 1.0, + |u| u.scanned_bytes_per_second, + ), + ( + |p| p.cost_per_memory_byte_second = 1.0, + |u| u.average_memory_bytes, + ), + ( + |p| p.cost_per_disk_byte_second = 1.0, + |u| u.disk_bytes as f64, + ), + ]; for candidate in pane_candidates(10_000.0) { - let cost = price( - &candidate.dag, - &[repeating(MINUTE)], - &arriving(10_000.0), - &PlanSelectionSettings::default(), - ) - .unwrap(); - let nodes = cost.per_node.values().map(|n| &n.usage); - let sum = |f: fn(&NodeUsage) -> f64| nodes.clone().map(f).sum::(); - assert!(close( - cost.usage.cpu_ops_per_second, - sum(|u| u.cpu_ops_per_second) - )); - assert!(close( - cost.usage.scanned_bytes_per_second, - sum(|u| u.scanned_bytes_per_second) - )); - assert!(close( - cost.usage.average_memory_bytes, - sum(|u| u.average_memory_bytes) - )); - assert_eq!( - cost.usage.peak_memory_bytes, - nodes.clone().map(|u| u.peak_memory_bytes).sum::() - ); - assert_eq!( - cost.usage.disk_bytes, - nodes.clone().map(|u| u.disk_bytes).sum::() - ); - assert_eq!(cost.usage.latency_per_query.len(), 1); + for (set, used) in resources { + let cost = price( + &candidate.dag, + &[repeating(MINUTE)], + &arriving(10_000.0), + &only(set), + ) + .unwrap(); + let charged: f64 = cost + .per_node + .values() + .filter_map(|n| n.cost_per_second) + .sum(); + assert!( + close(charged, used(&cost.usage)), + "{}: {charged} vs {}", + candidate.id, + used(&cost.usage) + ); + assert_eq!(cost.usage.latency_per_query.len(), 1); + } } } @@ -5704,6 +5834,7 @@ mod tests { fn state_on_disk_is_counted_as_disk_not_memory() { use asap_types::ir::Medium; let settings = unit_costs(|c| c.cost_per_disk_byte_second = 1e-9); + let work_only = unit_costs(|c| c.cost_per_memory_byte_second = 0.0); let state = |dag: &PhysicalASAPDAG| node_of(dag, is_build); for (on_disk, in_memory) in [ (Materialization::MatAtIngestion(Medium::Disk), AT_INGESTION), @@ -5712,22 +5843,31 @@ mod tests { let (disk_dag, memory_dag) = (kll_dag(on_disk), kll_dag(in_memory)); let disk = price(&disk_dag, &recurrences(1), &data(), &settings).unwrap(); let memory = price(&memory_dag, &recurrences(1), &data(), &settings).unwrap(); - let (on_disk_node, in_memory_node) = ( - &disk.per_node[&state(&disk_dag)], - &memory.per_node[&state(&memory_dag)], + // The bytes kept are the same, on disk instead of in memory. + let bytes = disk.usage.disk_bytes - memory.usage.disk_bytes; + assert!(bytes > 0); + assert!( + memory.per_node[&state(&memory_dag)] + .detail + .contains(&format!(" {bytes} bytes")), + "{in_memory:?}: {bytes}" ); - assert_eq!(on_disk_node.usage.average_memory_bytes, 0.0, "{on_disk:?}"); - assert_eq!(on_disk_node.usage.peak_memory_bytes, 0, "{on_disk:?}"); assert!( - in_memory_node.usage.average_memory_bytes > 0.0, - "{in_memory:?}" + disk.usage.average_memory_bytes < memory.usage.average_memory_bytes, + "{on_disk:?}" ); - // The bytes kept are the same, on disk instead of in memory. - let bytes = on_disk_node.usage.disk_bytes; - assert!(bytes > 0); - assert_eq!(bytes, in_memory_node.usage.peak_memory_bytes, "{on_disk:?}"); - let work = on_disk_node.usage.cpu_ops_per_second * 1e-6 - + on_disk_node.usage.scanned_bytes_per_second * 1e-7; + assert_eq!( + disk.usage.peak_memory_bytes + bytes, + memory.usage.peak_memory_bytes, + "{on_disk:?}" + ); + // On disk, the state costs only its disk price beside the work. + let work = price(&memory_dag, &recurrences(1), &data(), &work_only) + .unwrap() + .per_node[&state(&memory_dag)] + .cost_per_second + .unwrap(); + let on_disk_node = &disk.per_node[&state(&disk_dag)]; assert!( close( on_disk_node.cost_per_second.unwrap(), @@ -5805,17 +5945,28 @@ mod tests { &PlanSelectionSettings::default(), ) .unwrap(); - let node = &cost.per_node[&chain]; + let detail = &cost.per_node[&chain].detail; let workers = cost.usage.peak_cpu_cores - 1; assert!(workers > 1, "{workers}"); - let kept = node.usage.average_memory_bytes as u64; - let window = kept / 6; - assert_eq!( - node.usage.peak_memory_bytes, - kept + u64::from(workers - 1) * window, - "{}", - node.detail + // The chain keeps 6 panes (5 read and the one being built). + let kept: u64 = detail + .split("; keeps ") + .nth(1) + .and_then(|rest| rest.split(' ').next()) + .and_then(|bytes| bytes.parse().ok()) + .unwrap_or_else(|| panic!("{detail}")); + let copies = workers - 1; + let bytes = u64::from(copies) * (kept / 6); + assert!( + detail.contains(&format!( + "; {copies} partial copies of its newest window, {bytes} bytes" + )), + "{detail}" ); + // At the peak, not on average: the kept bytes are B1's only + // averaged memory beside negligible query-time holds. + assert!(cost.usage.peak_memory_bytes >= kept + bytes); + assert!(cost.usage.average_memory_bytes < (kept + bytes) as f64); } /// A candidate with the given latency, peak memory and disk under cost diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index 454a18ad8..a787ab4aa 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -20,8 +20,9 @@ // then the summary-capability rule's, one per choice of shared summary // builds), in enumeration order. Every candidate is optimized and // priced; the document shows the first `--max-displayed` (default 64) -// and the selected one, with `combinations` (how many exist) and -// `capped` (whether some are not shown). A candidate that cannot be +// and the selected one, with `combinations` (how many exist), +// `capped` (whether some are not shown) and `pass1_pruned` (the +// candidates Pass 1 pruned per target, with reasons). A candidate that cannot be // composed carries `error` instead of `dag`; // - stage2_physical_asap_optimization: one physical candidate per composed logical // candidate (operator implementation; every sub-DAG NotMaterialized), no @@ -68,18 +69,23 @@ // `pareto-front`). Cost models 1 and 2 use the illustrative unit costs. // Each shown candidate's `usage` lists what it uses; `cost_per_second` is null under // cost model 3, and `front` lists the Pareto front under `pareto-front`. -// `--deployment ''` declares what the deployment's executor can run -// and keep (`DeploymentCapabilities`): `{"supported_summaries": [{"family": -// "Kll", "readouts": ["Quantile"]}, ...], "raw_data_retained": true, -// "query_time_retention": false}`. Every field is optional: by default every -// sketch family and readout the planner can build is supported -// (`supported_summaries` null), raw data is retained, and no query-time -// result is kept across evaluations. Pass 1 offers no summary the -// deployment does not support (exact computations are always supported); -// without raw data, a candidate that scans raw data at query time is -// invalid; with `query_time_retention`, a tumbling-pane chain whose readers -// repeat at its pane width may be built at query time and kept across -// evaluations. The document's top-level `deployment` shows the inputs used. +// `--deployment-capabilities ''` declares what the deployment's +// executor can run and keep (`DeploymentCapabilities`): +// `{"supported_summaries": [{"family": "Kll", "readouts": ["Quantile"]}, ...], +// "raw_data_retained": true, "query_time_results_retained": false}`. Every +// field is optional: by default every sketch family and readout the planner +// can build is supported (`supported_summaries` null), raw data is retained, +// and no query-time result is kept across evaluations. Pass 1 offers no +// summary the deployment does not support (exact computations are always +// supported); Stage 1's `pass1_pruned` lists, per target with any, every +// candidate Pass 1 pruned and why (e.g. `UnsupportedByDeployment`), its fused +// computation's under `fused_pruned`. Without raw data retained, a candidate +// that scans raw data at query time is invalid, and stored data (at rest or +// mixed) is refused as a contradiction; with `query_time_results_retained`, +// a tumbling-pane chain whose readers evaluate together at its pane width is +// also offered built at query time and kept across evaluations, beside being +// shared within one evaluation. The document's top-level +// `deployment_capabilities` shows the inputs used. // `--sample-interval-ms` is the sample interval declared for every metric the // PromQL queries read (default 15000); the data declares no other facts. // `--table ''` (repeatable) declares a table the SQL queries read: `{"name": ..., @@ -90,12 +96,15 @@ use std::collections::{BTreeSet, HashSet}; +use asap_aware_mapping::per_computation_candidates::{ + ComputationCandidate, PerComputationCandidates, PruneReason, +}; use asap_aware_mapping::stage3_plan_selection::{ ClusterLimits, ClusterObjective, CostModel, PlanSelectionSettings, Rejection, ResourceUnitCost, ResourceUsage, Selection, SelectionError, }; use asap_aware_mapping::DeploymentCapabilities; -use asap_devtools::candidate_labels::candidate_labels; +use asap_devtools::candidate_labels::{candidate_labels, computation_label, query_of_each_target}; use asap_devtools::planner_layering_examples::{ example1, example2, pattern_a, pattern_b, PatternARecurrence, SHARED_ROWS_PER_SECOND, }; @@ -120,7 +129,7 @@ const USAGE: &str = [--name ]... [--table ...] [--epsilon --delta --metric ] [--sample-interval-ms ])) \ [--cost-model usage-over-time|provisioned-capacity|cluster-capacity [--cluster-memory-bytes \ --cluster-disk-bytes [--cluster-cpu-cores ] --cluster-objective lowest-latency|least-memory|pareto-front]] \ -[--deployment ] [--max-combinations ] [--max-displayed ] --out "; +[--deployment-capabilities ] [--max-combinations ] [--max-displayed ] --out "; /// The default `--max-combinations`: the most Stage 1 candidates (Pass 2's /// included) a run enumerates. @@ -180,6 +189,7 @@ fn document_of( "stage1_logical_asap_optimization": { "combinations": 0, "capped": false, + "pass1_pruned": [], "candidates": [], "error": format!("Pass 1: {error}"), }, @@ -206,7 +216,7 @@ fn run(args: Vec) -> Result<(), Failure> { let mut cost_model: Option = None; let (mut cluster_memory, mut cluster_disk, mut cluster_cores) = (None, None, None); let mut cluster_objective: Option = None; - let mut deployment = DeploymentCapabilities::default(); + let mut deployment_capabilities = DeploymentCapabilities::default(); let mut out = None; let mut args = args.into_iter(); while let Some(flag) = args.next() { @@ -255,9 +265,9 @@ fn run(args: Vec) -> Result<(), Failure> { cluster_cores = Some(value()?.parse::().map_err(|e| format!("{e}"))?) } "--cluster-objective" => cluster_objective = Some(value()?), - "--deployment" => { - deployment = - serde_json::from_str(&value()?).map_err(|e| format!("--deployment: {e}"))? + "--deployment-capabilities" => { + deployment_capabilities = serde_json::from_str(&value()?) + .map_err(|e| format!("--deployment-capabilities: {e}"))? } "--out" => out = Some(value()?), other => return Err(format!("unknown argument {other}").into()), @@ -336,7 +346,7 @@ fn run(args: Vec) -> Result<(), Failure> { &data, Some(max_combinations), &plan_selection, - &deployment, + &deployment_capabilities, ); document_of(workload_queries(&workload), &roots, result, max_displayed)? } @@ -404,14 +414,15 @@ fn run(args: Vec) -> Result<(), Failure> { &data, Some(max_combinations), &plan_selection, - &deployment, + &deployment_capabilities, ); document_of(described, &roots, result, max_displayed)? } _ => return Err("give exactly one of --example or --promql/--sql queries".into()), }; // Read-only: the deployment inputs the run planned with. - document["deployment"] = serde_json::to_value(&deployment).map_err(|e| e.to_string())?; + document["deployment_capabilities"] = + serde_json::to_value(&deployment_capabilities).map_err(|e| e.to_string())?; let text = serde_json::to_string_pretty(&document).map_err(|e| e.to_string())? + "\n"; std::fs::write(&out, text).map_err(|e| format!("{out}: {e}").into()) } @@ -479,6 +490,7 @@ fn stage_pipeline(queries: Vec, run: &PlanningStagesRun, max_displayed: u "stage1_logical_asap_optimization": { "combinations": run.stage1_logical_asap_optimization.len(), "capped": candidates.len() < run.stage1_logical_asap_optimization.len(), + "pass1_pruned": pass1_pruned_json(&run.stage1_pass1_per_computation), "candidates": candidates, }, "stage2_physical_asap_optimization": { "candidates": physical, "pruned": pruned }, @@ -486,6 +498,39 @@ fn stage_pipeline(queries: Vec, run: &PlanningStagesRun, max_displayed: u }) } +/// The candidates Pass 1 pruned, per target that has any: the target's +/// index, the query it is listed under, and each pruned candidate of its +/// own (`pruned`) and of its fused computation (`fused_pruned`) with the +/// reason, e.g. `UnsupportedByDeployment`. +fn pass1_pruned_json(inventory: &PerComputationCandidates) -> Vec { + let pruned_json = |pruned: &[(ComputationCandidate, PruneReason)]| -> Vec { + pruned + .iter() + .map(|(candidate, reason)| { + json!({ "candidate": computation_label(candidate), "reason": format!("{reason:?}") }) + }) + .collect() + }; + let query_of_target = query_of_each_target(inventory); + inventory + .targets + .iter() + .enumerate() + .filter_map(|(index, target)| { + let fused_pruned = target.fused.as_ref().map_or(&[][..], |f| &f.pruned[..]); + if target.choices.pruned.is_empty() && fused_pruned.is_empty() { + return None; + } + Some(json!({ + "target": index, + "query": format!("Q{}", query_of_target[index] + 1), + "pruned": pruned_json(&target.choices.pruned), + "fused_pruned": pruned_json(fused_pruned), + })) + }) + .collect() +} + /// Plan selection over every candidate, listing the costs and rejections /// of the `shown` ones. fn plan_selection_json( diff --git a/crates/devtools/src/candidate_labels.rs b/crates/devtools/src/candidate_labels.rs index 05234f822..65a96129f 100644 --- a/crates/devtools/src/candidate_labels.rs +++ b/crates/devtools/src/candidate_labels.rs @@ -45,10 +45,10 @@ pub fn candidate_labels(run: &PlanningStagesRun) -> BTreeMap { labels } -/// A Pass 1 choice, e.g. "Q1 exact · Q2 CMS+heap" (exact accumulators in -/// parentheses), each target listed under the first query that reads it. -pub fn choice_label(inventory: &PerComputationCandidates, choice: &[usize]) -> String { - let query_of_target: Vec = inventory +/// The query (index into `inventory.roots`) each target of `inventory` is +/// listed under: the first that reads it. +pub fn query_of_each_target(inventory: &PerComputationCandidates) -> Vec { + inventory .targets .iter() .map(|target| { @@ -64,7 +64,25 @@ pub fn choice_label(inventory: &PerComputationCandidates, choice: &[usiz }) .expect("every target is reached from a root") }) - .collect(); + .collect() +} + +/// A Pass 1 per-computation candidate, e.g. "exact", "Sum acc" or "Kll". +pub fn computation_label(candidate: &ComputationCandidate) -> String { + match candidate { + ComputationCandidate::Exact => "exact".to_string(), + ComputationCandidate::ExactAccumulator(kinds) => { + let kinds: Vec = kinds.iter().map(|kind| format!("{kind:?}")).collect(); + format!("{} acc", kinds.join("/")) + } + ComputationCandidate::Summary(algorithm) => sketch_name(algorithm), + } +} + +/// A Pass 1 choice, e.g. "Q1 exact · Q2 CMS+heap" (exact accumulators in +/// parentheses), each target listed under the first query that reads it. +pub fn choice_label(inventory: &PerComputationCandidates, choice: &[usize]) -> String { + let query_of_target = query_of_each_target(inventory); // Each target's chosen candidate, with the target it absorbs when the // candidate is its fused computation's. let chosen: Vec<_> = inventory @@ -339,6 +357,7 @@ pub fn physical_label(logical: &str, candidate: &PhysicalCandidate) -> String { Materialization::NotMaterialized => continue, Materialization::MatAtIngestion(_) => "ingestion time", Materialization::MatAtQuery(_) => "query time, shared", + Materialization::MatAtQueryKeptAcrossEvaluations(_) => "query time, kept", }; let node = candidate .dag diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index bbf428ddb..6242530c2 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -198,7 +198,7 @@ pub fn run_planning_stages_over( data, PlanningModels::builtin() .with_plan_selection(plan_selection) - .with_deployment(deployment), + .with_deployment_capabilities(deployment), ); for candidate in &run.stage1_logical_asap_optimization { if let Ok(roots) = &candidate.roots { diff --git a/crates/devtools/tests/stage_pipeline.rs b/crates/devtools/tests/stage_pipeline.rs index 6686dffa7..65f5be04a 100644 --- a/crates/devtools/tests/stage_pipeline.rs +++ b/crates/devtools/tests/stage_pipeline.rs @@ -710,39 +710,40 @@ fn cost_model_flag_chooses_how_candidates_are_compared() { assert!(cluster["costs"]["P3-m1"]["cost_per_second"].is_null()); } -/// `--deployment` declares the executor's capabilities, which the -/// document's `deployment` shows (defaults filled in; without the flag, -/// every summary supported, raw data retained, no query-time retention). -/// For Pattern B with KLL only and query-time retention: no DDSketch -/// candidate is offered, and the KLL panes gain B3, built at query time and +/// `--deployment-capabilities` declares the executor's capabilities, which +/// the document's `deployment_capabilities` shows (defaults filled in; +/// without the flag, every summary supported, raw data retained, no +/// query-time result retained). For Pattern B with KLL only and query-time +/// results retained: no DDSketch candidate is offered, and Stage 1's +/// `pass1_pruned` says why; the KLL panes gain B3, built at query time and /// kept (`P3-m2`, 8,884.53 cost/s); the whole-window KLL (`P2`) is still /// selected. Without raw data, every query-time scan is invalid and B1 /// (`P3-m1`) is selected. A deployment with an unknown field is refused. #[test] -fn deployment_flag_sets_the_capabilities_and_shows_them() { +fn deployment_capabilities_flag_sets_the_capabilities_and_shows_them() { let default = generate_example("planner-layering-3b", &[]); assert_eq!( - default["deployment"], + default["deployment_capabilities"], serde_json::json!({ "supported_summaries": null, "raw_data_retained": true, - "query_time_retention": false, + "query_time_results_retained": false, }) ); let kll_kept = generate_example( "planner-layering-3b", &[ - "--deployment", + "--deployment-capabilities", r#"{"supported_summaries": [{"family": "Kll", "readouts": ["Quantile"]}], - "query_time_retention": true}"#, + "query_time_results_retained": true}"#, ], ); assert_eq!( - kll_kept["deployment"], + kll_kept["deployment_capabilities"], serde_json::json!({ "supported_summaries": [{ "family": "Kll", "readouts": ["Quantile"] }], "raw_data_retained": true, - "query_time_retention": true, + "query_time_results_retained": true, }) ); let physical: Vec<&str> = kll_kept["stage2_physical_asap_optimization"]["candidates"] @@ -752,13 +753,30 @@ fn deployment_flag_sets_the_capabilities_and_shows_them() { .map(|c| c["id"].as_str().unwrap()) .collect(); assert_eq!(physical, ["P1", "P2", "P3", "P3-m1", "P3-m2"]); + // Pass 1 lists why DDSketch is gone. + assert_eq!( + kll_kept["stage1_logical_asap_optimization"]["pass1_pruned"], + serde_json::json!([{ + "target": 0, + "query": "Q1", + "pruned": [{ "candidate": "DDSketch", "reason": "UnsupportedByDeployment" }], + "fused_pruned": [], + }]) + ); + assert_eq!( + default["stage1_logical_asap_optimization"]["pass1_pruned"], + serde_json::json!([]) + ); let selection = &kll_kept["stage3_plan_selection"]; assert_eq!(selection["selected"], "P2"); assert_eq!(selection["costs"]["P3-m2"]["cost_per_second"], 8884.53); let no_raw = generate_example( "planner-layering-3b", - &["--deployment", r#"{"raw_data_retained": false}"#], + &[ + "--deployment-capabilities", + r#"{"raw_data_retained": false}"#, + ], ); let selection = &no_raw["stage3_plan_selection"]; assert_eq!(selection["selected"], "P3-m1"); @@ -776,12 +794,12 @@ fn deployment_flag_sets_the_capabilities_and_shows_them() { let (status, stderr) = failure(&[ "--example", "planner-layering-3b", - "--deployment", + "--deployment-capabilities", r#"{"raw_data": false}"#, ]); assert_eq!(status, Some(2)); assert!( - stderr.contains("--deployment: unknown field `raw_data`"), + stderr.contains("--deployment-capabilities: unknown field `raw_data`"), "{stderr}" ); } diff --git a/crates/integration-tests/tests/deployment_capabilities.rs b/crates/integration-tests/tests/deployment_capabilities.rs index 6f02b5f4b..c83e7427b 100644 --- a/crates/integration-tests/tests/deployment_capabilities.rs +++ b/crates/integration-tests/tests/deployment_capabilities.rs @@ -8,6 +8,7 @@ use asap_aware_mapping::stage3_plan_selection::PlanSelectionSettings; use asap_aware_mapping::{DeploymentCapabilities, SupportedSummary}; use asap_devtools::planning_stages::lower_sql_roots; use asap_types::ir::flat::FlatDag; +use asap_types::ir::{ASAPOp, Materialization, Operator}; use asap_types::post_asap::{SketchAlgorithm, StatisticKind}; use planner_layering_common::*; @@ -129,3 +130,93 @@ fn assert_removes_exactly( assert_eq!(found, expected, "{name}"); assert_selects_cheapest_valid(restricted); } + +/// Pattern B with a p50 panel beside its p99, both every minute over the +/// same 5-min window. One 1-min tumbling KLL chain serves both panels. +fn pattern_b_with_p50() -> asap_types::workload::PlanningWorkload { + let mut workload = pattern_b(); + let repeating = workload + .query_workload + .repeating_queries + .as_mut() + .expect("pattern B repeats"); + let mut p50 = repeating[0].clone(); + p50.query = asap_types::workload::Query("quantile_over_time(0.5, latency_ms[5m])".into()); + repeating.push(p50); + workload +} + +/// Physical candidate ids with their cost per second. +type CandidateCosts = Vec<(String, f64)>; + +/// The per-second cost of every physical candidate whose pane chain both +/// panels read is materialized at query time: shared within one evaluation +/// (`MatAtQuery`), and kept across evaluations +/// (`MatAtQueryKeptAcrossEvaluations`, which Stage 3 prices as such). +fn shared_chain_costs(run: &Run) -> (CandidateCosts, CandidateCosts) { + let (mut within_one, mut kept) = (Vec::new(), Vec::new()); + for p in &run.physical { + for n in &p.dag().nodes { + let shared_chain = matches!(n.payload, Operator::ASAP(ASAPOp::PaneChain { .. })) + && readers(p.dag(), &p.query_roots, n.id).len() == 2; + let Some(cost) = run.selection.costs.get(&p.id).filter(|_| shared_chain) else { + continue; + }; + let entry = (p.id.clone(), cost.cost_per_second.expect("cost model 1")); + let kept_detail = cost.per_node[&n.id].detail.contains("across evaluations"); + match n.materialization { + Materialization::MatAtQuery(_) => within_one.push(entry), + Materialization::MatAtQueryKeptAcrossEvaluations(_) if kept_detail => { + kept.push(entry) + } + Materialization::MatAtQueryKeptAcrossEvaluations(_) => { + panic!("{}: a kept chain priced as not kept", p.id) + } + _ => {} + } + } + } + (within_one, kept) +} + +/// With query-time results retained, a chain two panels read is offered +/// both ways: shared within one evaluation (rebuilt from the 5-min window +/// every minute, then reused by both panels) and kept across evaluations +/// (B3: one new pane per evaluation, the 4 older panes held in memory +/// between evaluations). Holding 1M per-series KLLs between evaluations +/// costs far more than rebuilding them, so the shared option is cheaper +/// and the kept one is not selected. Without query-time retention, the +/// kept option is not offered. +#[test] +fn query_time_results_retained_still_offers_the_chain_shared_within_one_evaluation() { + let keeping = DeploymentCapabilities { + query_time_results_retained: true, + ..DeploymentCapabilities::default() + }; + let run = run_promql_for(&pattern_b_with_p50(), &keeping); + let (within_one, kept) = shared_chain_costs(&run); + assert!(!within_one.is_empty(), "no shared option: kept {kept:?}"); + assert!(!kept.is_empty(), "no kept option: shared {within_one:?}"); + let cheapest = |options: &[(String, f64)]| { + options + .iter() + .map(|(_, cost)| *cost) + .fold(f64::INFINITY, f64::min) + }; + assert!( + cheapest(&within_one) < cheapest(&kept), + "shared {within_one:?}, kept {kept:?}" + ); + assert!( + kept.iter().all(|(id, _)| *id != run.selection.selected), + "{}", + run.selection.selected + ); + assert_selects_cheapest_valid(&run); + + let (_, kept) = shared_chain_costs(&run_promql_for( + &pattern_b_with_p50(), + &DeploymentCapabilities::default(), + )); + assert!(kept.is_empty(), "{kept:?}"); +} diff --git a/crates/integration-tests/tests/planner_layering_common/mod.rs b/crates/integration-tests/tests/planner_layering_common/mod.rs index 856664d1f..e5985adbb 100644 --- a/crates/integration-tests/tests/planner_layering_common/mod.rs +++ b/crates/integration-tests/tests/planner_layering_common/mod.rs @@ -620,8 +620,9 @@ pub fn evaluation_interval_ms(workload: &PlanningWorkload, query: usize) -> Opti pub enum MaterializationOption { /// Runs as data arrives; output stored before any query asks. MatAtIngestion, - /// Runs when a query first needs it; output kept for later executions - /// and other queries of the batch. + /// Runs when a query first needs it; output reused by the other + /// queries of the evaluation, or, as `MatAtQueryKeptAcrossEvaluations`, + /// kept for later evaluations. MatAtQuery, /// Runs at query time for each execution; output discarded. NotMaterialized, @@ -633,7 +634,9 @@ pub fn materialization(p: &Physical, node: NodeId) -> MaterializationOption { let n = p.dag().nodes.iter().find(|n| n.id == node).expect("node"); match n.materialization { Materialization::MatAtIngestion(_) => MaterializationOption::MatAtIngestion, - Materialization::MatAtQuery(_) => MaterializationOption::MatAtQuery, + Materialization::MatAtQuery(_) | Materialization::MatAtQueryKeptAcrossEvaluations(_) => { + MaterializationOption::MatAtQuery + } Materialization::NotMaterialized => MaterializationOption::NotMaterialized, } } @@ -693,7 +696,7 @@ pub fn window_form(dag: &impl ExportedDag, build: NodeId) -> WindowForm { /// How far back from an evaluation a materialized node's kept output /// reaches in event time, derived the way plan selection prices what it -/// keeps (the IR does not record retention): a pane chain keeps its `N` +/// keeps: a pane chain keeps its `N` /// panes of width `w`, plus the one being built when maintained at /// ingestion time; a whole-window build keeps its window, and the next one /// being built at ingestion time; each plus the offset of the time shifts diff --git a/crates/integration-tests/tests/planner_layering_example4.rs b/crates/integration-tests/tests/planner_layering_example4.rs index c9851b5fd..95d1de173 100644 --- a/crates/integration-tests/tests/planner_layering_example4.rs +++ b/crates/integration-tests/tests/planner_layering_example4.rs @@ -273,7 +273,7 @@ fn stage3_plan_selection_a_monthly_amortizes_ingestion_time_maintenance() { /// needs; the default deployment keeps none. fn keeping_query_time_results() -> DeploymentCapabilities { DeploymentCapabilities { - query_time_retention: true, + query_time_results_retained: true, ..DeploymentCapabilities::default() } } diff --git a/crates/types/src/ir/timing.rs b/crates/types/src/ir/timing.rs index 075a821d4..c496df143 100644 --- a/crates/types/src/ir/timing.rs +++ b/crates/types/src/ir/timing.rs @@ -3,8 +3,9 @@ //! edges. //! //! The logical DAG carries no timing. Materialization (#509 -//! §Materialization) gives each sub-DAG one of three options: -//! `NotMaterialized`, `MatAtIngestion(medium)` or `MatAtQuery(medium)`. +//! §Materialization) gives each sub-DAG one of its options: +//! `NotMaterialized`, `MatAtIngestion(medium)`, `MatAtQuery(medium)` or +//! `MatAtQueryKeptAcrossEvaluations(medium)`. //! [`MaterializationAssignment`] records that option per sub-DAG, named by //! the node whose output would be stored (any node: a summary state, a //! population, or e.g. the tumbling-window panes of #509 Example 4), and @@ -12,7 +13,8 @@ //! //! - `SummaryEstimate` and `EvaluatePopulation` run at query time; //! - a node assigned `MatAtIngestion` runs at ingestion time, and one -//! assigned `MatAtQuery` at query time. A `SummaryAgg` or +//! assigned `MatAtQuery` or `MatAtQueryKeptAcrossEvaluations` at query +//! time. A `SummaryAgg` or //! `MaintainPopulation` without an assignment is `NotMaterialized` and runs //! at query time. Every node upstream of an ingestion-time node also runs //! at ingestion time, so `MatAtIngestion` (or an ingestion timing fixed by diff --git a/crates/types/src/post_asap/execution_data_state.rs b/crates/types/src/post_asap/execution_data_state.rs index dc4240d51..03c982252 100644 --- a/crates/types/src/post_asap/execution_data_state.rs +++ b/crates/types/src/post_asap/execution_data_state.rs @@ -51,9 +51,15 @@ pub enum Materialization { NotMaterialized, /// Runs as data arrives; its output is stored until a query reads it. MatAtIngestion(Medium), - /// Runs when a query first needs it; its output is stored for later - /// executions and other queries of the batch. + /// Runs when a query first needs it; its output is reused by the other + /// queries of the same evaluation and discarded after it. MatAtQuery(Medium), + /// Runs at query time and keeps its output across evaluations: a + /// tumbling-pane chain whose readers evaluate together every pane + /// width builds its newest pane at each evaluation and keeps the older + /// ones for the next (#509 Example 4's B3). Offered only when the + /// deployment keeps query-time results. + MatAtQueryKeptAcrossEvaluations(Medium), } impl Materialization { @@ -61,7 +67,9 @@ impl Materialization { pub fn timing(self) -> ExecutionTiming { match self { Self::MatAtIngestion(_) => ExecutionTiming::IngestionTime, - Self::NotMaterialized | Self::MatAtQuery(_) => ExecutionTiming::QueryTime, + Self::NotMaterialized + | Self::MatAtQuery(_) + | Self::MatAtQueryKeptAcrossEvaluations(_) => ExecutionTiming::QueryTime, } } } diff --git a/crates/types/src/post_asap/family_capability.rs b/crates/types/src/post_asap/family_capability.rs index 6ef7ea170..fec977dbf 100644 --- a/crates/types/src/post_asap/family_capability.rs +++ b/crates/types/src/post_asap/family_capability.rs @@ -29,7 +29,7 @@ pub struct FamilyCapability { /// The kind of a `SummaryEstimate` statistic, without its arguments: the /// readout vocabulary of this table and of a deployment's supported /// summaries. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub enum StatisticKind { /// The total of the updates: a bare `PointCount` (no item value). Total, diff --git a/tools/dag-viewer/examples/planner-layering-example1.json b/tools/dag-viewer/examples/planner-layering-example1.json index 88811b081..80082ab92 100644 --- a/tools/dag-viewer/examples/planner-layering-example1.json +++ b/tools/dag-viewer/examples/planner-layering-example1.json @@ -1,6 +1,6 @@ { - "deployment": { - "query_time_retention": false, + "deployment_capabilities": { + "query_time_results_retained": false, "raw_data_retained": true, "supported_summaries": null }, @@ -22323,7 +22323,8 @@ } ], "capped": false, - "combinations": 27 + "combinations": 27, + "pass1_pruned": [] }, "stage2_physical_asap_optimization": { "candidates": [